org.apache.iceberg.TableMetadataParser Java Examples

private Path getMetadataFile(int metadataVersion) throws IOException {
  for (TableMetadataParser.Codec codec : TableMetadataParser.Codec.values()) {
    Path metadataFile = metadataFilePath(metadataVersion, codec);
    FileSystem fs = getFileSystem(metadataFile, conf);
    if (fs.exists(metadataFile)) {
      return metadataFile;
    }

    if (codec.equals(TableMetadataParser.Codec.GZIP)) {
      // we have to be backward-compatible with .metadata.json.gz files
      metadataFile = oldMetadataFilePath(metadataVersion, codec);
      fs = getFileSystem(metadataFile, conf);
      if (fs.exists(metadataFile)) {
        return metadataFile;
      }
    }
  }

  return null;
}
 

private String writeNewMetadata(TableMetadata metadata, int newVersion)
{
    String newTableMetadataFilePath = newTableMetadataFilePath(metadata, newVersion);
    OutputFile newMetadataLocation = fileIo.newOutputFile(newTableMetadataFilePath);

    // write the new metadata
    TableMetadataParser.write(metadata, newMetadataLocation);

    return newTableMetadataFilePath;
}
 

private void refreshFromMetadataLocation(String newLocation)
{
    // use null-safe equality check because new tables have a null metadata location
    if (Objects.equals(currentMetadataLocation, newLocation)) {
        shouldRefresh = false;
        return;
    }

    AtomicReference<TableMetadata> newMetadata = new AtomicReference<>();
    Tasks.foreach(newLocation)
            .retry(20)
            .exponentialBackoff(100, 5000, 600000, 4.0)
            .suppressFailureWhenFinished()
            .run(metadataLocation -> newMetadata.set(
                    TableMetadataParser.read(this, io().newInputFile(metadataLocation))));

    String newUUID = newMetadata.get().uuid();
    if (currentMetadata != null) {
        checkState(newUUID == null || newUUID.equals(currentMetadata.uuid()),
                "Table UUID does not match: current=%s != refreshed=%s", currentMetadata.uuid(), newUUID);
    }

    currentMetadata = newMetadata.get();
    currentMetadataLocation = newLocation;
    version = parseVersion(newLocation);
    shouldRefresh = false;
}
 

private synchronized void updateVersionAndMetadata(int newVersion, String metadataFile) {
  // update if the current version is out of date
  if (version == null || version != newVersion) {
    this.version = newVersion;
    this.currentMetadata = checkUUID(currentMetadata, TableMetadataParser.read(io(), metadataFile));
  }
}
 

/**
 * get iceberg table.
 *
 * @param tableName             table name
 * @param tableMetadataLocation table metadata location
 * @param includeInfoDetails    if true, will include more details like the manifest file content
 * @return iceberg table
 */
public IcebergTableWrapper getIcebergTable(final QualifiedName tableName, final String tableMetadataLocation,
                                           final boolean includeInfoDetails) {
    final long start = this.registry.clock().wallTime();
    try {
        this.icebergTableCriteria.checkCriteria(tableName, tableMetadataLocation);
        log.debug("Loading icebergTable {} from {}", tableName, tableMetadataLocation);
        final IcebergMetastoreTables icebergMetastoreTables = new IcebergMetastoreTables(
            new IcebergTableOps(conf, tableMetadataLocation,
                connectorContext.getConfig(),
                icebergTableOpsProxy));
        final Table table = icebergMetastoreTables.loadTable(
            HiveTableUtil.qualifiedNameToTableIdentifier(tableName));
        final Map<String, String> extraProperties = Maps.newHashMap();
        if (includeInfoDetails) {
            extraProperties.put(DirectSqlTable.PARAM_METADATA_CONTENT,
                TableMetadataParser.toJson(icebergMetastoreTables.getTableOps().current()));
        }
        return new IcebergTableWrapper(table, extraProperties);
    } catch (NotFoundException | NoSuchTableException e) {
        throw new InvalidMetaException(tableName, e);
    } finally {
        final long duration = registry.clock().wallTime() - start;
        log.info("Time taken to getIcebergTable {} is {} ms", tableName, duration);
        this.recordTimer(IcebergRequestMetrics.TagLoadTable.getMetricName(), duration);
        this.increaseCounter(IcebergRequestMetrics.TagLoadTable.getMetricName(), tableName);
    }
}
 

protected static List<String> metadataVersionFiles(String tableName) {
  return filterByExtension(tableName, getFileExtension(TableMetadataParser.Codec.NONE));
}
 

@Override
public void commit(TableMetadata base, TableMetadata metadata) {
  Pair<Integer, TableMetadata> current = versionAndMetadata();
  if (base != current.second()) {
    throw new CommitFailedException("Cannot commit changes based on stale table metadata");
  }

  if (base == metadata) {
    LOG.info("Nothing to commit.");
    return;
  }

  Preconditions.checkArgument(base == null || base.location().equals(metadata.location()),
      "Hadoop path-based tables cannot be relocated");
  Preconditions.checkArgument(
      !metadata.properties().containsKey(TableProperties.WRITE_METADATA_LOCATION),
      "Hadoop path-based tables cannot relocate metadata");

  String codecName = metadata.property(
      TableProperties.METADATA_COMPRESSION, TableProperties.METADATA_COMPRESSION_DEFAULT);
  TableMetadataParser.Codec codec = TableMetadataParser.Codec.fromName(codecName);
  String fileExtension = TableMetadataParser.getFileExtension(codec);
  Path tempMetadataFile = metadataPath(UUID.randomUUID().toString() + fileExtension);
  TableMetadataParser.write(metadata, io().newOutputFile(tempMetadataFile.toString()));

  int nextVersion = (current.first() != null ? current.first() : 0) + 1;
  Path finalMetadataFile = metadataFilePath(nextVersion, codec);
  FileSystem fs = getFileSystem(tempMetadataFile, conf);

  try {
    if (fs.exists(finalMetadataFile)) {
      throw new CommitFailedException(
          "Version %d already exists: %s", nextVersion, finalMetadataFile);
    }
  } catch (IOException e) {
    throw new RuntimeIOException(e,
        "Failed to check if next version exists: " + finalMetadataFile);
  }

  // this rename operation is the atomic commit operation
  renameToFinal(fs, tempMetadataFile, finalMetadataFile);

  // update the best-effort version pointer
  writeVersionHint(nextVersion);

  deleteRemovedMetadataFiles(base, metadata);

  this.shouldRefresh = true;
}
 

private Path metadataFilePath(int metadataVersion, TableMetadataParser.Codec codec) {
  return metadataPath("v" + metadataVersion + TableMetadataParser.getFileExtension(codec));
}
 

private Path oldMetadataFilePath(int metadataVersion, TableMetadataParser.Codec codec) {
  return metadataPath("v" + metadataVersion + TableMetadataParser.getOldFileExtension(codec));
}
 

File version(int versionNumber) {
  return new File(metadataDir, "v" + versionNumber + getFileExtension(TableMetadataParser.Codec.NONE));
}
 

TableMetadata readMetadataVersion(int version) {
  return TableMetadataParser.read(new TestTables.TestTableOperations("table", tableDir).io(),
      localInput(version(version)));
}