org.apache.iceberg.exceptions.ValidationException Java Examples

@SuppressWarnings("unchecked")
@Override
public BoundTransform<S, T> bind(Types.StructType struct, boolean caseSensitive) {
  BoundReference<S> boundRef = ref.bind(struct, caseSensitive);

  Transform<S, T> typeTransform;
  try {
    // TODO: Avoid using toString/fromString
    typeTransform = (Transform<S, T>) Transforms.fromString(boundRef.type(), transform.toString());
    ValidationException.check(typeTransform.canTransform(boundRef.type()),
        "Cannot bind: %s cannot transform %s values from '%s'", transform, boundRef.type(), ref.name());
  } catch (IllegalArgumentException e) {
    throw new ValidationException(
        "Cannot bind: %s cannot transform %s values from '%s'", transform, boundRef.type(), ref.name());
  }

  return new BoundTransform<>(boundRef, typeTransform);
}
 

@Test
public void testValidatedOverwriteWithAppendOutsideOfDeleteMetrics() {
  TableMetadata base = TestTables.readMetadata(TABLE_NAME);
  long baseId = base.currentSnapshot().snapshotId();

  OverwriteFiles overwrite = table.newOverwrite()
      .overwriteByRowFilter(and(equal("date", "2018-06-09"), lessThan("id", 10)))
      .addFile(FILE_10_TO_14) // in 2018-06-09 matches, but IDs are outside range
      .validateAddedFilesMatchOverwriteFilter();

  AssertHelpers.assertThrows("Should reject commit with file not matching delete expression",
      ValidationException.class, "Cannot append file with rows that do not match filter",
      overwrite::commit);

  Assert.assertEquals("Should not create a new snapshot",
      baseId, table.currentSnapshot().snapshotId());
}
 

@Override
public void deleteWhere(Filter[] filters) {
  Expression deleteExpr = SparkFilters.convert(filters);
  DeleteFiles delete = icebergTable.newDelete()
      .set("spark.app.id", sparkSession().sparkContext().applicationId())
      .deleteFromRowFilter(deleteExpr);

  String genieId = sparkSession().sparkContext().hadoopConfiguration().get("genie.job.id");
  if (genieId != null) {
    delete.set("genie-id", genieId);
  }

  try {
    delete.commit();
  } catch (ValidationException e) {
    throw new IllegalArgumentException("Failed to cleanly delete data files matching: " + deleteExpr, e);
  }
}
 

@Override
public Filter[] pushFilters(Filter[] filters) {
  List<Expression> expressions = Lists.newArrayListWithExpectedSize(filters.length);
  List<Filter> pushed = Lists.newArrayListWithExpectedSize(filters.length);

  for (Filter filter : filters) {
    Expression expr = SparkFilters.convert(filter);
    if (expr != null) {
      try {
        Binder.bind(table.schema().asStruct(), expr, caseSensitive);
        expressions.add(expr);
        pushed.add(filter);
      } catch (ValidationException e) {
        // binding to the table schema failed, so this expression cannot be pushed down
      }
    }
  }

  this.filterExpressions = expressions;
  this.pushedFilters = pushed.toArray(new Filter[0]);

  // Spark doesn't support residuals per task, so return all filters
  // to get Spark to handle record-level filtering
  return filters;
}
 

@Test
public void testOverwriteIncompatibleAdditionValidated() {
  table.newAppend()
      .appendFile(FILE_DAY_1)
      .commit();

  Snapshot baseSnapshot = table.currentSnapshot();
  validateSnapshot(null, baseSnapshot, FILE_DAY_1);

  OverwriteFiles overwrite = table.newOverwrite()
      .addFile(FILE_DAY_2_MODIFIED)
      .validateNoConflictingAppends(baseSnapshot.snapshotId(), EXPRESSION_DAY_2);

  table.newAppend()
      .appendFile(FILE_DAY_2)
      .commit();
  long committedSnapshotId = table.currentSnapshot().snapshotId();

  AssertHelpers.assertThrows("Should reject commit",
      ValidationException.class, "A file was appended",
      overwrite::commit);

  Assert.assertEquals("Should not create a new snapshot",
      committedSnapshotId, table.currentSnapshot().snapshotId());
}
 

@Test
public void testInvalidConversions() {
  StructType struct = StructType.of(required(16, "f", Types.FloatType.get()));

  for (Expression.Operation op : COMPARISONS) {
    UnboundPredicate<String> unbound = new UnboundPredicate<>(op, ref("f"), "12.40");

    try {
      unbound.bind(struct);
      Assert.fail("Should not convert string to float");
    } catch (ValidationException e) {
      Assert.assertEquals("Should ",
          e.getMessage(),
          "Invalid value for conversion to type float: 12.40 (java.lang.String)");
    }
  }
}
 

@Test
public void testValidationFailure() {
  table.newFastAppend()
      .appendFile(FILE_A)
      .appendFile(FILE_B)
      .commit();

  TableMetadata base = readMetadata();
  long baseId = base.currentSnapshot().snapshotId();

  ReplacePartitions replace = table.newReplacePartitions()
      .addFile(FILE_F)
      .addFile(FILE_G)
      .validateAppendOnly();

  AssertHelpers.assertThrows("Should reject commit with file not matching delete expression",
      ValidationException.class, "Cannot commit file that conflicts with existing partition",
      replace::commit);

  Assert.assertEquals("Should not create a new snapshot",
      baseId, readMetadata().currentSnapshot().snapshotId());
}
 

private boolean manifestHasDeletedFiles(
    StrictMetricsEvaluator metricsEvaluator, ManifestReader<F> reader, StructLikeWrapper partitionWrapper) {
  boolean isDelete = reader.isDeleteManifestReader();
  Evaluator inclusive = inclusiveDeleteEvaluator(reader.spec());
  Evaluator strict = strictDeleteEvaluator(reader.spec());
  boolean hasDeletedFiles = false;
  for (ManifestEntry<F> entry : reader.entries()) {
    F file = entry.file();
    boolean fileDelete = deletePaths.contains(file.path()) ||
        dropPartitions.contains(partitionWrapper.set(file.partition())) ||
        (isDelete && entry.sequenceNumber() > 0 && entry.sequenceNumber() < minSequenceNumber);
    if (fileDelete || inclusive.eval(file.partition())) {
      ValidationException.check(
          fileDelete || strict.eval(file.partition()) || metricsEvaluator.eval(file),
          "Cannot delete file where some, but not all, rows match filter %s: %s",
          this.deleteExpression, file.path());

      hasDeletedFiles = true;
      if (failAnyDelete) {
        throw new DeleteException(reader.spec().partitionToPath(file.partition()));
      }
      break; // as soon as a deleted file is detected, stop scanning
    }
  }
  return hasDeletedFiles;
}
 

@Test
public void testOverwriteCompatibleAdditionStrictValidated() {
  table.newAppend()
      .appendFile(FILE_DAY_2)
      .commit();

  Snapshot baseSnapshot = table.currentSnapshot();
  validateSnapshot(null, baseSnapshot, FILE_DAY_2);

  OverwriteFiles overwrite = table.newOverwrite()
      .deleteFile(FILE_DAY_2)
      .addFile(FILE_DAY_2_MODIFIED)
      .validateNoConflictingAppends(baseSnapshot.snapshotId(), alwaysTrue());

  table.newAppend()
      .appendFile(FILE_DAY_1)
      .commit();
  long committedSnapshotId = table.currentSnapshot().snapshotId();

  AssertHelpers.assertThrows("Should reject commit",
      ValidationException.class, "A file was appended",
      overwrite::commit);

  Assert.assertEquals("Should not create a new snapshot",
      committedSnapshotId, table.currentSnapshot().snapshotId());
}
 

@Test
public void testDeleteNonExistentFile() {
  Assert.assertEquals("Table should start empty", 0, listManifestFiles().size());

  table.newAppend()
      .appendFile(FILE_A)
      .appendFile(FILE_B)
      .commit();

  TableMetadata base = readMetadata();
  Assert.assertEquals("Should create 1 manifest for initial write",
      1, base.currentSnapshot().allManifests().size());

  AssertHelpers.assertThrows("Expected an exception",
      ValidationException.class,
      "Missing required files to delete: /path/to/data-c.parquet",
      () -> table.newRewrite()
          .rewriteFiles(Sets.newSet(FILE_C), Sets.newSet(FILE_D))
          .commit());

  Assert.assertEquals("Only 1 manifests should exist", 1, listManifestFiles().size());
}
 

/**
 * Converts checked exceptions to unchecked exceptions.
 *
 * @param cause a checked exception object which is to be converted to its unchecked equivalent.
 * @param message exception message as a format string
 * @param args format specifiers
 * @return unchecked exception.
 */
public static RuntimeException toUncheckedException(Throwable cause, String message, Object... args) {
  if (cause instanceof RuntimeException) {
    return (RuntimeException) cause;

  } else if (cause instanceof org.apache.spark.sql.catalyst.analysis.NoSuchDatabaseException) {
    return new NoSuchNamespaceException(cause, message, args);

  } else if (cause instanceof org.apache.spark.sql.catalyst.analysis.NoSuchTableException) {
    return new NoSuchTableException(cause, message, args);

  } else if (cause instanceof AnalysisException) {
    return new ValidationException(cause, message, args);

  } else if (cause instanceof IOException) {
    return new RuntimeIOException((IOException) cause, message, args);

  } else {
    return new RuntimeException(String.format(message, args), cause);
  }
}
 

@Test
public void testValidatedOverwriteWithAppendOutsideOfDelete() {
  // ensure the overwrite results in a merge
  table.updateProperties().set(TableProperties.MANIFEST_MIN_MERGE_COUNT, "1").commit();

  TableMetadata base = TestTables.readMetadata(TABLE_NAME);
  long baseId = base.currentSnapshot().snapshotId();

  OverwriteFiles overwrite = table.newOverwrite()
      .overwriteByRowFilter(equal("date", "2018-06-08"))
      .addFile(FILE_10_TO_14) // in 2018-06-09, NOT in 2018-06-08
      .validateAddedFilesMatchOverwriteFilter();

  AssertHelpers.assertThrows("Should reject commit with file not matching delete expression",
      ValidationException.class, "Cannot append file with rows that do not match filter",
      overwrite::commit);

  Assert.assertEquals("Should not create a new snapshot",
      baseId, table.currentSnapshot().snapshotId());
}
 

@Test
public void testValidatedOverwriteWithAppendSuccess() {
  TableMetadata base = TestTables.readMetadata(TABLE_NAME);
  long baseId = base.currentSnapshot().snapshotId();

  OverwriteFiles overwrite = table.newOverwrite()
      .overwriteByRowFilter(and(equal("date", "2018-06-09"), lessThan("id", 20)))
      .addFile(FILE_10_TO_14) // in 2018-06-09 matches and IDs are inside range
      .validateAddedFilesMatchOverwriteFilter();

  AssertHelpers.assertThrows("Should reject commit with file not matching delete expression",
      ValidationException.class, "Cannot append file with rows that do not match filter",
      overwrite::commit);

  Assert.assertEquals("Should not create a new snapshot",
      baseId, table.currentSnapshot().snapshotId());
}
 

public TableMetadata replaceCurrentSnapshot(Snapshot snapshot) {
  // there can be operations (viz. rollback, cherrypick) where an existing snapshot could be replacing current
  if (snapshotsById.containsKey(snapshot.snapshotId())) {
    return setCurrentSnapshotTo(snapshot);
  }

  ValidationException.check(formatVersion == 1 || snapshot.sequenceNumber() > lastSequenceNumber,
      "Cannot add snapshot with sequence number %s older than last sequence number %s",
      snapshot.sequenceNumber(), lastSequenceNumber);

  List<Snapshot> newSnapshots = ImmutableList.<Snapshot>builder()
      .addAll(snapshots)
      .add(snapshot)
      .build();
  List<HistoryEntry> newSnapshotLog = ImmutableList.<HistoryEntry>builder()
      .addAll(snapshotLog)
      .add(new SnapshotLogEntry(snapshot.timestampMillis(), snapshot.snapshotId()))
      .build();

  return new TableMetadata(null, formatVersion, uuid, location,
      snapshot.sequenceNumber(), snapshot.timestampMillis(), lastColumnId, schema, defaultSpecId, specs, properties,
      snapshot.snapshotId(), newSnapshots, newSnapshotLog, addPreviousFile(file, lastUpdatedMillis));
}
 

private TableMetadata setCurrentSnapshotTo(Snapshot snapshot) {
  ValidationException.check(snapshotsById.containsKey(snapshot.snapshotId()),
      "Cannot set current snapshot to unknown: %s", snapshot.snapshotId());
  ValidationException.check(formatVersion == 1 || snapshot.sequenceNumber() <= lastSequenceNumber,
      "Last sequence number %s is less than existing snapshot sequence number %s",
      lastSequenceNumber, snapshot.sequenceNumber());

  if (currentSnapshotId == snapshot.snapshotId()) {
    // change is a noop
    return this;
  }

  long nowMillis = System.currentTimeMillis();
  List<HistoryEntry> newSnapshotLog = ImmutableList.<HistoryEntry>builder()
      .addAll(snapshotLog)
      .add(new SnapshotLogEntry(nowMillis, snapshot.snapshotId()))
      .build();

  return new TableMetadata(null, formatVersion, uuid, location,
      lastSequenceNumber, nowMillis, lastColumnId, schema, defaultSpecId, specs, properties,
      snapshot.snapshotId(), snapshots, newSnapshotLog, addPreviousFile(file, lastUpdatedMillis));
}
 

public TableMetadata removeSnapshotLogEntries(Set<Long> snapshotIds) {
  List<HistoryEntry> newSnapshotLog = Lists.newArrayList();
  for (HistoryEntry logEntry : snapshotLog) {
    if (!snapshotIds.contains(logEntry.snapshotId())) {
      // copy the log entries that are still valid
      newSnapshotLog.add(logEntry);
    }
  }

  ValidationException.check(currentSnapshotId < 0 || // not set
          Iterables.getLast(newSnapshotLog).snapshotId() == currentSnapshotId,
      "Cannot set invalid snapshot log: latest entry is not the current snapshot");

  return new TableMetadata(null, formatVersion, uuid, location,
      lastSequenceNumber, System.currentTimeMillis(), lastColumnId, schema, defaultSpecId, specs, properties,
      currentSnapshotId, snapshots, newSnapshotLog, addPreviousFile(file, lastUpdatedMillis));
}
 

private Expression bindUnaryOperation(BoundTerm<T> boundTerm) {
  switch (op()) {
    case IS_NULL:
      if (boundTerm.ref().field().isRequired()) {
        return Expressions.alwaysFalse();
      }
      return new BoundUnaryPredicate<>(Operation.IS_NULL, boundTerm);
    case NOT_NULL:
      if (boundTerm.ref().field().isRequired()) {
        return Expressions.alwaysTrue();
      }
      return new BoundUnaryPredicate<>(Operation.NOT_NULL, boundTerm);
    default:
      throw new ValidationException("Operation must be IS_NULL or NOT_NULL");
  }
}
 

@Test
public void testInvalidUpdatePartitionSpecForV1Table() throws Exception {
  Schema schema = new Schema(
      Types.NestedField.required(1, "x", Types.LongType.get())
  );

  PartitionSpec spec = PartitionSpec.builderFor(schema).withSpecId(5)
      .add(1, 1005, "x_partition", "bucket[4]")
      .build();
  String location = "file://tmp/db/table";
  TableMetadata metadata = TableMetadata.newTableMetadata(schema, spec, location, ImmutableMap.of());

  AssertHelpers.assertThrows("Should fail to update an invalid partition spec",
      ValidationException.class, "Spec does not use sequential IDs that are required in v1",
      () -> metadata.updatePartitionSpec(spec));
}
 

private static void validateReplacedPartitions(TableMetadata meta, Long parentId,
                                               Set<StructLikeWrapper> replacedPartitions) {
  if (replacedPartitions != null) {
    ValidationException.check(parentId == null || isCurrentAncestor(meta, parentId),
        "Cannot cherry-pick overwrite, based on non-ancestor of the current state: %s", parentId);
    List<DataFile> newFiles = SnapshotUtil.newFiles(parentId, meta.currentSnapshot().snapshotId(), meta::snapshot);
    StructLikeWrapper partitionWrapper = StructLikeWrapper.wrap(null);
    for (DataFile newFile : newFiles) {
      ValidationException.check(!replacedPartitions.contains(partitionWrapper.set(newFile.partition())),
          "Cannot cherry-pick replace partitions with changed partition: %s",
          newFile.partition());
    }
  }
}
 

@Test
public void testCherryPickDynamicOverwriteConflict() {
  table.newAppend()
      .appendFile(FILE_A)
      .commit();

  // stage an overwrite that replaces FILE_A
  table.newReplacePartitions()
      .addFile(REPLACEMENT_FILE_A)
      .stageOnly()
      .commit();

  Snapshot staged = Iterables.getLast(table.snapshots());
  Assert.assertEquals("Should find the staged overwrite snapshot", DataOperations.OVERWRITE, staged.operation());

  // add another append so that the original commit can't be fast-forwarded
  table.newAppend()
      .appendFile(CONFLICT_FILE_A)
      .commit();
  long lastSnapshotId = table.currentSnapshot().snapshotId();

  // pick the snapshot into the current state
  AssertHelpers.assertThrows("Should reject partition replacement when a partition has been modified",
      ValidationException.class, "Cannot cherry-pick replace partitions with changed partition",
      () -> table.manageSnapshots()
      .cherrypick(staged.snapshotId())
      .commit());

  Assert.assertEquals("Failed cherry-pick should not change the table state",
      lastSnapshotId, table.currentSnapshot().snapshotId());
  validateTableFiles(table, FILE_A, CONFLICT_FILE_A);
}
 

@Test
public void testCherryPickDynamicOverwriteDeleteConflict() {
  table.newAppend()
      .appendFile(FILE_A)
      .commit();

  // stage an overwrite that replaces FILE_A
  table.newReplacePartitions()
      .addFile(REPLACEMENT_FILE_A)
      .stageOnly()
      .commit();

  Snapshot staged = Iterables.getLast(table.snapshots());
  Assert.assertEquals("Should find the staged overwrite snapshot", DataOperations.OVERWRITE, staged.operation());

  // add FILE_B s
  table.newAppend()
      .appendFile(FILE_B)
      .commit();

  // delete FILE_A so the replace operation is no longer valid
  table.newDelete()
      .deleteFile(FILE_A)
      .commit();
  long lastSnapshotId = table.currentSnapshot().snapshotId();

  // pick the snapshot into the current state
  AssertHelpers.assertThrows("Should reject partition replacement when a partition has been modified",
      ValidationException.class, "Missing required files to delete",
      () -> table.manageSnapshots()
          .cherrypick(staged.snapshotId())
          .commit());

  Assert.assertEquals("Failed cherry-pick should not change the table state",
      lastSnapshotId, table.currentSnapshot().snapshotId());
  validateTableFiles(table, FILE_B);
}
 

@Test
public void testCherryPickFromBranch() {
  table.newAppend()
      .appendFile(FILE_A)
      .commit();
  long branchSnapshotId = table.currentSnapshot().snapshotId();

  // add a second commit before replacing FILE_A
  table.newAppend()
      .appendFile(FILE_B)
      .commit();

  // replace FILE_A
  table.newReplacePartitions()
      .addFile(REPLACEMENT_FILE_A)
      .commit();
  long replaceSnapshotId = table.currentSnapshot().snapshotId();

  // rewrite history so the replacement is in a branch, not base directly on an ancestor of the current state
  table.manageSnapshots()
      .rollbackTo(branchSnapshotId)
      .commit();
  long lastSnapshotId = table.currentSnapshot().snapshotId();

  // pick the snapshot into the current state
  AssertHelpers.assertThrows("Should reject partition replacement when a partition has been modified",
      ValidationException.class, "Cannot cherry-pick overwrite not based on an ancestor of the current state",
      () -> table.manageSnapshots()
          .cherrypick(replaceSnapshotId)
          .commit());

  Assert.assertEquals("Failed cherry-pick should not change the table state",
      lastSnapshotId, table.currentSnapshot().snapshotId());
  validateTableFiles(table, FILE_A);
}
 

@Override
public List<ManifestFile> apply(TableMetadata base) {
  if (writeSpec().fields().size() <= 0) {
    // replace all data in an unpartitioned table
    deleteByRowFilter(Expressions.alwaysTrue());
  }

  try {
    return super.apply(base);
  } catch (ManifestFilterManager.DeleteException e) {
    throw new ValidationException(
        "Cannot commit file that conflicts with existing partition: %s", e.partition());
  }
}
 

private void validateFilesCounts() {
  Iterable<ManifestFile> createdManifests = Iterables.concat(newManifests, addedManifests, rewrittenAddedManifests);
  int createdManifestsFilesCount = activeFilesCount(createdManifests);

  Iterable<ManifestFile> replacedManifests = Iterables.concat(rewrittenManifests, deletedManifests);
  int replacedManifestsFilesCount = activeFilesCount(replacedManifests);

  if (createdManifestsFilesCount != replacedManifestsFilesCount) {
    throw new ValidationException(
        "Replaced and created manifests must have the same number of active files: %d (new), %d (old)",
        createdManifestsFilesCount, replacedManifestsFilesCount);
  }
}
 

private void validateDeletedManifests(Set<ManifestFile> currentManifests) {
  // directly deleted manifests must be still present in the current snapshot
  deletedManifests.stream()
      .filter(manifest -> !currentManifests.contains(manifest))
      .findAny()
      .ifPresent(manifest -> {
        throw new ValidationException("Manifest is missing: %s", manifest.path());
      });
}
 

@Test
public void testOverwriteIncompatibleBaseExpirationEmptyTableValidated() {
  Assert.assertNull("Should be empty table", table.currentSnapshot());

  OverwriteFiles overwrite = table.newOverwrite()
      .addFile(FILE_DAY_2_MODIFIED)
      .validateNoConflictingAppends(null, EXPRESSION_DAY_2);

  table.newAppend()
      .appendFile(FILE_DAY_2)
      .commit(); // id 1

  table.newDelete()
      .deleteFile(FILE_DAY_1)
      .commit(); // id 2

  table.expireSnapshots()
      .expireSnapshotId(1L)
      .commit();
  long committedSnapshotId = table.currentSnapshot().snapshotId();

  AssertHelpers.assertThrows("Should reject commit",
      ValidationException.class, "Cannot determine history",
      overwrite::commit);

  Assert.assertEquals("Should not create a new snapshot",
      committedSnapshotId, table.currentSnapshot().snapshotId());
}
 

private static void validateCurrentSnapshot(TableMetadata meta, Long requiredSnapshotId) {
  if (requiredSnapshotId != null) {
    ValidationException.check(meta.currentSnapshot().snapshotId() == requiredSnapshotId,
        "Cannot fast-forward to non-append snapshot; current has changed: current=%s != required=%s",
        meta.currentSnapshot().snapshotId(), requiredSnapshotId);
  }
}
 

@Override
public Snapshot apply() {
  TableMetadata base = refresh();

  if (targetSnapshotId == null) {
    // if no target snapshot was configured then NOOP by returning current state
    return base.currentSnapshot();
  }

  switch (managerOperation) {
    case CHERRYPICK:
      if (base.snapshot(targetSnapshotId).parentId() != null &&
          base.currentSnapshot().snapshotId() == base.snapshot(targetSnapshotId).parentId()) {
        // the snapshot to cherrypick is already based on the current state: fast-forward
        validate(base);
        return base.snapshot(targetSnapshotId);
      } else {
        // validate(TableMetadata) is called in apply(TableMetadata) after this apply refreshes the table state
        return super.apply();
      }

    case ROLLBACK:
      return base.snapshot(targetSnapshotId);

    default:
      throw new ValidationException("Invalid SnapshotManagerOperation: only cherrypick, rollback are supported");
  }
}
 

@Override
public ManageSnapshots rollbackTo(long snapshotId) {
  TableMetadata current = current();
  ValidationException.check(current.snapshot(snapshotId) != null,
      "Cannot roll back to unknown snapshot id: %s", snapshotId);
  ValidationException.check(
      isCurrentAncestor(current, snapshotId),
      "Cannot roll back to snapshot, not an ancestor of the current state: %s", snapshotId);
  return setCurrentSnapshot(snapshotId);
}
 

@Override
public ManageSnapshots setCurrentSnapshot(long snapshotId) {
  ValidationException.check(current().snapshot(snapshotId) != null,
      "Cannot roll back to unknown snapshot id: %s", snapshotId);

  this.managerOperation = SnapshotManagerOperation.ROLLBACK;
  this.targetSnapshotId = snapshotId;

  return this;
}