Java Code Examples for org.apache.iceberg.exceptions.ValidationException#check()

@SuppressWarnings("unchecked")
@Override
public BoundTransform<S, T> bind(Types.StructType struct, boolean caseSensitive) {
  BoundReference<S> boundRef = ref.bind(struct, caseSensitive);

  Transform<S, T> typeTransform;
  try {
    // TODO: Avoid using toString/fromString
    typeTransform = (Transform<S, T>) Transforms.fromString(boundRef.type(), transform.toString());
    ValidationException.check(typeTransform.canTransform(boundRef.type()),
        "Cannot bind: %s cannot transform %s values from '%s'", transform, boundRef.type(), ref.name());
  } catch (IllegalArgumentException e) {
    throw new ValidationException(
        "Cannot bind: %s cannot transform %s values from '%s'", transform, boundRef.type(), ref.name());
  }

  return new BoundTransform<>(boundRef, typeTransform);
}
 

private boolean manifestHasDeletedFiles(
    StrictMetricsEvaluator metricsEvaluator, ManifestReader<F> reader, StructLikeWrapper partitionWrapper) {
  boolean isDelete = reader.isDeleteManifestReader();
  Evaluator inclusive = inclusiveDeleteEvaluator(reader.spec());
  Evaluator strict = strictDeleteEvaluator(reader.spec());
  boolean hasDeletedFiles = false;
  for (ManifestEntry<F> entry : reader.entries()) {
    F file = entry.file();
    boolean fileDelete = deletePaths.contains(file.path()) ||
        dropPartitions.contains(partitionWrapper.set(file.partition())) ||
        (isDelete && entry.sequenceNumber() > 0 && entry.sequenceNumber() < minSequenceNumber);
    if (fileDelete || inclusive.eval(file.partition())) {
      ValidationException.check(
          fileDelete || strict.eval(file.partition()) || metricsEvaluator.eval(file),
          "Cannot delete file where some, but not all, rows match filter %s: %s",
          this.deleteExpression, file.path());

      hasDeletedFiles = true;
      if (failAnyDelete) {
        throw new DeleteException(reader.spec().partitionToPath(file.partition()));
      }
      break; // as soon as a deleted file is detected, stop scanning
    }
  }
  return hasDeletedFiles;
}
 

public TableMetadata removeSnapshotLogEntries(Set<Long> snapshotIds) {
  List<HistoryEntry> newSnapshotLog = Lists.newArrayList();
  for (HistoryEntry logEntry : snapshotLog) {
    if (!snapshotIds.contains(logEntry.snapshotId())) {
      // copy the log entries that are still valid
      newSnapshotLog.add(logEntry);
    }
  }

  ValidationException.check(currentSnapshotId < 0 || // not set
          Iterables.getLast(newSnapshotLog).snapshotId() == currentSnapshotId,
      "Cannot set invalid snapshot log: latest entry is not the current snapshot");

  return new TableMetadata(null, formatVersion, uuid, location,
      lastSequenceNumber, System.currentTimeMillis(), lastColumnId, schema, defaultSpecId, specs, properties,
      currentSnapshotId, snapshots, newSnapshotLog, addPreviousFile(file, lastUpdatedMillis));
}
 

public TableMetadata replaceCurrentSnapshot(Snapshot snapshot) {
  // there can be operations (viz. rollback, cherrypick) where an existing snapshot could be replacing current
  if (snapshotsById.containsKey(snapshot.snapshotId())) {
    return setCurrentSnapshotTo(snapshot);
  }

  ValidationException.check(formatVersion == 1 || snapshot.sequenceNumber() > lastSequenceNumber,
      "Cannot add snapshot with sequence number %s older than last sequence number %s",
      snapshot.sequenceNumber(), lastSequenceNumber);

  List<Snapshot> newSnapshots = ImmutableList.<Snapshot>builder()
      .addAll(snapshots)
      .add(snapshot)
      .build();
  List<HistoryEntry> newSnapshotLog = ImmutableList.<HistoryEntry>builder()
      .addAll(snapshotLog)
      .add(new SnapshotLogEntry(snapshot.timestampMillis(), snapshot.snapshotId()))
      .build();

  return new TableMetadata(null, formatVersion, uuid, location,
      snapshot.sequenceNumber(), snapshot.timestampMillis(), lastColumnId, schema, defaultSpecId, specs, properties,
      snapshot.snapshotId(), newSnapshots, newSnapshotLog, addPreviousFile(file, lastUpdatedMillis));
}
 

private TableMetadata setCurrentSnapshotTo(Snapshot snapshot) {
  ValidationException.check(snapshotsById.containsKey(snapshot.snapshotId()),
      "Cannot set current snapshot to unknown: %s", snapshot.snapshotId());
  ValidationException.check(formatVersion == 1 || snapshot.sequenceNumber() <= lastSequenceNumber,
      "Last sequence number %s is less than existing snapshot sequence number %s",
      lastSequenceNumber, snapshot.sequenceNumber());

  if (currentSnapshotId == snapshot.snapshotId()) {
    // change is a noop
    return this;
  }

  long nowMillis = System.currentTimeMillis();
  List<HistoryEntry> newSnapshotLog = ImmutableList.<HistoryEntry>builder()
      .addAll(snapshotLog)
      .add(new SnapshotLogEntry(nowMillis, snapshot.snapshotId()))
      .build();

  return new TableMetadata(null, formatVersion, uuid, location,
      lastSequenceNumber, nowMillis, lastColumnId, schema, defaultSpecId, specs, properties,
      snapshot.snapshotId(), snapshots, newSnapshotLog, addPreviousFile(file, lastUpdatedMillis));
}
 

private static void validateReplacedPartitions(TableMetadata meta, Long parentId,
                                               Set<StructLikeWrapper> replacedPartitions) {
  if (replacedPartitions != null) {
    ValidationException.check(parentId == null || isCurrentAncestor(meta, parentId),
        "Cannot cherry-pick overwrite, based on non-ancestor of the current state: %s", parentId);
    List<DataFile> newFiles = SnapshotUtil.newFiles(parentId, meta.currentSnapshot().snapshotId(), meta::snapshot);
    StructLikeWrapper partitionWrapper = StructLikeWrapper.wrap(null);
    for (DataFile newFile : newFiles) {
      ValidationException.check(!replacedPartitions.contains(partitionWrapper.set(newFile.partition())),
          "Cannot cherry-pick replace partitions with changed partition: %s",
          newFile.partition());
    }
  }
}
 

private static void validateCurrentSnapshot(TableMetadata meta, Long requiredSnapshotId) {
  if (requiredSnapshotId != null) {
    ValidationException.check(meta.currentSnapshot().snapshotId() == requiredSnapshotId,
        "Cannot fast-forward to non-append snapshot; current has changed: current=%s != required=%s",
        meta.currentSnapshot().snapshotId(), requiredSnapshotId);
  }
}
 

@Override
public ManageSnapshots rollbackTo(long snapshotId) {
  TableMetadata current = current();
  ValidationException.check(current.snapshot(snapshotId) != null,
      "Cannot roll back to unknown snapshot id: %s", snapshotId);
  ValidationException.check(
      isCurrentAncestor(current, snapshotId),
      "Cannot roll back to snapshot, not an ancestor of the current state: %s", snapshotId);
  return setCurrentSnapshot(snapshotId);
}
 

@Override
public ManageSnapshots setCurrentSnapshot(long snapshotId) {
  ValidationException.check(current().snapshot(snapshotId) != null,
      "Cannot roll back to unknown snapshot id: %s", snapshotId);

  this.managerOperation = SnapshotManagerOperation.ROLLBACK;
  this.targetSnapshotId = snapshotId;

  return this;
}
 

private static Map<Long, Snapshot> indexAndValidateSnapshots(List<Snapshot> snapshots, long lastSequenceNumber) {
  ImmutableMap.Builder<Long, Snapshot> builder = ImmutableMap.builder();
  for (Snapshot snap : snapshots) {
    ValidationException.check(snap.sequenceNumber() <= lastSequenceNumber,
        "Invalid snapshot with sequence number %s greater than last sequence number %s",
        snap.sequenceNumber(), lastSequenceNumber);
    builder.put(snap.snapshotId(), snap);
  }
  return builder.build();
}
 

public TableMetadata replaceProperties(Map<String, String> newProperties) {
  ValidationException.check(newProperties != null, "Cannot set properties to null");
  return new TableMetadata(null, formatVersion, uuid, location,
      lastSequenceNumber, System.currentTimeMillis(), lastColumnId, schema, defaultSpecId, specs, newProperties,
      currentSnapshotId, snapshots, snapshotLog, addPreviousFile(file, lastUpdatedMillis, newProperties));
}
 

public TableMetadata addStagedSnapshot(Snapshot snapshot) {
  ValidationException.check(formatVersion == 1 || snapshot.sequenceNumber() > lastSequenceNumber,
      "Cannot add snapshot with sequence number %s older than last sequence number %s",
      snapshot.sequenceNumber(), lastSequenceNumber);

  List<Snapshot> newSnapshots = ImmutableList.<Snapshot>builder()
      .addAll(snapshots)
      .add(snapshot)
      .build();

  return new TableMetadata(null, formatVersion, uuid, location,
      snapshot.sequenceNumber(), snapshot.timestampMillis(), lastColumnId, schema, defaultSpecId, specs, properties,
      currentSnapshotId, newSnapshots, snapshotLog, addPreviousFile(file, lastUpdatedMillis));
}
 

public TableMetadata updatePartitionSpec(PartitionSpec newPartitionSpec) {
  PartitionSpec.checkCompatibility(newPartitionSpec, schema);
  ValidationException.check(formatVersion > 1 || PartitionSpec.hasSequentialIds(newPartitionSpec),
      "Spec does not use sequential IDs that are required in v1: %s", newPartitionSpec);

  // if the spec already exists, use the same ID. otherwise, use 1 more than the highest ID.
  int newDefaultSpecId = INITIAL_SPEC_ID;
  for (PartitionSpec spec : specs) {
    if (newPartitionSpec.compatibleWith(spec)) {
      newDefaultSpecId = spec.specId();
      break;
    } else if (newDefaultSpecId <= spec.specId()) {
      newDefaultSpecId = spec.specId() + 1;
    }
  }

  Preconditions.checkArgument(defaultSpecId != newDefaultSpecId,
      "Cannot set default partition spec to the current default");

  ImmutableList.Builder<PartitionSpec> builder = ImmutableList.<PartitionSpec>builder()
      .addAll(specs);
  if (!specsById.containsKey(newDefaultSpecId)) {
    // get a fresh spec to ensure the spec ID is set to the new default
    builder.add(freshSpec(newDefaultSpecId, schema, newPartitionSpec));
  }

  return new TableMetadata(null, formatVersion, uuid, location,
      lastSequenceNumber, System.currentTimeMillis(), lastColumnId, schema, newDefaultSpecId,
      builder.build(), properties,
      currentSnapshotId, snapshots, snapshotLog, addPreviousFile(file, lastUpdatedMillis));
}
 

/**
 * Throws a {@link ValidationException} if any deleted file was not present in a filtered manifest.
 *
 * @param manifests a set of filtered manifests
 */
private void validateRequiredDeletes(ManifestFile... manifests) {
  if (failMissingDeletePaths) {
    Set<CharSequence> deletedFiles = deletedFiles(manifests);
    ValidationException.check(deletedFiles.containsAll(deletePaths),
        "Missing required files to delete: %s",
        COMMA.join(Iterables.filter(deletePaths, path -> !deletedFiles.contains(path))));
  }
}
 

@Override
public RewriteManifestsActionResult execute() {
  List<ManifestFile> matchingManifests = findMatchingManifests();
  if (matchingManifests.isEmpty()) {
    return RewriteManifestsActionResult.empty();
  }

  long totalSizeBytes = 0L;
  int numEntries = 0;

  for (ManifestFile manifest : matchingManifests) {
    ValidationException.check(hasFileCounts(manifest), "No file counts in manifest: " + manifest.path());

    totalSizeBytes += manifest.length();
    numEntries += manifest.addedFilesCount() + manifest.existingFilesCount() + manifest.deletedFilesCount();
  }

  int targetNumManifests = targetNumManifests(totalSizeBytes);
  int targetNumManifestEntries = targetNumManifestEntries(numEntries, targetNumManifests);

  Dataset<Row> manifestEntryDF = buildManifestEntryDF(matchingManifests);

  List<ManifestFile> newManifests;
  if (spec.fields().size() < 1) {
    newManifests = writeManifestsForUnpartitionedTable(manifestEntryDF, targetNumManifests);
  } else {
    newManifests = writeManifestsForPartitionedTable(manifestEntryDF, targetNumManifests, targetNumManifestEntries);
  }

  replaceManifests(matchingManifests, newManifests);

  return new RewriteManifestsActionResult(matchingManifests, newManifests);
}
 

static void checkCompatibility(PartitionSpec spec, Schema schema) {
  for (PartitionField field : spec.fields) {
    Type sourceType = schema.findType(field.sourceId());
    ValidationException.check(sourceType != null,
        "Cannot find source column for partition field: %s", field);
    ValidationException.check(sourceType.isPrimitiveType(),
        "Cannot partition by non-primitive source field: %s", sourceType);
    ValidationException.check(
        field.transform().canTransform(sourceType),
        "Invalid source type %s for transform: %s",
        sourceType, field.transform());
  }
}
 

@Override
public BoundReference<T> bind(Types.StructType struct, boolean caseSensitive) {
  Schema schema = new Schema(struct.fields());
  Types.NestedField field = caseSensitive ?
      schema.findField(name) :
      schema.caseInsensitiveFindField(name);

  ValidationException.check(field != null,
      "Cannot find field '%s' in struct: %s", name, schema.asStruct());

  return new BoundReference<>(field, schema.accessorForField(field.fieldId()));
}
 

@Override
public <T> R predicate(BoundPredicate<T> pred) {
  ValidationException.check(pred.term() instanceof BoundReference,
      "Visitor %s does not support expression: %s", this, pred.term());

  if (pred.isLiteralPredicate()) {
    BoundLiteralPredicate<T> literalPred = pred.asLiteralPredicate();
    switch (pred.op()) {
      case LT:
        return lt((BoundReference<T>) pred.term(), literalPred.literal());
      case LT_EQ:
        return ltEq((BoundReference<T>) pred.term(), literalPred.literal());
      case GT:
        return gt((BoundReference<T>) pred.term(), literalPred.literal());
      case GT_EQ:
        return gtEq((BoundReference<T>) pred.term(), literalPred.literal());
      case EQ:
        return eq((BoundReference<T>) pred.term(), literalPred.literal());
      case NOT_EQ:
        return notEq((BoundReference<T>) pred.term(), literalPred.literal());
      case STARTS_WITH:
        return startsWith((BoundReference<T>) pred.term(),  literalPred.literal());
      default:
        throw new IllegalStateException("Invalid operation for BoundLiteralPredicate: " + pred.op());
    }

  } else if (pred.isUnaryPredicate()) {
    switch (pred.op()) {
      case IS_NULL:
        return isNull((BoundReference<T>) pred.term());
      case NOT_NULL:
        return notNull((BoundReference<T>) pred.term());
      default:
        throw new IllegalStateException("Invalid operation for BoundUnaryPredicate: " + pred.op());
    }

  } else if (pred.isSetPredicate()) {
    switch (pred.op()) {
      case IN:
        return in((BoundReference<T>) pred.term(), pred.asSetPredicate().literalSet());
      case NOT_IN:
        return notIn((BoundReference<T>) pred.term(), pred.asSetPredicate().literalSet());
      default:
        throw new IllegalStateException("Invalid operation for BoundSetPredicate: " + pred.op());
    }
  }

  throw new IllegalStateException("Unsupported bound predicate: " + pred.getClass().getName());
}