Java Code Examples for org.apache.iceberg.PartitionSpec#Builder

The following examples show how to use org.apache.iceberg.PartitionSpec#Builder . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: PartitionFields.java    From presto with Apache License 2.0 5 votes vote down vote up 
public static PartitionSpec parsePartitionFields(Schema schema, List<String> fields)
{
    PartitionSpec.Builder builder = PartitionSpec.builderFor(schema);
    for (String field : fields) {
        parsePartitionField(builder, field);
    }
    return builder.build();
}
Example 2
Source File: PartitionFields.java    From presto with Apache License 2.0 5 votes vote down vote up 
public static void parsePartitionField(PartitionSpec.Builder builder, String field)
{
    @SuppressWarnings("PointlessBooleanExpression")
    boolean matched = false ||
            tryMatch(field, IDENTITY_PATTERN, match -> builder.identity(match.group())) ||
            tryMatch(field, YEAR_PATTERN, match -> builder.year(match.group(1))) ||
            tryMatch(field, MONTH_PATTERN, match -> builder.month(match.group(1))) ||
            tryMatch(field, DAY_PATTERN, match -> builder.day(match.group(1))) ||
            tryMatch(field, HOUR_PATTERN, match -> builder.hour(match.group(1))) ||
            tryMatch(field, BUCKET_PATTERN, match -> builder.bucket(match.group(1), parseInt(match.group(2)))) ||
            tryMatch(field, TRUNCATE_PATTERN, match -> builder.truncate(match.group(1), parseInt(match.group(2))));
    if (!matched) {
        throw new IllegalArgumentException("Invalid partition field declaration: " + field);
    }
}
Example 3
Source File: TestPartitionFields.java    From presto with Apache License 2.0 5 votes vote down vote up 
private static PartitionSpec partitionSpec(Consumer<PartitionSpec.Builder> consumer)
{
    Schema schema = new Schema(
            NestedField.required(1, "order_key", LongType.get()),
            NestedField.required(2, "ts", TimestampType.withoutZone()),
            NestedField.required(3, "price", DoubleType.get()),
            NestedField.optional(4, "comment", StringType.get()),
            NestedField.optional(5, "notes", ListType.ofRequired(6, StringType.get())));

    PartitionSpec.Builder builder = PartitionSpec.builderFor(schema);
    consumer.accept(builder);
    return builder.build();
}
Example 4
Source File: SparkSchemaUtil.java    From iceberg with Apache License 2.0 5 votes vote down vote up 
private static PartitionSpec identitySpec(Schema schema, List<String> partitionNames) {
  if (partitionNames == null || partitionNames.isEmpty()) {
    return null;
  }

  PartitionSpec.Builder builder = PartitionSpec.builderFor(schema);
  for (String partitionName : partitionNames) {
    builder.identity(partitionName);
  }

  return builder.build();
}
Example 5
Source File: IcebergCatalog.java    From dremio-oss with Apache License 2.0 5 votes vote down vote up 
public static PartitionSpec getIcebergPartitionSpec(BatchSchema batchSchema,
                                                    List<String> partitionColumns) {
  SchemaConverter schemaConverter = new SchemaConverter();
  Schema schema;

  // match partition column name with name in schema
  List<String> partitionColumnsInSchemaCase = new ArrayList<>();
  if (partitionColumns != null) {
    List<String> invalidPartitionColumns = new ArrayList<>();
    for (String partitionColumn : partitionColumns) {
      Optional<Field> fieldFromSchema = batchSchema.findFieldIgnoreCase(partitionColumn);
      if (fieldFromSchema.isPresent()) {
        if (fieldFromSchema.get().getType().getTypeID() == ArrowType.ArrowTypeID.Time) {
          throw UserException.validationError().message("Partition column %s of type time is not supported", fieldFromSchema.get().getName()).buildSilently();
        }
        partitionColumnsInSchemaCase.add(fieldFromSchema.get().getName());
      } else {
        invalidPartitionColumns.add(partitionColumn);
      }
    }
    if (!invalidPartitionColumns.isEmpty()) {
      throw UserException.validationError().message("Partition column(s) %s are not found in table.", invalidPartitionColumns).buildSilently();
    }
  }

  try {
    schema = schemaConverter.toIceberg(batchSchema);
    PartitionSpec.Builder partitionSpecBuilder = PartitionSpec.builderFor(schema);
      for (String column : partitionColumnsInSchemaCase) {
        partitionSpecBuilder.identity(column);
    }
    return partitionSpecBuilder.build();
  } catch (Exception ex) {
    throw UserException.validationError(ex).buildSilently();
  }
}
Example 6
Source File: Spark3Util.java    From iceberg with Apache License 2.0 4 votes vote down vote up 
/**
 * Converts Spark transforms into a {@link PartitionSpec}.
 *
 * @param schema the table schema
 * @param partitioning Spark Transforms
 * @return a PartitionSpec
 */
public static PartitionSpec toPartitionSpec(Schema schema, Transform[] partitioning) {
  if (partitioning == null || partitioning.length == 0) {
    return PartitionSpec.unpartitioned();
  }

  PartitionSpec.Builder builder = PartitionSpec.builderFor(schema);
  for (Transform transform : partitioning) {
    Preconditions.checkArgument(transform.references().length == 1,
        "Cannot convert transform with more than one column reference: %s", transform);
    String colName = DOT.join(transform.references()[0].fieldNames());
    switch (transform.name()) {
      case "identity":
        builder.identity(colName);
        break;
      case "bucket":
        builder.bucket(colName, findWidth(transform));
        break;
      case "years":
        builder.year(colName);
        break;
      case "months":
        builder.month(colName);
        break;
      case "date":
      case "days":
        builder.day(colName);
        break;
      case "date_hour":
      case "hours":
        builder.hour(colName);
        break;
      case "truncate":
        builder.truncate(colName, findWidth(transform));
        break;
      default:
        throw new UnsupportedOperationException("Transform is not supported: " + transform);
    }
  }

  return builder.build();
}