Java Code Examples for org.apache.iceberg.types.Types.NestedField#optional()

private static Schema toIcebergSchema(List<ColumnMetadata> columns)
{
    List<NestedField> icebergColumns = new ArrayList<>();
    for (ColumnMetadata column : columns) {
        if (!column.isHidden()) {
            int index = icebergColumns.size();
            Type type = toIcebergType(column.getType());
            NestedField field = column.isNullable()
                    ? NestedField.optional(index, column.getName(), type, column.getComment())
                    : NestedField.required(index, column.getName(), type, column.getComment());
            icebergColumns.add(field);
        }
    }
    Schema schema = new Schema(icebergColumns);
    AtomicInteger nextFieldId = new AtomicInteger(1);
    return TypeUtil.assignFreshIds(schema, nextFieldId::getAndIncrement);
}
 

@Test
public void map() {
  org.apache.iceberg.Schema icebergSchema = new org.apache.iceberg.Schema(
    NestedField.optional(1, "map",
      Types.MapType.ofOptional(2, 3, Types.IntegerType.get(), Types.FloatType.get()))
  );

  List<Field> children = Arrays.asList(
    CompleteType.INT.toField("key"),
    CompleteType.FLOAT.toField("value")
  );
  BatchSchema schema = BatchSchema.newBuilder()
    .addField(new CompleteType(new ArrowType.Map(false), children).toField("map"))
    .build();

  BatchSchema result = schemaConverter.fromIceberg(icebergSchema);
  // dremio silently drops map type columns
  assertEquals(result.getFieldCount(), 0);

  org.apache.iceberg.Schema icebergResult = schemaConverter.toIceberg(schema);
  assertEquals(icebergSchema.toString(), icebergResult.toString());
}
 

private static PartitionSpec partitionSpec(Consumer<PartitionSpec.Builder> consumer)
{
    Schema schema = new Schema(
            NestedField.required(1, "order_key", LongType.get()),
            NestedField.required(2, "ts", TimestampType.withoutZone()),
            NestedField.required(3, "price", DoubleType.get()),
            NestedField.optional(4, "comment", StringType.get()),
            NestedField.optional(5, "notes", ListType.ofRequired(6, StringType.get())));

    PartitionSpec.Builder builder = PartitionSpec.builderFor(schema);
    consumer.accept(builder);
    return builder.build();
}
 

public static NestedField toIcebergColumn(Field field) {
  try {
    return NestedField.optional(0, field.getName(), toIcebergType(CompleteType.fromField(field)));
  } catch (Exception e) {
    throw UserException.unsupportedError(e)
      .message("conversion from arrow type to iceberg type failed for field " + field.getName())
      .buildSilently();
  }
}
 

@Before
public void setUp() {
  schema = new Schema(
    NestedField.optional(1, "n_nationkey", Types.IntegerType.get()),
    NestedField.optional(2, "n_name", Types.StringType.get()),
    NestedField.optional(3, "n_regionkey", Types.IntegerType.get()),
    NestedField.optional(4, "n_comment", Types.StringType.get())
  );
}
 

@Test
public void primitiveBasic() {
  org.apache.iceberg.Schema icebergSchema = new org.apache.iceberg.Schema(
    NestedField.optional(1, "boolean", Types.BooleanType.get()),
    NestedField.optional(2, "int", Types.IntegerType.get()),
    NestedField.optional(3, "long", Types.LongType.get()),
    NestedField.optional(4, "float", Types.FloatType.get()),
    NestedField.optional(5, "double", Types.DoubleType.get()),
    NestedField.optional(6, "decimal_38_16", Types.DecimalType.of(38, 16)),
    NestedField.optional(7, "string", Types.StringType.get()),
    NestedField.optional(8, "binary", Types.BinaryType.get()),
    NestedField.optional(9, "date", Types.DateType.get()),
    NestedField.optional(10, "time", Types.TimeType.get()),
    NestedField.optional(11, "fixed_32", Types.FixedType.ofLength(32)),
    NestedField.optional(12, "timestamp", Types.TimestampType.withZone())
  );

  BatchSchema schema = BatchSchema.newBuilder()
    .addField(CompleteType.BIT.toField("boolean"))
    .addField(CompleteType.INT.toField("int"))
    .addField(CompleteType.BIGINT.toField("long"))
    .addField(CompleteType.FLOAT.toField("float"))
    .addField(CompleteType.DOUBLE.toField("double"))
    .addField(new CompleteType(new Decimal(38, 16)).toField("decimal_38_16"))
    .addField(CompleteType.VARCHAR.toField("string"))
    .addField(CompleteType.VARBINARY.toField("binary"))
    .addField(CompleteType.DATE.toField("date"))
    .addField(CompleteType.TIME.toField("time"))
    .addField(new CompleteType(new ArrowType.FixedSizeBinary(32)).toField("fixed_32"))
    .addField(CompleteType.TIMESTAMP.toField("timestamp"))
    .build();

  assertEquals(schema, schemaConverter.fromIceberg(icebergSchema));
  assertEquals(icebergSchema.toString(), schemaConverter.toIceberg(schema).toString());
}
 

@Test
public void missingArrowTypes() {
  org.apache.iceberg.Schema icebergSchema = new org.apache.iceberg.Schema(
    NestedField.optional(1, "uuid", Types.UUIDType.get())
  );

  BatchSchema schema = BatchSchema.newBuilder()
    .addField(new CompleteType(new FixedSizeBinary(16)).toField("uuid"))
    .build();

  BatchSchema result = schemaConverter.fromIceberg(icebergSchema);
  assertEquals(result, schema);
}
 

@Test
public void testPartitionComparatorField() {
  BatchSchema inputschema = BatchSchema.newBuilder()
    .addField(CompleteType.BIT.toField("boolean"))
    .addField(CompleteType.INT.toField("int"))
    .addField(CompleteType.BIT.toField(WriterPrel.PARTITION_COMPARATOR_FIELD))
    .build();

  org.apache.iceberg.Schema expectedSchema = new org.apache.iceberg.Schema(
    NestedField.optional(1, "boolean", Types.BooleanType.get()),
    NestedField.optional(2, "int", Types.IntegerType.get()));

  SchemaConverter convert = new SchemaConverter();
  assertEquals(convert.toIceberg(inputschema).toString(), expectedSchema.toString());
}
 

@Test
public void unsupportedIcebergTypes() {
  org.apache.iceberg.Schema schema = new org.apache.iceberg.Schema(
    NestedField.optional(1, "timestamp_nozone_field", Types.TimestampType.withoutZone())
  );

  expectedEx.expect(UserException.class);
  expectedEx.expectMessage("conversion from iceberg type to arrow type failed for field timestamp_nozone_field");
  SchemaConverter convert = new SchemaConverter();
  convert.fromIceberg(schema);
}
 

@Before
public void setUp() {
  schema = new Schema(
    NestedField.optional(1, "n_nationkey", Types.IntegerType.get()),
    NestedField.optional(2, "n_name", Types.StringType.get()),
    NestedField.optional(3, "n_regionkey", Types.IntegerType.get()),
    NestedField.optional(4, "n_comment", Types.StringType.get())
  );
}
 

@Before
public void setUp() {
  schema = new Schema(
    NestedField.optional(1, ID, Types.IntegerType.get()),
    NestedField.optional(2, NAME, Types.StringType.get())
  );

  spec = PartitionSpec
    .builderFor(schema)
    .identity(ID)
    .identity(NAME)
    .build();
}
 

@Test
public void mixed() throws Exception {
  BatchSchema schema = BatchSchema.newBuilder()
    .addField(CompleteType.INT.toField("rownum"))
    .addField(CompleteType.VARCHAR.toField("name"))
    .addField(CompleteType.INT.toField("age"))
    .addField(CompleteType.FLOAT.toField("gpa"))
    .addField(CompleteType.BIGINT.toField("studentnum"))
    .addField(CompleteType.TIMESTAMP.toField("create_time"))
    .addField(CompleteType.VARCHAR.asList().toField("interests"))
    .addField(CompleteType.struct(
      CompleteType.VARCHAR.toField("color"),
      CompleteType.VARCHAR.toField("sport"),
      CompleteType.VARCHAR.toField("food")
    ).toField("favorites"))
    .build();

  org.apache.iceberg.Schema expectedSchema = new org.apache.iceberg.Schema(
    NestedField.optional(1, "rownum", Types.IntegerType.get()),
    NestedField.optional(2, "name", Types.StringType.get()),
    NestedField.optional(3, "age", Types.IntegerType.get()),
    NestedField.optional(4, "gpa", Types.FloatType.get()),
    NestedField.optional(5, "studentnum", Types.LongType.get()),
    NestedField.optional(6, "create_time", Types.TimestampType.withZone()),
    NestedField.optional(7, "interests",
      Types.ListType.ofOptional(9, Types.StringType.get())),
    NestedField.optional(8, "favorites",
      Types.StructType.of(
        NestedField.optional(10, "color", Types.StringType.get()),
        NestedField.optional(11, "sport", Types.StringType.get()),
        NestedField.optional(12, "food", Types.StringType.get())
      ))
  );

  org.apache.iceberg.Schema icebergResult = schemaConverter.toIceberg(schema);
  assertEquals(expectedSchema.toString(), icebergResult.toString());

  TemporaryFolder folder = new TemporaryFolder();
  folder.create();

  String rootPath = folder.getRoot().toString();
  Configuration conf = new Configuration();
  IcebergCatalog catalog = new IcebergCatalog(rootPath, conf);
  catalog.beginCreateTable(schema, Collections.emptyList());
  catalog.endCreateTable();

  Table table = new HadoopTables(conf).load(rootPath);
  assertEquals(expectedSchema.toString(), table.schema().toString());
}