org.apache.iceberg.types.Types.DoubleType Java Examples

@Test
public void convertPrimitive() {
  Schema iceberg = new Schema(
      Types.NestedField.optional(0, INTEGER_FIELD, IntegerType.get()),
      Types.NestedField.optional(1, BOOLEAN_FIELD, BooleanType.get()),
      Types.NestedField.required(2, DOUBLE_FIELD, DoubleType.get()),
      Types.NestedField.required(3, STRING_FIELD, StringType.get()),
      Types.NestedField.optional(4, DATE_FIELD, DateType.get()),
      Types.NestedField.optional(5, TIMESTAMP_FIELD, TimestampType.withZone()),
      Types.NestedField.optional(6, LONG_FIELD, LongType.get()),
      Types.NestedField.optional(7, FLOAT_FIELD, FloatType.get()),
      Types.NestedField.optional(8, TIME_FIELD, TimeType.get()),
      Types.NestedField.optional(9, BINARY_FIELD, Types.BinaryType.get()),
      Types.NestedField.optional(10, DECIMAL_FIELD, Types.DecimalType.of(1, 1)),
      Types.NestedField.optional(12, LIST_FIELD, Types.ListType.ofOptional(13, Types.IntegerType.get())),
      Types.NestedField.required(14, MAP_FIELD, Types.MapType.ofOptional(15, 16,
          StringType.get(), IntegerType.get())),
      Types.NestedField.optional(17, FIXED_WIDTH_BINARY_FIELD, Types.FixedType.ofLength(10)));

  org.apache.arrow.vector.types.pojo.Schema arrow = ArrowSchemaUtil.convert(iceberg);

  validate(iceberg, arrow);
}
 

@Test
public void testPrimitive() throws IOException {
  Schema icebergSchema = new Schema(
      optional(1, "b", BooleanType.get()),
      optional(2, "i", IntegerType.get()),
      optional(3, "l", LongType.get()),
      optional(4, "f", FloatType.get()),
      optional(5, "d", DoubleType.get()),
      optional(6, "dec", DecimalType.of(0, 2)),
      optional(7, "s", StringType.get()),
      optional(8, "bi", BinaryType.get())
  );

  ResourceSchema pigSchema = SchemaUtil.convert(icebergSchema);
  assertEquals(
      "b:boolean,i:int,l:long,f:float,d:double,dec:bigdecimal,s:chararray,bi:bytearray", pigSchema.toString());
}
 

private static PartitionSpec partitionSpec(Consumer<PartitionSpec.Builder> consumer)
{
    Schema schema = new Schema(
            NestedField.required(1, "order_key", LongType.get()),
            NestedField.required(2, "ts", TimestampType.withoutZone()),
            NestedField.required(3, "price", DoubleType.get()),
            NestedField.optional(4, "comment", StringType.get()),
            NestedField.optional(5, "notes", ListType.ofRequired(6, StringType.get())));

    PartitionSpec.Builder builder = PartitionSpec.builderFor(schema);
    consumer.accept(builder);
    return builder.build();
}
 

@Test
public void testComplex() throws IOException {
  convertToPigSchema(
      new Schema(
          optional(1, "bag", ListType.ofOptional(2, BooleanType.get())),
          optional(3, "map", MapType.ofOptional(4, 5, StringType.get(), DoubleType.get())),
          optional(6, "tuple",
              StructType.of(optional(7, "i", IntegerType.get()), optional(8, "f", FloatType.get())))
      ), "bag:{(boolean)},map:[double],tuple:(i:int,f:float)", null
  );
}
 

@Test
public void nestedBags() throws IOException {
  convertToPigSchema(new Schema(
      optional(1, "nested",
          ListType.ofOptional(2,
              ListType.ofOptional(3,
                  ListType.ofOptional(4, DoubleType.get()))))
  ), "nested:{({({(double)})})}", "");
}
 

@Test
public void mapConversions() throws IOException {
  // consistent behavior for maps conversions. The below test case, correctly does not specify map key types
  convertToPigSchema(
      new Schema(
          required(
              1, "a",
              MapType.ofRequired(
                  2, 3,
                  StringType.get(),
                  ListType.ofRequired(
                      4, StructType.of(
                          required(5, "b", LongType.get()),
                          required(6, "c", StringType.get())))))),
      "a:[{(b:long,c:chararray)}]",
      "We do not specify the map key type here");
  // struct<a:map<string,map<string,double>>> -> (a:[[double]])
  // As per https://pig.apache.org/docs/latest/basic.html#map-schema. It seems that
  // we  only need to specify value type as keys are always of type chararray
  convertToPigSchema(
      new Schema(
          StructType.of(
              required(1, "a", MapType.ofRequired(
                  2, 3,
                  StringType.get(),
                  MapType.ofRequired(4, 5, StringType.get(), DoubleType.get())))
          ).fields()),
      "a:[[double]]",
      "A map key type does not need to be specified");
}
 

@Test(expected = FrontendException.class)
public void invalidMap() throws IOException {
  convertToPigSchema(new Schema(
      optional(1, "invalid", MapType.ofOptional(2, 3, IntegerType.get(), DoubleType.get()))
  ), "", "");
}
 

@Test
public void testMetricsForTopLevelWithMultipleRowGroup() throws Exception {
  Assume.assumeTrue("Skip test for formats that do not support small row groups", supportsSmallRowGroups());

  int recordCount = 201;
  List<Record> records = new ArrayList<>(recordCount);

  for (int i = 0; i < recordCount; i++) {
    Record newRecord = GenericRecord.create(SIMPLE_SCHEMA);
    newRecord.setField("booleanCol", i == 0 ? false : true);
    newRecord.setField("intCol", i + 1);
    newRecord.setField("longCol", i == 0 ? null : i + 1L);
    newRecord.setField("floatCol", i + 1.0F);
    newRecord.setField("doubleCol", i == 0 ? null : i + 1.0D);
    newRecord.setField("decimalCol", i == 0 ? null : new BigDecimal(i + "").add(new BigDecimal("1.00")));
    newRecord.setField("stringCol", "AAA");
    newRecord.setField("dateCol", DateTimeUtil.dateFromDays(i + 1));
    newRecord.setField("timeCol", DateTimeUtil.timeFromMicros(i + 1L));
    newRecord.setField("timestampColAboveEpoch", DateTimeUtil.timestampFromMicros(i + 1L));
    newRecord.setField("fixedCol", fixed);
    newRecord.setField("binaryCol", ByteBuffer.wrap("S".getBytes()));
    newRecord.setField("timestampColBelowEpoch", DateTimeUtil.timestampFromMicros((i + 1L) * -1L));
    records.add(newRecord);
  }

  // create file with multiple row groups. by using smaller number of bytes
  InputFile recordsFile = writeRecordsWithSmallRowGroups(SIMPLE_SCHEMA, records.toArray(new Record[0]));

  Assert.assertNotNull(recordsFile);
  // rowgroup size should be > 1
  Assert.assertEquals(3, splitCount(recordsFile));

  Metrics metrics = getMetrics(recordsFile);
  Assert.assertEquals(201L, (long) metrics.recordCount());
  assertCounts(1, 201L, 0L, metrics);
  assertBounds(1, Types.BooleanType.get(), false, true, metrics);
  assertBounds(2, Types.IntegerType.get(), 1, 201, metrics);
  assertCounts(3, 201L, 1L, metrics);
  assertBounds(3, Types.LongType.get(), 2L, 201L, metrics);
  assertCounts(4, 201L, 0L, metrics);
  assertBounds(4, Types.FloatType.get(), 1.0F, 201.0F, metrics);
  assertCounts(5, 201L, 1L, metrics);
  assertBounds(5, Types.DoubleType.get(), 2.0D, 201.0D, metrics);
  assertCounts(6, 201L, 1L, metrics);
  assertBounds(6, Types.DecimalType.of(10, 2), new BigDecimal("2.00"),
      new BigDecimal("201.00"), metrics);
}