org.apache.avro.LogicalTypes Java Examples
The following examples show how to use
org.apache.avro.LogicalTypes.
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: ParquetAvro.java From iceberg with Apache License 2.0 | 7 votes |
@Override @SuppressWarnings("unchecked") public Conversion<Object> getConversionFor(LogicalType logicalType) { if (logicalType == null) { return null; } if (logicalType instanceof LogicalTypes.Decimal) { LogicalTypes.Decimal decimal = (LogicalTypes.Decimal) logicalType; if (decimal.getPrecision() <= 9) { return (Conversion<Object>) intDecimalConversion; } else if (decimal.getPrecision() <= 18) { return (Conversion<Object>) longDecimalConversion; } else { return (Conversion<Object>) fixedDecimalConversion; } } else if ("uuid".equals(logicalType.getName())) { return (Conversion<Object>) uuidConversion; } return super.getConversionFor(logicalType); }
Example #2
Source File: GenerateAvroJavaTask.java From gradle-avro-plugin with Apache License 2.0 | 6 votes |
@TaskAction protected void process() { getLogger().debug("Using outputCharacterEncoding {}", getOutputCharacterEncoding().getOrNull()); getLogger().debug("Using stringType {}", stringTypeProvider.get().name()); getLogger().debug("Using fieldVisibility {}", fieldVisibilityProvider.get().name()); getLogger().debug("Using templateDirectory '{}'", getTemplateDirectory().getOrNull()); getLogger().debug("Using createSetters {}", isCreateSetters().get()); getLogger().debug("Using createOptionalGetters {}", isCreateOptionalGetters().get()); getLogger().debug("Using gettersReturnOptional {}", isGettersReturnOptional().get()); getLogger().debug("Using optionalGettersForNullableFieldsOnly {}", isOptionalGettersForNullableFieldsOnly().get()); getLogger().debug("Using enableDecimalLogicalType {}", isEnableDecimalLogicalType().get()); getLogger().debug("Using logicalTypeFactories {}", logicalTypeFactories.get().entrySet().stream().collect(Collectors.toMap( Map.Entry::getKey, (Map.Entry<String, Class<? extends LogicalTypes.LogicalTypeFactory>> e) -> e.getValue().getName() ))); getLogger().debug("Using customConversions {}", customConversions.get().stream().map(v -> ((Class) v).getName()).collect(Collectors.toList())); getLogger().info("Found {} files", getInputs().getSourceFiles().getFiles().size()); failOnUnsupportedFiles(); processFiles(); }
Example #3
Source File: TypeConverterUtils.java From components with Apache License 2.0 | 6 votes |
/** * Generate a schema from output type and format * * @param outputType * @param outputFormat * @return */ public static Schema getSchema(TypeConverterOutputTypes outputType, String outputFormat) { Schema result = Schema.create(outputType.getTargetType()); switch (outputType) { case Date: result = LogicalTypes.date().addToSchema(result); break; case Time: result = LogicalTypes.timeMillis().addToSchema(result); break; case DateTime: result = LogicalTypes.timestampMillis().addToSchema(result); break; } return result; }
Example #4
Source File: TestAvroTypeUtil.java From nifi with Apache License 2.0 | 6 votes |
@Test public void testConvertAvroRecordToMapWithFieldTypeOfBinaryAndLogicalTypeDecimal() { // Create a field schema like {"type":"binary","name":"amount","logicalType":"decimal","precision":18,"scale":8} final LogicalTypes.Decimal decimalType = LogicalTypes.decimal(18, 8); final Schema fieldSchema = Schema.create(Type.BYTES); decimalType.addToSchema(fieldSchema); // Create a field named "amount" using the field schema above final Schema.Field field = new Schema.Field("amount", fieldSchema, null, (Object)null); // Create an overall record schema with the amount field final Schema avroSchema = Schema.createRecord(Collections.singletonList(field)); // Create an example Avro record with the amount field of type binary and a logical type of decimal final BigDecimal expectedBigDecimal = new BigDecimal("1234567890.12345678"); final GenericRecord genericRecord = new GenericData.Record(avroSchema); genericRecord.put("amount", new Conversions.DecimalConversion().toBytes(expectedBigDecimal, fieldSchema, decimalType)); // Convert the Avro schema to a Record schema thenConvertAvroSchemaToRecordSchema(avroSchema, expectedBigDecimal, genericRecord); }
Example #5
Source File: TestAvroUtils.java From envelope with Apache License 2.0 | 6 votes |
@Test public void toTypeSchemaDecimalNullable() throws Exception { // Defaults to precision 10, scale 0 Schema schema = AvroUtils.typeFor(DataTypes.createDecimalType()); assertEquals("Invalid type", Schema.Type.UNION, schema.getType()); assertEquals("Invalid union size", 2, schema.getTypes().size()); for (Schema s : schema.getTypes()) { assertThat("Invalid union types", s.getType(), anyOf(is(Schema.Type.BYTES), is(Schema.Type.NULL))); if (s.getType().equals(Schema.Type.BYTES)) { assertEquals("Invalid LogicalType", LogicalTypes.decimal(10, 0), s.getLogicalType()); } } //System.out.println(schema.toString(true)); }
Example #6
Source File: TestGenericLogicalTypes.java From parquet-mr with Apache License 2.0 | 6 votes |
@Test public void testWriteUUIDReadStringSchema() throws IOException { Schema uuidSchema = record("R", field("uuid", LogicalTypes.uuid().addToSchema(Schema.create(STRING)))); GenericRecord u1 = instance(uuidSchema, "uuid", UUID.randomUUID()); GenericRecord u2 = instance(uuidSchema, "uuid", UUID.randomUUID()); Schema stringUuidSchema = Schema.create(STRING); stringUuidSchema.addProp(GenericData.STRING_PROP, "String"); Schema stringSchema = record("R", field("uuid", stringUuidSchema)); GenericRecord s1 = instance(stringSchema, "uuid", u1.get("uuid").toString()); GenericRecord s2 = instance(stringSchema, "uuid", u2.get("uuid").toString()); File test = write(GENERIC, uuidSchema, u1, u2); Assert.assertEquals("Should read UUIDs as Strings", Arrays.asList(s1, s2), read(GENERIC, stringSchema, test)); }
Example #7
Source File: TestAvroTypeUtil.java From nifi with Apache License 2.0 | 6 votes |
@Test public void testConvertAvroRecordToMapWithFieldTypeOfFixedAndLogicalTypeDecimal() { // Create a field schema like {"type":"fixed","name":"amount","size":16,"logicalType":"decimal","precision":18,"scale":8} final LogicalTypes.Decimal decimalType = LogicalTypes.decimal(18, 8); final Schema fieldSchema = Schema.createFixed("amount", null, null, 16); decimalType.addToSchema(fieldSchema); // Create a field named "amount" using the field schema above final Schema.Field field = new Schema.Field("amount", fieldSchema, null, (Object)null); // Create an overall record schema with the amount field final Schema avroSchema = Schema.createRecord(Collections.singletonList(field)); // Create an example Avro record with the amount field of type fixed and a logical type of decimal final BigDecimal expectedBigDecimal = new BigDecimal("1234567890.12345678"); final GenericRecord genericRecord = new GenericData.Record(avroSchema); genericRecord.put("amount", new Conversions.DecimalConversion().toFixed(expectedBigDecimal, fieldSchema, decimalType)); // Convert the Avro schema to a Record schema thenConvertAvroSchemaToRecordSchema(avroSchema, expectedBigDecimal, genericRecord); }
Example #8
Source File: TestReflectLogicalTypes.java From parquet-mr with Apache License 2.0 | 6 votes |
@Test public void testReadUUIDArray() throws IOException { Schema uuidArraySchema = SchemaBuilder.record(RecordWithUUIDArray.class.getName()) .fields() .name("uuids").type().array().items().stringType().noDefault() .endRecord(); LogicalTypes.uuid().addToSchema( uuidArraySchema.getField("uuids").schema().getElementType()); UUID u1 = UUID.randomUUID(); UUID u2 = UUID.randomUUID(); GenericRecord r = new GenericData.Record(uuidArraySchema); r.put("uuids", Arrays.asList(u1.toString(), u2.toString())); RecordWithUUIDArray expected = new RecordWithUUIDArray(); expected.uuids = new UUID[] {u1, u2}; File test = write(uuidArraySchema, r); Assert.assertEquals("Should convert Strings to UUIDs", expected, read(REFLECT, uuidArraySchema, test).get(0)); }
Example #9
Source File: AvroRecordConverter.java From DataflowTemplates with Apache License 2.0 | 6 votes |
private Optional<Date> readDate( GenericRecord record, Schema.Type avroType, LogicalType logicalType, String fieldName) { switch (avroType) { case INT: if (logicalType == null || !LogicalTypes.date().equals(logicalType)) { throw new IllegalArgumentException( "Cannot interpret Avrotype INT Logicaltype " + logicalType + " as DATE"); } // Avro Date is number of days since Jan 1, 1970. // Have to convert to Java Date first before creating google.cloud.core.Date return Optional.ofNullable((Integer) record.get(fieldName)) .map(x -> new java.util.Date((long) x * 24L * 3600L * 1000L)) .map(Date::fromJavaUtilDate); case STRING: return Optional.ofNullable((Utf8) record.get(fieldName)) .map(Utf8::toString) .map(Date::parseDate); default: throw new IllegalArgumentException("Cannot interpret " + avroType + " as DATE"); } }
Example #10
Source File: AvroRecordConverter.java From DataflowTemplates with Apache License 2.0 | 6 votes |
private Optional<Timestamp> readTimestamp( GenericRecord record, Schema.Type avroType, LogicalType logicalType, String fieldName) { switch (avroType) { case LONG: if (LogicalTypes.timestampMillis().equals(logicalType)) { return Optional.ofNullable((Long) record.get(fieldName)) .map(x -> Timestamp.ofTimeMicroseconds(1000L * x)); } if (LogicalTypes.timestampMicros().equals(logicalType)) { return Optional.ofNullable((Long) record.get(fieldName)) .map(Timestamp::ofTimeMicroseconds); } // Default to micro-seconds. return Optional.ofNullable((Long) record.get(fieldName)).map(Timestamp::ofTimeMicroseconds); case STRING: return Optional.ofNullable((Utf8) record.get(fieldName)) .map(Utf8::toString) .map(Timestamp::parseTimestamp); default: throw new IllegalArgumentException("Cannot interpret " + avroType + " as TIMESTAMP"); } }
Example #11
Source File: TestGenericLogicalTypes.java From parquet-mr with Apache License 2.0 | 6 votes |
@Test public void testWriteNullableUUID() throws IOException { Schema nullableUuidSchema = record("R", optionalField("uuid", LogicalTypes.uuid().addToSchema(Schema.create(STRING)))); GenericRecord u1 = instance(nullableUuidSchema, "uuid", UUID.randomUUID()); GenericRecord u2 = instance(nullableUuidSchema, "uuid", null); Schema stringUuidSchema = Schema.create(STRING); stringUuidSchema.addProp(GenericData.STRING_PROP, "String"); Schema nullableStringSchema = record("R", optionalField("uuid", stringUuidSchema)); GenericRecord s1 = instance(nullableStringSchema, "uuid", u1.get("uuid").toString()); GenericRecord s2 = instance(nullableStringSchema, "uuid", null); File test = write(GENERIC, nullableUuidSchema, u1, u2); Assert.assertEquals("Should read UUIDs as Strings", Arrays.asList(s1, s2), read(GENERIC, nullableStringSchema, test)); }
Example #12
Source File: GenerateAvroJavaTask.java From gradle-avro-plugin with Apache License 2.0 | 6 votes |
/** * Registers the logical types to be used in this run. * This must be called before the Schemas are parsed, or they will not be applied correctly. * Since {@link LogicalTypes} is a static registry, this may result in side-effects. */ private void registerLogicalTypes() { Map<String, Class<? extends LogicalTypes.LogicalTypeFactory>> logicalTypeFactoryMap = logicalTypeFactories.get(); Set<Map.Entry<String, Class<? extends LogicalTypes.LogicalTypeFactory>>> logicalTypeFactoryEntries = logicalTypeFactoryMap.entrySet(); for (Map.Entry<String, Class<? extends LogicalTypes.LogicalTypeFactory>> entry : logicalTypeFactoryEntries) { String logicalTypeName = entry.getKey(); Class<? extends LogicalTypes.LogicalTypeFactory> logicalTypeFactoryClass = entry.getValue(); try { LogicalTypes.LogicalTypeFactory logicalTypeFactory = logicalTypeFactoryClass.getDeclaredConstructor().newInstance(); LogicalTypes.register(logicalTypeName, logicalTypeFactory); } catch (ReflectiveOperationException ex) { getLogger().error("Could not instantiate logicalTypeFactory class \"" + logicalTypeFactoryClass.getName() + "\""); } } }
Example #13
Source File: TestGenericLogicalTypes.java From parquet-mr with Apache License 2.0 | 6 votes |
@Test public void testReadUUIDWithParquetUUID() throws IOException { Schema uuidSchema = record("R", field("uuid", LogicalTypes.uuid().addToSchema(Schema.create(STRING)))); GenericRecord u1 = instance(uuidSchema, "uuid", UUID.randomUUID()); GenericRecord u2 = instance(uuidSchema, "uuid", UUID.randomUUID()); File test = write(conf(AvroWriteSupport.WRITE_PARQUET_UUID, true), uuidSchema, u1, u2); Assert.assertEquals("Should read UUID objects", Arrays.asList(u1, u2), read(GENERIC, uuidSchema, test)); GenericRecord s1 = instance(uuidSchema, "uuid", u1.get("uuid").toString()); GenericRecord s2 = instance(uuidSchema, "uuid", u2.get("uuid").toString()); Assert.assertEquals("Should read UUID as Strings", Arrays.asList(s1, s2), read(GenericData.get(), uuidSchema, test)); }
Example #14
Source File: TestReflectLogicalTypes.java From parquet-mr with Apache License 2.0 | 6 votes |
@Test public void testReadUUIDList() throws IOException { Schema uuidListSchema = SchemaBuilder.record(RecordWithUUIDList.class.getName()) .fields() .name("uuids").type().array().items().stringType().noDefault() .endRecord(); uuidListSchema.getField("uuids").schema().addProp( SpecificData.CLASS_PROP, List.class.getName()); LogicalTypes.uuid().addToSchema( uuidListSchema.getField("uuids").schema().getElementType()); UUID u1 = UUID.randomUUID(); UUID u2 = UUID.randomUUID(); GenericRecord r = new GenericData.Record(uuidListSchema); r.put("uuids", Arrays.asList(u1.toString(), u2.toString())); RecordWithUUIDList expected = new RecordWithUUIDList(); expected.uuids = Arrays.asList(u1, u2); File test = write(uuidListSchema, r); Assert.assertEquals("Should convert Strings to UUIDs", expected, read(REFLECT, uuidListSchema, test).get(0)); }
Example #15
Source File: TestAvroUtils.java From envelope with Apache License 2.0 | 5 votes |
@Test public void toTypeSchemaDateNotNullable() throws Exception { Schema schema = AvroUtils.typeFor(DataTypes.DateType, false); assertEquals("Invalid type", Schema.Type.INT, schema.getType()); assertEquals("Invalid LogicalType", LogicalTypes.date(), schema.getLogicalType()); //System.out.println(schema.toString(true)); }
Example #16
Source File: TestReflectLogicalTypes.java From parquet-mr with Apache License 2.0 | 5 votes |
@Test public void testReadUUIDGenericRecordWithParquetUUID() throws IOException { Schema uuidSchema = SchemaBuilder.record("RecordWithUUID") .fields().requiredString("uuid").endRecord(); LogicalTypes.uuid().addToSchema(uuidSchema.getField("uuid").schema()); UUID u1 = UUID.randomUUID(); UUID u2 = UUID.randomUUID(); RecordWithStringUUID r1 = new RecordWithStringUUID(); r1.uuid = u1.toString(); RecordWithStringUUID r2 = new RecordWithStringUUID(); r2.uuid = u2.toString(); List<GenericData.Record> expected = Arrays.asList( new GenericData.Record(uuidSchema), new GenericData.Record(uuidSchema)); expected.get(0).put("uuid", u1); expected.get(1).put("uuid", u2); File test = write( AvroTestUtil.conf(AvroWriteSupport.WRITE_PARQUET_UUID, true), uuidSchema, r1, r2); Assert.assertEquals("Should convert Strings to UUIDs", expected, read(REFLECT, uuidSchema, test)); Schema uuidStringSchema = SchemaBuilder .record(RecordWithStringUUID.class.getName()) .fields().requiredString("uuid").endRecord(); LogicalTypes.uuid().addToSchema(uuidStringSchema.getField("uuid").schema()); Assert.assertEquals("Should not convert to UUID if accessor is String", Arrays.asList(r1, r2), read(REFLECT, uuidStringSchema, test)); }
Example #17
Source File: AvroSchemaTest.java From component-runtime with Apache License 2.0 | 5 votes |
@Test void checkDateConversionFromExternalAvro() { final org.apache.avro.Schema avro = SchemaBuilder .record("test") .fields() .name("date") .type(LogicalTypes.timestampMillis().addToSchema(Schema.create(Schema.Type.LONG))) .noDefault() .endRecord(); assertEquals(DATETIME, new AvroSchema(avro).getEntries().iterator().next().getType()); }
Example #18
Source File: TestReflectLogicalTypes.java From parquet-mr with Apache License 2.0 | 5 votes |
@Test public void testWriteUUID() throws IOException { Schema uuidSchema = SchemaBuilder.record(RecordWithUUID.class.getName()) .fields().requiredString("uuid").endRecord(); LogicalTypes.uuid().addToSchema(uuidSchema.getField("uuid").schema()); UUID u1 = UUID.randomUUID(); UUID u2 = UUID.randomUUID(); RecordWithUUID r1 = new RecordWithUUID(); r1.uuid = u1; RecordWithUUID r2 = new RecordWithUUID(); r2.uuid = u2; List<RecordWithStringUUID> expected = Arrays.asList( new RecordWithStringUUID(), new RecordWithStringUUID()); expected.get(0).uuid = u1.toString(); expected.get(1).uuid = u2.toString(); File test = write(REFLECT, uuidSchema, r1, r2); // verify that the field's type overrides the logical type Schema uuidStringSchema = SchemaBuilder .record(RecordWithStringUUID.class.getName()) .fields().requiredString("uuid").endRecord(); Assert.assertEquals("Should read uuid as String without UUID conversion", expected, read(REFLECT, uuidStringSchema, test)); LogicalTypes.uuid().addToSchema(uuidStringSchema.getField("uuid").schema()); Assert.assertEquals("Should read uuid as String without UUID logical type", expected, read(ReflectData.get(), uuidStringSchema, test)); }
Example #19
Source File: TestReflectLogicalTypes.java From parquet-mr with Apache License 2.0 | 5 votes |
@Test public void testWriteUUIDList() throws IOException { Schema uuidListSchema = SchemaBuilder.record(RecordWithUUIDList.class.getName()) .fields() .name("uuids").type().array().items().stringType().noDefault() .endRecord(); uuidListSchema.getField("uuids").schema().addProp( SpecificData.CLASS_PROP, List.class.getName()); LogicalTypes.uuid().addToSchema( uuidListSchema.getField("uuids").schema().getElementType()); Schema stringArraySchema = SchemaBuilder.record("RecordWithUUIDArray") .fields() .name("uuids").type().array().items().stringType().noDefault() .endRecord(); stringArraySchema.getField("uuids").schema() .addProp(SpecificData.CLASS_PROP, List.class.getName()); UUID u1 = UUID.randomUUID(); UUID u2 = UUID.randomUUID(); GenericRecord expected = new GenericData.Record(stringArraySchema); expected.put("uuids", Arrays.asList(u1.toString(), u2.toString())); RecordWithUUIDList r = new RecordWithUUIDList(); r.uuids = Arrays.asList(u1, u2); File test = write(REFLECT, uuidListSchema, r); Assert.assertEquals("Should read UUIDs as Strings", expected, read(REFLECT, stringArraySchema, test).get(0)); }
Example #20
Source File: PulsarMetadata.java From pulsar with Apache License 2.0 | 5 votes |
@VisibleForTesting static Type convertType(Schema.Type avroType, LogicalType logicalType) { switch (avroType) { case BOOLEAN: return BooleanType.BOOLEAN; case INT: if (logicalType == LogicalTypes.timeMillis()) { return TIME; } else if (logicalType == LogicalTypes.date()) { return DATE; } return IntegerType.INTEGER; case LONG: if (logicalType == LogicalTypes.timestampMillis()) { return TIMESTAMP; } return BigintType.BIGINT; case FLOAT: return RealType.REAL; case DOUBLE: return DoubleType.DOUBLE; case BYTES: return VarbinaryType.VARBINARY; case STRING: return VarcharType.VARCHAR; case ENUM: return VarcharType.VARCHAR; default: log.error("Cannot convert type: %s", avroType); return null; } }
Example #21
Source File: TestAvroSchemaConverter.java From parquet-mr with Apache License 2.0 | 5 votes |
@Test public void testUUIDTypeWithParquetUUID() throws Exception { Schema uuid = LogicalTypes.uuid().addToSchema(Schema.create(STRING)); Schema expected = Schema.createRecord("myrecord", null, null, false, Arrays.asList(new Schema.Field("uuid", uuid, null, null))); testRoundTripConversion(AvroTestUtil.conf(AvroWriteSupport.WRITE_PARQUET_UUID, true), expected, "message myrecord {\n" + " required fixed_len_byte_array(16) uuid (UUID);\n" + "}\n"); }
Example #22
Source File: TypeConverterUtils.java From components with Apache License 2.0 | 5 votes |
/** * @param srcSchema THe schema to investigate. * @return The Date/Time output type that corresponds to the logical type of the schema, or null if none. */ public static TypeConverterOutputTypes getDateTimeTypeFromLogicalType(Schema srcSchema) { LogicalType srcLogicalType = AvroUtils.unwrapIfNullable(srcSchema).getLogicalType(); if (srcLogicalType instanceof LogicalTypes.Date) { return TypeConverterOutputTypes.Date; } else if (srcLogicalType instanceof LogicalTypes.TimeMillis) { return TypeConverterOutputTypes.Time; } else if (srcLogicalType instanceof LogicalTypes.TimestampMillis) { return TypeConverterOutputTypes.DateTime; } else { return null; } }
Example #23
Source File: TestGenericLogicalTypes.java From parquet-mr with Apache License 2.0 | 5 votes |
@Test public void testWriteUUIDReadStringMissingLogicalType() throws IOException { Schema uuidSchema = record("R", field("uuid", LogicalTypes.uuid().addToSchema(Schema.create(STRING)))); GenericRecord u1 = instance(uuidSchema, "uuid", UUID.randomUUID()); GenericRecord u2 = instance(uuidSchema, "uuid", UUID.randomUUID()); GenericRecord s1 = instance(uuidSchema, "uuid", new Utf8(u1.get("uuid").toString())); GenericRecord s2 = instance(uuidSchema, "uuid", new Utf8(u2.get("uuid").toString())); File test = write(GENERIC, uuidSchema, u1, u2); Assert.assertEquals("Should read UUIDs as Strings", Arrays.asList(s1, s2), read(GenericData.get(), uuidSchema, test)); }
Example #24
Source File: LogicalTypeValidatingVisitor.java From data-highway with Apache License 2.0 | 5 votes |
@Override public void onVisit(Schema schema, Collection<String> breadcrumb) { try { LogicalTypes.fromSchema(schema); } catch (Exception e) { String path = breadcrumb.stream().collect(joining("/", "/", "")); throw new IllegalArgumentException("Invalid logical type declared at " + path, e); } }
Example #25
Source File: SnowflakeWriter.java From components with Apache License 2.0 | 5 votes |
private Object formatIfAnySnowflakeDateType(Object inputValue, Schema s) { if (LogicalTypes.fromSchemaIgnoreInvalid(s) == LogicalTypes.timeMillis()) { return formatter.formatTimeMillis(inputValue); } else if (LogicalTypes.fromSchemaIgnoreInvalid(s) == LogicalTypes.date()) { return formatter.formatDate(inputValue); } else if (LogicalTypes.fromSchemaIgnoreInvalid(s) == LogicalTypes.timestampMillis()) { return formatter.formatTimestampMillis(inputValue); } else { return inputValue; } }
Example #26
Source File: AvroSchema.java From component-runtime with Apache License 2.0 | 5 votes |
private Type doMapType(final Schema schema) { switch (schema.getType()) { case LONG: if (Boolean.parseBoolean(readProp(schema, Type.DATETIME.name())) || LogicalTypes.timestampMillis().equals(LogicalTypes.fromSchemaIgnoreInvalid(schema))) { return Type.DATETIME; } return Type.LONG; default: return Type.valueOf(schema.getType().name()); } }
Example #27
Source File: TestSchemaConversions.java From iceberg with Apache License 2.0 | 5 votes |
@Test public void testList() { Type list = Types.ListType.ofRequired(34, Types.UUIDType.get()); Schema schema = addElementId(34, SchemaBuilder.array().items( LogicalTypes.uuid().addToSchema(Schema.createFixed("uuid_fixed", null, null, 16)))); Assert.assertEquals("Avro schema to list", list, AvroSchemaUtil.convert(schema)); Assert.assertEquals("List to Avro schema", schema, AvroSchemaUtil.convert(list)); }
Example #28
Source File: AvroRowSerializationSchema.java From flink with Apache License 2.0 | 5 votes |
private byte[] convertFromDecimal(Schema schema, BigDecimal decimal) { final LogicalType logicalType = schema.getLogicalType(); if (logicalType instanceof LogicalTypes.Decimal) { final LogicalTypes.Decimal decimalType = (LogicalTypes.Decimal) logicalType; // rescale to target type final BigDecimal rescaled = decimal.setScale(decimalType.getScale(), BigDecimal.ROUND_UNNECESSARY); // byte array must contain the two's-complement representation of the // unscaled integer value in big-endian byte order return decimal.unscaledValue().toByteArray(); } else { throw new RuntimeException("Unsupported decimal type."); } }
Example #29
Source File: AvroRowSerializationSchema.java From flink with Apache License 2.0 | 5 votes |
private int convertFromDate(Schema schema, Date date) { final LogicalType logicalType = schema.getLogicalType(); if (logicalType == LogicalTypes.date()) { // adopted from Apache Calcite final long time = date.getTime(); final long converted = time + (long) LOCAL_TZ.getOffset(time); return (int) (converted / 86400000L); } else { throw new RuntimeException("Unsupported date type."); } }
Example #30
Source File: DataSourceUtils.java From hudi with Apache License 2.0 | 5 votes |
/** * Given an Avro field schema checks whether the field is of Logical Date Type or not. * * @param fieldSchema avro field schema * @return boolean indicating whether fieldSchema is of Avro's Date Logical Type */ private static boolean isLogicalTypeDate(Schema fieldSchema) { if (fieldSchema.getType() == Schema.Type.UNION) { return fieldSchema.getTypes().stream().anyMatch(schema -> schema.getLogicalType() == LogicalTypes.date()); } return fieldSchema.getLogicalType() == LogicalTypes.date(); }