Java Code Examples for org.apache.avro.LogicalTypes#Decimal

The following examples show how to use org.apache.avro.LogicalTypes#Decimal . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: ParquetAvro.java    From iceberg with Apache License 2.0 7 votes vote down vote up
@Override
@SuppressWarnings("unchecked")
public Conversion<Object> getConversionFor(LogicalType logicalType) {
  if (logicalType == null) {
    return null;
  }

  if (logicalType instanceof LogicalTypes.Decimal) {
    LogicalTypes.Decimal decimal = (LogicalTypes.Decimal) logicalType;
    if (decimal.getPrecision() <= 9) {
      return (Conversion<Object>) intDecimalConversion;
    } else if (decimal.getPrecision() <= 18) {
      return (Conversion<Object>) longDecimalConversion;
    } else {
      return (Conversion<Object>) fixedDecimalConversion;
    }
  } else if ("uuid".equals(logicalType.getName())) {
    return (Conversion<Object>) uuidConversion;
  }
  return super.getConversionFor(logicalType);
}
 
Example 2
Source File: TestAvroTypeUtil.java    From nifi with Apache License 2.0 6 votes vote down vote up
@Test
public void testConvertAvroRecordToMapWithFieldTypeOfBinaryAndLogicalTypeDecimal() {
    // Create a field schema like {"type":"binary","name":"amount","logicalType":"decimal","precision":18,"scale":8}
    final LogicalTypes.Decimal decimalType = LogicalTypes.decimal(18, 8);
    final Schema fieldSchema = Schema.create(Type.BYTES);
    decimalType.addToSchema(fieldSchema);

    // Create a field named "amount" using the field schema above
    final Schema.Field field = new Schema.Field("amount", fieldSchema, null, (Object)null);

    // Create an overall record schema with the amount field
    final Schema avroSchema = Schema.createRecord(Collections.singletonList(field));

    // Create an example Avro record with the amount field of type binary and a logical type of decimal
    final BigDecimal expectedBigDecimal = new BigDecimal("1234567890.12345678");
    final GenericRecord genericRecord = new GenericData.Record(avroSchema);
    genericRecord.put("amount", new Conversions.DecimalConversion().toBytes(expectedBigDecimal, fieldSchema, decimalType));

    // Convert the Avro schema to a Record schema
    thenConvertAvroSchemaToRecordSchema(avroSchema, expectedBigDecimal, genericRecord);
}
 
Example 3
Source File: AvroSchemaConverter.java    From parquet-mr with Apache License 2.0 6 votes vote down vote up
private LogicalTypeAnnotation convertLogicalType(LogicalType logicalType) {
  if (logicalType == null) {
    return null;
  } else if (logicalType instanceof LogicalTypes.Decimal) {
    LogicalTypes.Decimal decimal = (LogicalTypes.Decimal) logicalType;
    return decimalType(decimal.getScale(), decimal.getPrecision());
  } else if (logicalType instanceof LogicalTypes.Date) {
    return dateType();
  } else if (logicalType instanceof LogicalTypes.TimeMillis) {
    return timeType(true, MILLIS);
  } else if (logicalType instanceof LogicalTypes.TimeMicros) {
    return timeType(true, MICROS);
  } else if (logicalType instanceof LogicalTypes.TimestampMillis) {
    return timestampType(true, MILLIS);
  } else if (logicalType instanceof LogicalTypes.TimestampMicros) {
    return timestampType(true, MICROS);
  } else if (logicalType.getName().equals(LogicalTypes.uuid().getName()) && writeParquetUUID) {
    return uuidType();
  }
  return null;
}
 
Example 4
Source File: TestAvroTypeUtil.java    From nifi with Apache License 2.0 6 votes vote down vote up
@Test
public void testConvertAvroRecordToMapWithFieldTypeOfFixedAndLogicalTypeDecimal() {
   // Create a field schema like {"type":"fixed","name":"amount","size":16,"logicalType":"decimal","precision":18,"scale":8}
   final LogicalTypes.Decimal decimalType = LogicalTypes.decimal(18, 8);
    final Schema fieldSchema = Schema.createFixed("amount", null, null, 16);
    decimalType.addToSchema(fieldSchema);

    // Create a field named "amount" using the field schema above
    final Schema.Field field = new Schema.Field("amount", fieldSchema, null, (Object)null);

    // Create an overall record schema with the amount field
    final Schema avroSchema = Schema.createRecord(Collections.singletonList(field));

    // Create an example Avro record with the amount field of type fixed and a logical type of decimal
    final BigDecimal expectedBigDecimal = new BigDecimal("1234567890.12345678");
    final GenericRecord genericRecord = new GenericData.Record(avroSchema);
    genericRecord.put("amount", new Conversions.DecimalConversion().toFixed(expectedBigDecimal, fieldSchema, decimalType));

    // Convert the Avro schema to a Record schema
    thenConvertAvroSchemaToRecordSchema(avroSchema, expectedBigDecimal, genericRecord);
}
 
Example 5
Source File: AvroRowSerializationSchema.java    From flink with Apache License 2.0 6 votes vote down vote up
private byte[] convertFromDecimal(Schema schema, BigDecimal decimal) {
	final LogicalType logicalType = schema.getLogicalType();
	if (logicalType instanceof LogicalTypes.Decimal) {
		final LogicalTypes.Decimal decimalType = (LogicalTypes.Decimal) logicalType;
		// rescale to target type
		final BigDecimal rescaled = decimal.setScale(decimalType.getScale(), BigDecimal.ROUND_UNNECESSARY);
		// byte array must contain the two's-complement representation of the
		// unscaled integer value in big-endian byte order
		return decimal.unscaledValue().toByteArray();
	} else {
		throw new RuntimeException("Unsupported decimal type.");
	}
}
 
Example 6
Source File: TestAvroTypeUtil.java    From nifi with Apache License 2.0 6 votes vote down vote up
@Test
public void testFixedDecimalConversion(){
    final LogicalTypes.Decimal decimalType = LogicalTypes.decimal(18, 8);
    final Schema fieldSchema = Schema.createFixed("mydecimal", "no doc", "myspace", 18);
    decimalType.addToSchema(fieldSchema);
    final Object convertedValue = AvroTypeUtil.convertToAvroObject("2.5", fieldSchema, StandardCharsets.UTF_8);
    assertTrue(convertedValue instanceof GenericFixed);
    final GenericFixed genericFixed = (GenericFixed)convertedValue;
    final BigDecimal bigDecimal = new Conversions.DecimalConversion().fromFixed(genericFixed, fieldSchema, decimalType);
    assertEquals(new BigDecimal("2.5").setScale(8), bigDecimal);
}
 
Example 7
Source File: AvroNestedReader.java    From pentaho-hadoop-shims with Apache License 2.0 5 votes vote down vote up
/**
 * @param pentahoType
 * @param avroData
 * @param fieldSchema
 * @return
 */
public Object convertToKettleValue( AvroInputField pentahoType, ByteBuffer avroData, Schema fieldSchema ) {
  Object pentahoData = null;
  if ( avroData != null ) {
    try {
      switch ( pentahoType.getPentahoType() ) {
        case ValueMetaInterface.TYPE_BIGNUMBER:
          Conversions.DecimalConversion converter = new Conversions.DecimalConversion();
          Schema schema = fieldSchema;
          if ( schema.getType().equals( Schema.Type.UNION ) ) {
            List<Schema> schemas = schema.getTypes();
            for ( Schema s : schemas ) {
              if ( !s.getName().equalsIgnoreCase( "null" ) ) {
                schema = s;
                break;
              }
            }
          }
          Object precision = schema.getObjectProp( AvroSpec.DECIMAL_PRECISION );
          Object scale = schema.getObjectProp( AvroSpec.DECIMAL_SCALE );
          LogicalTypes.Decimal decimalType =
            LogicalTypes.decimal( Integer.parseInt( precision.toString() ), Integer.parseInt( scale.toString() ) );
          pentahoData = converter.fromBytes( avroData, m_schemaToUse, decimalType );
          break;
        case ValueMetaInterface.TYPE_BINARY:
          pentahoData = new byte[ avroData.remaining() ];
          avroData.get( (byte[]) pentahoData );
          break;
      }
    } catch ( Exception e ) {
      // If unable to do the type conversion just ignore. null will be returned.
    }
  }
  return pentahoData;
}
 
Example 8
Source File: AvroRowSerializationSchema.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
private byte[] convertFromDecimal(Schema schema, BigDecimal decimal) {
	final LogicalType logicalType = schema.getLogicalType();
	if (logicalType instanceof LogicalTypes.Decimal) {
		final LogicalTypes.Decimal decimalType = (LogicalTypes.Decimal) logicalType;
		// rescale to target type
		final BigDecimal rescaled = decimal.setScale(decimalType.getScale(), BigDecimal.ROUND_UNNECESSARY);
		// byte array must contain the two's-complement representation of the
		// unscaled integer value in big-endian byte order
		return decimal.unscaledValue().toByteArray();
	} else {
		throw new RuntimeException("Unsupported decimal type.");
	}
}
 
Example 9
Source File: PentahoAvroRecordReader.java    From pentaho-hadoop-shims with Apache License 2.0 5 votes vote down vote up
private Object convertToPentahoType( int pentahoType, ByteBuffer avroData, Schema.Field field ) {
  Object pentahoData = null;
  if ( avroData != null ) {
    try {
      switch ( pentahoType ) {
        case ValueMetaInterface.TYPE_BIGNUMBER:
          Conversions.DecimalConversion converter = new Conversions.DecimalConversion();
          Schema schema = field.schema();
          if ( schema.getType().equals( Schema.Type.UNION ) ) {
            List<Schema> schemas = field.schema().getTypes();
            for ( Schema s : schemas ) {
              if ( !s.getName().equalsIgnoreCase( "null" ) ) {
                schema = s;
                break;
              }
            }
          }
          Object precision = schema.getObjectProp( AvroSpec.DECIMAL_PRECISION );
          Object scale = schema.getObjectProp( AvroSpec.DECIMAL_SCALE );
          LogicalTypes.Decimal decimalType =
            LogicalTypes.decimal( Integer.parseInt( precision.toString() ), Integer.parseInt( scale.toString() ) );
          pentahoData = converter.fromBytes( avroData, avroSchema, decimalType );
          break;
        case ValueMetaInterface.TYPE_BINARY:
          pentahoData = new byte[ avroData.remaining() ];
          avroData.get( (byte[]) pentahoData );
          break;
      }
    } catch ( Exception e ) {
      // If unable to do the type conversion just ignore. null will be returned.
    }
  }
  return pentahoData;
}
 
Example 10
Source File: ConvertAvroTypeToSQL.java    From components with Apache License 2.0 5 votes vote down vote up
private int convertAvroLogicialType(LogicalType logicalType) {

        Integer sqlType = this.config.CONVERT_LOGICALTYPE_TO_SQLTYPE.get(logicalType);
        if(sqlType != null){
            return sqlType;
        }

        if (logicalType == LogicalTypes.timestampMillis()) {
            sqlType = Types.TIMESTAMP;
        } else if (logicalType instanceof LogicalTypes.Decimal) {
            sqlType = Types.NUMERIC;
        } else if (logicalType == LogicalTypes.date()) {
            sqlType = Types.DATE;
        } else if (logicalType == LogicalTypes.uuid()) {
            sqlType = Types.VARCHAR;
        } else if (logicalType == LogicalTypes.timestampMicros()) {
            sqlType = Types.TIMESTAMP;
        } else if (logicalType == LogicalTypes.timeMillis()) {
            sqlType = Types.TIME;
        } else if (logicalType == LogicalTypes.timeMicros()) {
            sqlType = Types.TIME;
        } else {
            // All logical type should be supported
            throw new UnsupportedOperationException("Logical type " + logicalType + " not supported");
        }

        return sqlType;
    }
 
Example 11
Source File: AvroRowSerializationSchema.java    From flink with Apache License 2.0 5 votes vote down vote up
private byte[] convertFromDecimal(Schema schema, BigDecimal decimal) {
	final LogicalType logicalType = schema.getLogicalType();
	if (logicalType instanceof LogicalTypes.Decimal) {
		final LogicalTypes.Decimal decimalType = (LogicalTypes.Decimal) logicalType;
		// rescale to target type
		final BigDecimal rescaled = decimal.setScale(decimalType.getScale(), BigDecimal.ROUND_UNNECESSARY);
		// byte array must contain the two's-complement representation of the
		// unscaled integer value in big-endian byte order
		return decimal.unscaledValue().toByteArray();
	} else {
		throw new RuntimeException("Unsupported decimal type.");
	}
}
 
Example 12
Source File: AvroRowDeserializationSchema.java    From flink with Apache License 2.0 4 votes vote down vote up
private BigDecimal convertToDecimal(Schema schema, byte[] bytes) {
	final LogicalTypes.Decimal decimalType = (LogicalTypes.Decimal) schema.getLogicalType();
	return new BigDecimal(new BigInteger(bytes), decimalType.getScale());
}
 
Example 13
Source File: HoodieRealtimeRecordReaderUtils.java    From hudi with Apache License 2.0 4 votes vote down vote up
/**
 * Convert the projected read from delta record into an array writable.
 */
public static Writable avroToArrayWritable(Object value, Schema schema) {

  if (value == null) {
    return null;
  }

  switch (schema.getType()) {
    case STRING:
      return new Text(value.toString());
    case BYTES:
      return new BytesWritable(((ByteBuffer)value).array());
    case INT:
      return new IntWritable((Integer) value);
    case LONG:
      return new LongWritable((Long) value);
    case FLOAT:
      return new FloatWritable((Float) value);
    case DOUBLE:
      return new DoubleWritable((Double) value);
    case BOOLEAN:
      return new BooleanWritable((Boolean) value);
    case NULL:
      return null;
    case RECORD:
      GenericRecord record = (GenericRecord) value;
      Writable[] recordValues = new Writable[schema.getFields().size()];
      int recordValueIndex = 0;
      for (Schema.Field field : schema.getFields()) {
        recordValues[recordValueIndex++] = avroToArrayWritable(record.get(field.name()), field.schema());
      }
      return new ArrayWritable(Writable.class, recordValues);
    case ENUM:
      return new Text(value.toString());
    case ARRAY:
      GenericArray arrayValue = (GenericArray) value;
      Writable[] arrayValues = new Writable[arrayValue.size()];
      int arrayValueIndex = 0;
      for (Object obj : arrayValue) {
        arrayValues[arrayValueIndex++] = avroToArrayWritable(obj, schema.getElementType());
      }
      // Hive 1.x will fail here, it requires values2 to be wrapped into another ArrayWritable
      return new ArrayWritable(Writable.class, arrayValues);
    case MAP:
      Map mapValue = (Map) value;
      Writable[] mapValues = new Writable[mapValue.size()];
      int mapValueIndex = 0;
      for (Object entry : mapValue.entrySet()) {
        Map.Entry mapEntry = (Map.Entry) entry;
        Writable[] nestedMapValues = new Writable[2];
        nestedMapValues[0] = new Text(mapEntry.getKey().toString());
        nestedMapValues[1] = avroToArrayWritable(mapEntry.getValue(), schema.getValueType());
        mapValues[mapValueIndex++] = new ArrayWritable(Writable.class, nestedMapValues);
      }
      // Hive 1.x will fail here, it requires values3 to be wrapped into another ArrayWritable
      return new ArrayWritable(Writable.class, mapValues);
    case UNION:
      List<Schema> types = schema.getTypes();
      if (types.size() != 2) {
        throw new IllegalArgumentException("Only support union with 2 fields");
      }
      Schema s1 = types.get(0);
      Schema s2 = types.get(1);
      if (s1.getType() == Schema.Type.NULL) {
        return avroToArrayWritable(value, s2);
      } else if (s2.getType() == Schema.Type.NULL) {
        return avroToArrayWritable(value, s1);
      } else {
        throw new IllegalArgumentException("Only support union with null");
      }
    case FIXED:
      if (schema.getLogicalType() != null && schema.getLogicalType().getName().equals("decimal")) {
        LogicalTypes.Decimal decimal = (LogicalTypes.Decimal) LogicalTypes.fromSchema(schema);
        HiveDecimalWritable writable = new HiveDecimalWritable(((GenericFixed) value).bytes(),
            decimal.getScale());
        return HiveDecimalWritable.enforcePrecisionScale(writable,
            decimal.getPrecision(),
            decimal.getScale());
      }
      return new BytesWritable(((GenericFixed) value).bytes());
    default:
      return null;
  }
}
 
Example 14
Source File: TestJdbcCommon.java    From nifi with Apache License 2.0 4 votes vote down vote up
private void testConvertToAvroStreamForBigDecimal(BigDecimal bigDecimal, int dbPrecision, int defaultPrecision, int expectedPrecision, int expectedScale) throws SQLException, IOException {

        final ResultSetMetaData metadata = mock(ResultSetMetaData.class);
        when(metadata.getColumnCount()).thenReturn(1);
        when(metadata.getColumnType(1)).thenReturn(Types.NUMERIC);
        when(metadata.getColumnName(1)).thenReturn("The.Chairman");
        when(metadata.getTableName(1)).thenReturn("1the::table");
        when(metadata.getPrecision(1)).thenReturn(dbPrecision);
        when(metadata.getScale(1)).thenReturn(expectedScale);

        final ResultSet rs = JdbcCommonTestUtils.resultSetReturningMetadata(metadata);

        when(rs.getObject(Mockito.anyInt())).thenReturn(bigDecimal);

        final ByteArrayOutputStream baos = new ByteArrayOutputStream();

        final JdbcCommon.AvroConversionOptions options = JdbcCommon.AvroConversionOptions
                .builder().convertNames(true).useLogicalTypes(true).defaultPrecision(defaultPrecision).build();
        JdbcCommon.convertToAvroStream(rs, baos, options, null);

        final byte[] serializedBytes = baos.toByteArray();

        final InputStream instream = new ByteArrayInputStream(serializedBytes);

        final GenericData genericData = new GenericData();
        genericData.addLogicalTypeConversion(new Conversions.DecimalConversion());

        final DatumReader<GenericRecord> datumReader = new GenericDatumReader<>(null, null, genericData);
        try (final DataFileStream<GenericRecord> dataFileReader = new DataFileStream<>(instream, datumReader)) {
            final Schema generatedUnion = dataFileReader.getSchema().getField("The_Chairman").schema();
            // null and decimal.
            assertEquals(2, generatedUnion.getTypes().size());
            final LogicalType logicalType = generatedUnion.getTypes().get(1).getLogicalType();
            assertNotNull(logicalType);
            assertEquals("decimal", logicalType.getName());
            LogicalTypes.Decimal decimalType = (LogicalTypes.Decimal) logicalType;
            assertEquals(expectedPrecision, decimalType.getPrecision());
            assertEquals(expectedScale, decimalType.getScale());

            GenericRecord record = null;
            while (dataFileReader.hasNext()) {
                record = dataFileReader.next(record);
                assertEquals("_1the__table", record.getSchema().getName());
                assertEquals(bigDecimal, record.get("The_Chairman"));
            }
        }
    }
 
Example 15
Source File: AvroSchemaConverter.java    From Flink-CEPplus with Apache License 2.0 4 votes vote down vote up
private static TypeInformation<?> convertToTypeInfo(Schema schema) {
	switch (schema.getType()) {
		case RECORD:
			final List<Schema.Field> fields = schema.getFields();

			final TypeInformation<?>[] types = new TypeInformation<?>[fields.size()];
			final String[] names = new String[fields.size()];
			for (int i = 0; i < fields.size(); i++) {
				final Schema.Field field = fields.get(i);
				types[i] = convertToTypeInfo(field.schema());
				names[i] = field.name();
			}
			return Types.ROW_NAMED(names, types);
		case ENUM:
			return Types.STRING;
		case ARRAY:
			// result type might either be ObjectArrayTypeInfo or BasicArrayTypeInfo for Strings
			return Types.OBJECT_ARRAY(convertToTypeInfo(schema.getElementType()));
		case MAP:
			return Types.MAP(Types.STRING, convertToTypeInfo(schema.getValueType()));
		case UNION:
			final Schema actualSchema;
			if (schema.getTypes().size() == 2 && schema.getTypes().get(0).getType() == Schema.Type.NULL) {
				actualSchema = schema.getTypes().get(1);
			} else if (schema.getTypes().size() == 2 && schema.getTypes().get(1).getType() == Schema.Type.NULL) {
				actualSchema = schema.getTypes().get(0);
			} else if (schema.getTypes().size() == 1) {
				actualSchema = schema.getTypes().get(0);
			} else {
				// use Kryo for serialization
				return Types.GENERIC(Object.class);
			}
			return convertToTypeInfo(actualSchema);
		case FIXED:
			// logical decimal type
			if (schema.getLogicalType() instanceof LogicalTypes.Decimal) {
				return Types.BIG_DEC;
			}
			// convert fixed size binary data to primitive byte arrays
			return Types.PRIMITIVE_ARRAY(Types.BYTE);
		case STRING:
			// convert Avro's Utf8/CharSequence to String
			return Types.STRING;
		case BYTES:
			// logical decimal type
			if (schema.getLogicalType() instanceof LogicalTypes.Decimal) {
				return Types.BIG_DEC;
			}
			return Types.PRIMITIVE_ARRAY(Types.BYTE);
		case INT:
			// logical date and time type
			final LogicalType logicalType = schema.getLogicalType();
			if (logicalType == LogicalTypes.date()) {
				return Types.SQL_DATE;
			} else if (logicalType == LogicalTypes.timeMillis()) {
				return Types.SQL_TIME;
			}
			return Types.INT;
		case LONG:
			// logical timestamp type
			if (schema.getLogicalType() == LogicalTypes.timestampMillis()) {
				return Types.SQL_TIMESTAMP;
			}
			return Types.LONG;
		case FLOAT:
			return Types.FLOAT;
		case DOUBLE:
			return Types.DOUBLE;
		case BOOLEAN:
			return Types.BOOLEAN;
		case NULL:
			return Types.VOID;
	}
	throw new IllegalArgumentException("Unsupported Avro type '" + schema.getType() + "'.");
}
 
Example 16
Source File: AvroSchemaConverter190Int96Avro18.java    From datacollector with Apache License 2.0 4 votes vote down vote up
private Type convertFieldUsingLogicalType(String fieldName, Schema schema, Type.Repetition repetition) {
  LOG.debug("Converting field: {} using LogicalType", fieldName);
  Types.PrimitiveBuilder<PrimitiveType> builder;
  Schema.Type type = schema.getType();

  LogicalType logicalType = schema.getLogicalType();

  if (type.equals(Schema.Type.BOOLEAN)) {
    builder = Types.primitive(BOOLEAN, repetition);
  } else if (type.equals(Schema.Type.INT)) {
    builder = Types.primitive(INT32, repetition);
  } else if (type.equals(Schema.Type.LONG)) {
    // Special case handling timestamp until int96 fully supported or logical types correctly supported
    if (logicalType instanceof LogicalTypes.TimestampMillis || logicalType instanceof LogicalTypes.TimestampMicros) {
      LOG.debug("Logical type is a timestamp millis or micros");
      builder = Types.primitive(INT96, repetition);
    } else {
      builder = Types.primitive(INT64, repetition);
    }
  } else if (type.equals(Schema.Type.FLOAT)) {
    builder = Types.primitive(FLOAT, repetition);
  } else if (type.equals(Schema.Type.DOUBLE)) {
    builder = Types.primitive(DOUBLE, repetition);
  } else if (type.equals(Schema.Type.BYTES)) {
    builder = Types.primitive(BINARY, repetition);
  } else if (type.equals(Schema.Type.STRING)) {
    builder = Types.primitive(BINARY, repetition).as(UTF8);
  } else if (type.equals(Schema.Type.RECORD)) {
    return new GroupType(repetition, fieldName, convertFields(schema.getFields()));
  } else if (type.equals(Schema.Type.ENUM)) {
    builder = Types.primitive(BINARY, repetition).as(ENUM);
  } else if (type.equals(Schema.Type.ARRAY)) {
    if (writeOldListStructure) {
      return ConversionPatterns.listType(repetition, fieldName,
          convertField("array", schema.getElementType(), REPEATED));
    } else {
      return ConversionPatterns.listOfElements(repetition, fieldName,
          convertField(AvroWriteSupport.LIST_ELEMENT_NAME, schema.getElementType()));
    }
  } else if (type.equals(Schema.Type.MAP)) {
    Type valType = convertField("value", schema.getValueType());
    // avro map key type is always string
    return ConversionPatterns.stringKeyMapType(repetition, fieldName, valType);
  } else if (type.equals(Schema.Type.FIXED)) {
    builder = Types.primitive(FIXED_LEN_BYTE_ARRAY, repetition)
                   .length(schema.getFixedSize());
  } else if (type.equals(Schema.Type.UNION)) {
    return convertUnion(fieldName, schema, repetition);
  } else {
    throw new UnsupportedOperationException("Cannot convert Avro type " + type);
  }

  // schema translation can only be done for known logical types because this
  // creates an equivalence

  if (logicalType != null &&
      !(logicalType instanceof LogicalTypes.TimestampMillis || logicalType instanceof LogicalTypes.TimestampMicros)) {
    if (logicalType instanceof LogicalTypes.Decimal) {
      builder = builder.as(DECIMAL)
                       .precision(((LogicalTypes.Decimal) logicalType).getPrecision())
                       .scale(((LogicalTypes.Decimal) logicalType).getScale());

    } else {
      OriginalType annotation = convertLogicalType(logicalType);
      if (annotation != null) {
        builder.as(annotation);
      }
    }
  }

  return builder.named(fieldName);
}
 
Example 17
Source File: AvroSchemaConverter.java    From flink with Apache License 2.0 4 votes vote down vote up
private static TypeInformation<?> convertToTypeInfo(Schema schema) {
	switch (schema.getType()) {
		case RECORD:
			final List<Schema.Field> fields = schema.getFields();

			final TypeInformation<?>[] types = new TypeInformation<?>[fields.size()];
			final String[] names = new String[fields.size()];
			for (int i = 0; i < fields.size(); i++) {
				final Schema.Field field = fields.get(i);
				types[i] = convertToTypeInfo(field.schema());
				names[i] = field.name();
			}
			return Types.ROW_NAMED(names, types);
		case ENUM:
			return Types.STRING;
		case ARRAY:
			// result type might either be ObjectArrayTypeInfo or BasicArrayTypeInfo for Strings
			return Types.OBJECT_ARRAY(convertToTypeInfo(schema.getElementType()));
		case MAP:
			return Types.MAP(Types.STRING, convertToTypeInfo(schema.getValueType()));
		case UNION:
			final Schema actualSchema;
			if (schema.getTypes().size() == 2 && schema.getTypes().get(0).getType() == Schema.Type.NULL) {
				actualSchema = schema.getTypes().get(1);
			} else if (schema.getTypes().size() == 2 && schema.getTypes().get(1).getType() == Schema.Type.NULL) {
				actualSchema = schema.getTypes().get(0);
			} else if (schema.getTypes().size() == 1) {
				actualSchema = schema.getTypes().get(0);
			} else {
				// use Kryo for serialization
				return Types.GENERIC(Object.class);
			}
			return convertToTypeInfo(actualSchema);
		case FIXED:
			// logical decimal type
			if (schema.getLogicalType() instanceof LogicalTypes.Decimal) {
				return Types.BIG_DEC;
			}
			// convert fixed size binary data to primitive byte arrays
			return Types.PRIMITIVE_ARRAY(Types.BYTE);
		case STRING:
			// convert Avro's Utf8/CharSequence to String
			return Types.STRING;
		case BYTES:
			// logical decimal type
			if (schema.getLogicalType() instanceof LogicalTypes.Decimal) {
				return Types.BIG_DEC;
			}
			return Types.PRIMITIVE_ARRAY(Types.BYTE);
		case INT:
			// logical date and time type
			final org.apache.avro.LogicalType logicalType = schema.getLogicalType();
			if (logicalType == LogicalTypes.date()) {
				return Types.SQL_DATE;
			} else if (logicalType == LogicalTypes.timeMillis()) {
				return Types.SQL_TIME;
			}
			return Types.INT;
		case LONG:
			// logical timestamp type
			if (schema.getLogicalType() == LogicalTypes.timestampMillis()) {
				return Types.SQL_TIMESTAMP;
			}
			return Types.LONG;
		case FLOAT:
			return Types.FLOAT;
		case DOUBLE:
			return Types.DOUBLE;
		case BOOLEAN:
			return Types.BOOLEAN;
		case NULL:
			return Types.VOID;
	}
	throw new IllegalArgumentException("Unsupported Avro type '" + schema.getType() + "'.");
}
 
Example 18
Source File: AvroRowDeserializationSchema.java    From flink with Apache License 2.0 4 votes vote down vote up
private BigDecimal convertToDecimal(Schema schema, byte[] bytes) {
	final LogicalTypes.Decimal decimalType = (LogicalTypes.Decimal) schema.getLogicalType();
	return new BigDecimal(new BigInteger(bytes), decimalType.getScale());
}
 
Example 19
Source File: AvroSchemaConverter.java    From parquet-mr with Apache License 2.0 4 votes vote down vote up
@SuppressWarnings("deprecation")
private Type convertField(String fieldName, Schema schema, Type.Repetition repetition) {
  Types.PrimitiveBuilder<PrimitiveType> builder;
  Schema.Type type = schema.getType();
  LogicalType logicalType = schema.getLogicalType();
  if (type.equals(Schema.Type.BOOLEAN)) {
    builder = Types.primitive(BOOLEAN, repetition);
  } else if (type.equals(Schema.Type.INT)) {
    builder = Types.primitive(INT32, repetition);
  } else if (type.equals(Schema.Type.LONG)) {
    builder = Types.primitive(INT64, repetition);
  } else if (type.equals(Schema.Type.FLOAT)) {
    builder = Types.primitive(FLOAT, repetition);
  } else if (type.equals(Schema.Type.DOUBLE)) {
    builder = Types.primitive(DOUBLE, repetition);
  } else if (type.equals(Schema.Type.BYTES)) {
    builder = Types.primitive(BINARY, repetition);
  } else if (type.equals(Schema.Type.STRING)) {
    if (logicalType != null && logicalType.getName().equals(LogicalTypes.uuid().getName()) && writeParquetUUID) {
      builder = Types.primitive(FIXED_LEN_BYTE_ARRAY, repetition)
          .length(LogicalTypeAnnotation.UUIDLogicalTypeAnnotation.BYTES);
    } else {
      builder = Types.primitive(BINARY, repetition).as(stringType());
    }
  } else if (type.equals(Schema.Type.RECORD)) {
    return new GroupType(repetition, fieldName, convertFields(schema.getFields()));
  } else if (type.equals(Schema.Type.ENUM)) {
    builder = Types.primitive(BINARY, repetition).as(enumType());
  } else if (type.equals(Schema.Type.ARRAY)) {
    if (writeOldListStructure) {
      return ConversionPatterns.listType(repetition, fieldName,
          convertField("array", schema.getElementType(), REPEATED));
    } else {
      return ConversionPatterns.listOfElements(repetition, fieldName,
          convertField(AvroWriteSupport.LIST_ELEMENT_NAME, schema.getElementType()));
    }
  } else if (type.equals(Schema.Type.MAP)) {
    Type valType = convertField("value", schema.getValueType());
    // avro map key type is always string
    return ConversionPatterns.stringKeyMapType(repetition, fieldName, valType);
  } else if (type.equals(Schema.Type.FIXED)) {
    builder = Types.primitive(FIXED_LEN_BYTE_ARRAY, repetition)
        .length(schema.getFixedSize());
  } else if (type.equals(Schema.Type.UNION)) {
    return convertUnion(fieldName, schema, repetition);
  } else {
    throw new UnsupportedOperationException("Cannot convert Avro type " + type);
  }

  // schema translation can only be done for known logical types because this
  // creates an equivalence
  if (logicalType != null) {
    if (logicalType instanceof LogicalTypes.Decimal) {
      LogicalTypes.Decimal decimal = (LogicalTypes.Decimal) logicalType;
      builder = builder.as(decimalType(decimal.getScale(), decimal.getPrecision()));
    } else {
      LogicalTypeAnnotation annotation = convertLogicalType(logicalType);
      if (annotation != null) {
        builder.as(annotation);
      }
    }
  }

  return builder.named(fieldName);
}
 
Example 20
Source File: AvroRowDeserializationSchema.java    From Flink-CEPplus with Apache License 2.0 4 votes vote down vote up
private BigDecimal convertToDecimal(Schema schema, byte[] bytes) {
	final LogicalTypes.Decimal decimalType = (LogicalTypes.Decimal) schema.getLogicalType();
	return new BigDecimal(new BigInteger(bytes), decimalType.getScale());
}