org.apache.avro.Conversion Java Examples

@Override
@SuppressWarnings("unchecked")
public Conversion<Object> getConversionFor(LogicalType logicalType) {
  if (logicalType == null) {
    return null;
  }

  if (logicalType instanceof LogicalTypes.Decimal) {
    LogicalTypes.Decimal decimal = (LogicalTypes.Decimal) logicalType;
    if (decimal.getPrecision() <= 9) {
      return (Conversion<Object>) intDecimalConversion;
    } else if (decimal.getPrecision() <= 18) {
      return (Conversion<Object>) longDecimalConversion;
    } else {
      return (Conversion<Object>) fixedDecimalConversion;
    }
  } else if ("uuid".equals(logicalType.getName())) {
    return (Conversion<Object>) uuidConversion;
  }
  return super.getConversionFor(logicalType);
}
 

@Override
@SuppressWarnings("unchecked")
public <T> Conversion<T> getConversionByClass(Class<T> datumClass, LogicalType logicalType) {
  if (logicalType == null) {
    return null;
  }

  if (logicalType instanceof ParquetDecimal) {
    ParquetDecimal decimal = (ParquetDecimal) logicalType;
    if (decimal.precision() <= 9) {
      return (Conversion<T>) intDecimalConversion;
    } else if (decimal.precision() <= 18) {
      return (Conversion<T>) longDecimalConversion;
    } else {
      return (Conversion<T>) fixedDecimalConversion;
    }
  } else if ("uuid".equals(logicalType.getName())) {
    return (Conversion<T>) uuidConversion;
  }
  return super.getConversionByClass(datumClass, logicalType);
}
 

@Test
public void testWriteDecimalBytes() throws IOException {
  Schema bytesSchema = Schema.create(Schema.Type.BYTES);
  Schema bytesRecord = record("R", field("dec", bytesSchema));
  Schema decimalSchema = DECIMAL_9_2.addToSchema(Schema.create(Schema.Type.BYTES));
  Schema decimalRecord = record("R", field("dec", decimalSchema));

  GenericRecord r1 = instance(decimalRecord, "dec", D1);
  GenericRecord r2 = instance(decimalRecord, "dec", D2);

  Conversion<BigDecimal> conversion = new Conversions.DecimalConversion();

  // use the conversion directly instead of relying on the write side
  GenericRecord r1bytes = instance(bytesRecord, "dec",
      conversion.toBytes(D1, bytesSchema, DECIMAL_9_2));
  GenericRecord r2bytes = instance(bytesRecord, "dec",
      conversion.toBytes(D2, bytesSchema, DECIMAL_9_2));

  List<GenericRecord> expected = Arrays.asList(r1bytes, r2bytes);

  File test = write(GENERIC, decimalRecord, r1, r2);
  Assert.assertEquals("Should read BigDecimals as bytes",
      expected, read(GENERIC, bytesRecord, test));
}
 

@Test
public void testReadDecimalBytes() throws IOException {
  Schema bytesSchema = Schema.create(Schema.Type.BYTES);
  Schema bytesRecord = record("R", field("dec", bytesSchema));
  Schema decimalSchema = DECIMAL_9_2.addToSchema(Schema.create(Schema.Type.BYTES));
  Schema decimalRecord = record("R", field("dec", decimalSchema));

  GenericRecord r1 = instance(decimalRecord, "dec", D1);
  GenericRecord r2 = instance(decimalRecord, "dec", D2);
  List<GenericRecord> expected = Arrays.asList(r1, r2);

  Conversion<BigDecimal> conversion = new Conversions.DecimalConversion();

  // use the conversion directly instead of relying on the write side
  GenericRecord r1bytes = instance(bytesRecord, "dec",
      conversion.toBytes(D1, bytesSchema, DECIMAL_9_2));
  GenericRecord r2bytes = instance(bytesRecord, "dec",
      conversion.toBytes(D2, bytesSchema, DECIMAL_9_2));

  File test = write(bytesRecord, r1bytes, r2bytes);
  Assert.assertEquals("Should convert bytes to BigDecimals",
      expected, read(GENERIC, decimalRecord, test));
}
 

@Test
public void testWriteDecimalFixed() throws IOException {
  Schema fixedSchema = Schema.createFixed("aFixed", null, null, 4);
  Schema fixedRecord = record("R", field("dec", fixedSchema));
  Schema decimalSchema = DECIMAL_9_2.addToSchema(
      Schema.createFixed("aFixed", null, null, 4));
  Schema decimalRecord = record("R", field("dec", decimalSchema));

  GenericRecord r1 = instance(decimalRecord, "dec", D1);
  GenericRecord r2 = instance(decimalRecord, "dec", D2);

  Conversion<BigDecimal> conversion = new Conversions.DecimalConversion();

  // use the conversion directly instead of relying on the write side
  GenericRecord r1fixed = instance(fixedRecord, "dec",
      conversion.toFixed(D1, fixedSchema, DECIMAL_9_2));
  GenericRecord r2fixed = instance(fixedRecord, "dec",
      conversion.toFixed(D2, fixedSchema, DECIMAL_9_2));
  List<GenericRecord> expected = Arrays.asList(r1fixed, r2fixed);

  File test = write(GENERIC, decimalRecord, r1, r2);
  Assert.assertEquals("Should read BigDecimals as fixed",
      expected, read(GENERIC, fixedRecord, test));
}
 

@Test
public void testReadDecimalFixed() throws IOException {
  Schema fixedSchema = Schema.createFixed("aFixed", null, null, 4);
  Schema fixedRecord = record("R", field("dec", fixedSchema));
  Schema decimalSchema = DECIMAL_9_2.addToSchema(
      Schema.createFixed("aFixed", null, null, 4));
  Schema decimalRecord = record("R", field("dec", decimalSchema));

  GenericRecord r1 = instance(decimalRecord, "dec", D1);
  GenericRecord r2 = instance(decimalRecord, "dec", D2);
  List<GenericRecord> expected = Arrays.asList(r1, r2);

  Conversion<BigDecimal> conversion = new Conversions.DecimalConversion();

  // use the conversion directly instead of relying on the write side
  GenericRecord r1fixed = instance(fixedRecord, "dec",
      conversion.toFixed(D1, fixedSchema, DECIMAL_9_2));
  GenericRecord r2fixed = instance(fixedRecord, "dec",
      conversion.toFixed(D2, fixedSchema, DECIMAL_9_2));

  File test = write(fixedRecord, r1fixed, r2fixed);
  Assert.assertEquals("Should convert fixed to BigDecimals",
      expected, read(GENERIC, decimalRecord, test));
}
 

private <D> Object convert(Schema schema, LogicalType logicalType,
                           Conversion<D> conversion, Object datum) {
  if (conversion == null) {
    return datum;
  }
  Class<D> fromClass = conversion.getConvertedType();
  switch (schema.getType()) {
    case RECORD:  return conversion.toRecord(fromClass.cast(datum), schema, logicalType);
    case ENUM:    return conversion.toEnumSymbol(fromClass.cast(datum), schema, logicalType);
    case ARRAY:   return conversion.toArray(fromClass.cast(datum), schema, logicalType);
    case MAP:     return conversion.toMap(fromClass.cast(datum), schema, logicalType);
    case FIXED:   return conversion.toFixed(fromClass.cast(datum), schema, logicalType);
    case STRING:  return conversion.toCharSequence(fromClass.cast(datum), schema, logicalType);
    case BYTES:   return conversion.toBytes(fromClass.cast(datum), schema, logicalType);
    case INT:     return conversion.toInt(fromClass.cast(datum), schema, logicalType);
    case LONG:    return conversion.toLong(fromClass.cast(datum), schema, logicalType);
    case FLOAT:   return conversion.toFloat(fromClass.cast(datum), schema, logicalType);
    case DOUBLE:  return conversion.toDouble(fromClass.cast(datum), schema, logicalType);
    case BOOLEAN: return conversion.toBoolean(fromClass.cast(datum), schema, logicalType);
  }
  return datum;
}
 

@Override
public void write(T record) {
  if (rootLogicalType != null) {
    Conversion<?> conversion = model.getConversionByClass(
        record.getClass(), rootLogicalType);

    recordConsumer.startMessage();
    writeRecordFields(rootSchema, rootAvroSchema,
        convert(rootAvroSchema, rootLogicalType, conversion, record));
    recordConsumer.endMessage();

  } else {
    recordConsumer.startMessage();
    writeRecordFields(rootSchema, rootAvroSchema, record);
    recordConsumer.endMessage();
  }
}
 

private <D> Object convert(Schema schema, LogicalType logicalType,
    Conversion<D> conversion, Object datum) {
  if (conversion == null) {
    return datum;
  }
  Class<D> fromClass = conversion.getConvertedType();
  switch (schema.getType()) {
    case RECORD:  return conversion.toRecord(fromClass.cast(datum), schema, logicalType);
    case ENUM:    return conversion.toEnumSymbol(fromClass.cast(datum), schema, logicalType);
    case ARRAY:   return conversion.toArray(fromClass.cast(datum), schema, logicalType);
    case MAP:     return conversion.toMap(fromClass.cast(datum), schema, logicalType);
    case FIXED:   return conversion.toFixed(fromClass.cast(datum), schema, logicalType);
    case STRING:  return conversion.toCharSequence(fromClass.cast(datum), schema, logicalType);
    case BYTES:   return conversion.toBytes(fromClass.cast(datum), schema, logicalType);
    case INT:     return conversion.toInt(fromClass.cast(datum), schema, logicalType);
    case LONG:    return conversion.toLong(fromClass.cast(datum), schema, logicalType);
    case FLOAT:   return conversion.toFloat(fromClass.cast(datum), schema, logicalType);
    case DOUBLE:  return conversion.toDouble(fromClass.cast(datum), schema, logicalType);
    case BOOLEAN: return conversion.toBoolean(fromClass.cast(datum), schema, logicalType);
    default: break;
  }
  return datum;
}
 

@Override
public void write(T record) {
  if (rootLogicalType != null) {
    Conversion<?> conversion = model.getConversionByClass(
        record.getClass(), rootLogicalType);

    recordConsumer.startMessage();
    writeRecordFields(rootSchema, rootAvroSchema,
        convert(rootAvroSchema, rootLogicalType, conversion, record));
    recordConsumer.endMessage();

  } else {
    recordConsumer.startMessage();
    writeRecordFields(rootSchema, rootAvroSchema, record);
    recordConsumer.endMessage();
  }
}
 

@Override
@SuppressWarnings("unchecked")
public Conversion<Object> getConversionFor(LogicalType logicalType) {
  if (logicalType == null) {
    return null;
  }

  if (logicalType instanceof LogicalTypes.Decimal) {
    LogicalTypes.Decimal decimal = (LogicalTypes.Decimal) logicalType;
    if (decimal.getPrecision() <= 9) {
      return (Conversion<Object>) intDecimalConversion;
    } else if (decimal.getPrecision() <= 18) {
      return (Conversion<Object>) longDecimalConversion;
    } else {
      return (Conversion<Object>) fixedDecimalConversion;
    }
  } else if ("uuid".equals(logicalType.getName())) {
    return (Conversion<Object>) uuidConversion;
  }
  return super.getConversionFor(logicalType);
}
 

@Override
@SuppressWarnings("unchecked")
public <T> Conversion<T> getConversionByClass(Class<T> datumClass, LogicalType logicalType) {
  if (logicalType == null) {
    return null;
  }

  if (logicalType instanceof ParquetDecimal) {
    ParquetDecimal decimal = (ParquetDecimal) logicalType;
    if (decimal.precision() <= 9) {
      return (Conversion<T>) intDecimalConversion;
    } else if (decimal.precision() <= 18) {
      return (Conversion<T>) longDecimalConversion;
    } else {
      return (Conversion<T>) fixedDecimalConversion;
    }
  } else if ("uuid".equals(logicalType.getName())) {
    return (Conversion<T>) uuidConversion;
  }
  return super.getConversionByClass(datumClass, logicalType);
}
 

/**
 * Calls an appropriate write method based on the value.
 * Value MUST not be null.
 *
 * @param type the Parquet type
 * @param avroSchema the Avro schema
 * @param value a non-null value to write
 */
private void writeValue(Type type, Schema avroSchema, Object value) {
  Schema nonNullAvroSchema = AvroSchemaConverter.getNonNull(avroSchema);
  LogicalType logicalType = nonNullAvroSchema.getLogicalType();
  if (logicalType != null) {
    Conversion<?> conversion = model.getConversionByClass(
        value.getClass(), logicalType);
    writeValueWithoutConversion(type, nonNullAvroSchema,
        convert(nonNullAvroSchema, logicalType, conversion, value));
  } else {
    writeValueWithoutConversion(type, nonNullAvroSchema, value);
  }
}
 

public LogicalTypePrimitiveContainer(ParentValueContainer wrapped,
                                     Schema schema, Conversion conversion) {
  this.wrapped = wrapped;
  this.schema = schema;
  this.logicalType = schema.getLogicalType();
  this.conversion = conversion;
}
 

public AvroRecordConverter(MessageType parquetSchema, Schema avroSchema,
                           GenericData baseModel) {
  this(null, parquetSchema, avroSchema, baseModel);
  LogicalType logicalType = avroSchema.getLogicalType();
  Conversion<?> conversion = baseModel.getConversionFor(logicalType);
  this.rootContainer = ParentValueContainer.getConversionContainer(new ParentValueContainer() {
    @Override
    @SuppressWarnings("unchecked")
    public void add(Object value) {
      AvroRecordConverter.this.currentRecord = (T) value;
    }
  }, conversion, avroSchema);
}
 

/**
 * Calls an appropriate write method based on the value.
 * Value MUST not be null.
 *
 * @param type the Parquet type
 * @param avroSchema the Avro schema
 * @param value a non-null value to write
 */
private void writeValue(Type type, Schema avroSchema, Object value) {
  Schema nonNullAvroSchema = AvroSchemaConverter.getNonNull(avroSchema);
  LogicalType logicalType = nonNullAvroSchema.getLogicalType();
  if (logicalType != null) {
    Conversion<?> conversion = model.getConversionByClass(
        value.getClass(), logicalType);
    writeValueWithoutConversion(type, nonNullAvroSchema,
        convert(nonNullAvroSchema, logicalType, conversion, value));
  } else {
    writeValueWithoutConversion(type, nonNullAvroSchema, value);
  }
}
 

@Override
public AvroExtension customConversion(Class<? extends Conversion<?>> conversionClass) {
    customConversions.add(conversionClass);
    return this;
}
 

public void setCustomConversions(Iterable<Class<? extends Conversion<?>>> customConversions) {
    this.customConversions.set(customConversions);
}
 

public void setCustomConversions(Provider<Iterable<Class<? extends Conversion<?>>>> provider) {
    this.customConversions.set(provider);
}
 

@Override
public ListProperty<Class<? extends Conversion<?>>> getCustomConversions() {
    return customConversions;
}
 

public void setCustomConversions(Iterable<Class<? extends Conversion<?>>> customConversions) {
    this.customConversions.set(customConversions);
}
 

public void setCustomConversions(Provider<Iterable<Class<? extends Conversion<?>>>> provider) {
    this.customConversions.set(provider);
}
 

@Optional
@Input
public ListProperty<Class<? extends Conversion<?>>> getCustomConversions() {
    return customConversions;
}
 

private static Converter newConverter(Schema schema, Type type,
    GenericData model, Class<?> knownClass, ParentValueContainer setter) {
  LogicalType logicalType = schema.getLogicalType();
  Conversion<?> conversion;

  if (knownClass != null) {
    conversion = model.getConversionByClass(knownClass, logicalType);
  } else {
    conversion = model.getConversionFor(logicalType);
  }

  ParentValueContainer parent = ParentValueContainer
      .getConversionContainer(setter, conversion, schema);

  switch (schema.getType()) {
  case BOOLEAN:
    return new AvroConverters.FieldBooleanConverter(parent);
  case INT:
    Class<?> intDatumClass = getDatumClass(conversion, knownClass, schema, model);
    if (intDatumClass == null) {
      return new AvroConverters.FieldIntegerConverter(parent);
    }
    if (intDatumClass == byte.class || intDatumClass == Byte.class) {
      return new AvroConverters.FieldByteConverter(parent);
    }
    if (intDatumClass == char.class || intDatumClass == Character.class) {
      return new AvroConverters.FieldCharConverter(parent);
    }
    if (intDatumClass == short.class || intDatumClass == Short.class) {
      return new AvroConverters.FieldShortConverter(parent);
    }
    return new AvroConverters.FieldIntegerConverter(parent);
  case LONG:
    return new AvroConverters.FieldLongConverter(parent);
  case FLOAT:
    return new AvroConverters.FieldFloatConverter(parent);
  case DOUBLE:
    return new AvroConverters.FieldDoubleConverter(parent);
  case BYTES:
    Class<?> byteDatumClass = getDatumClass(conversion, knownClass, schema, model);
    if (byteDatumClass == null) {
      return new AvroConverters.FieldByteBufferConverter(parent);
    }
    if (byteDatumClass.isArray() && byteDatumClass.getComponentType() == byte.class) {
      return new AvroConverters.FieldByteArrayConverter(parent);
    }
    return new AvroConverters.FieldByteBufferConverter(parent);
  case STRING:
    if (logicalType != null && logicalType.getName().equals(LogicalTypes.uuid().getName())) {
      return new AvroConverters.FieldUUIDConverter(parent, type.asPrimitiveType());
    }
    return newStringConverter(schema, model, parent);
  case RECORD:
    return new AvroRecordConverter(parent, type.asGroupType(), schema, model);
  case ENUM:
    return new AvroConverters.FieldEnumConverter(parent, schema, model);
  case ARRAY:
    Class<?> arrayDatumClass = getDatumClass(conversion, knownClass, schema, model);
    if (arrayDatumClass != null && arrayDatumClass.isArray()) {
      return new AvroArrayConverter(parent, type.asGroupType(), schema, model,
          arrayDatumClass);
    }
    return new AvroCollectionConverter(parent, type.asGroupType(), schema,
        model, arrayDatumClass);
  case MAP:
    return new MapConverter(parent, type.asGroupType(), schema, model);
  case UNION:
    return new AvroUnionConverter(parent, type, schema, model);
  case FIXED:
    return new AvroConverters.FieldFixedConverter(parent, schema, model);
  default:
    throw new UnsupportedOperationException(String.format(
        "Cannot convert Avro type: %s to Parquet type: %s", schema, type));
  }
}
 

private static Converter newConverter(Schema schema, Type type,
    GenericData model, ParentValueContainer setter) {

  LogicalType logicalType = schema.getLogicalType();
  // the expected type is always null because it is determined by the parent
  // datum class, which never helps for generic. when logical types are added
  // to specific, this should pass the expected type here.
  Conversion<?> conversion = model.getConversionFor(logicalType);
  ParentValueContainer parent = ParentValueContainer
      .getConversionContainer(setter, conversion, schema);

  switch (schema.getType()) {
  case ARRAY:
    return new AvroArrayConverter(parent, type.asGroupType(), schema, model);
  case BOOLEAN:
    return new AvroConverters.FieldBooleanConverter(parent);
  case BYTES:
    return new AvroConverters.FieldByteBufferConverter(parent);
  case DOUBLE:
    return new AvroConverters.FieldDoubleConverter(parent);
  case ENUM:
    return new FieldEnumConverter(parent, schema, model);
  case FIXED:
    return new FieldFixedConverter(parent, schema, model);
  case FLOAT:
    return new AvroConverters.FieldFloatConverter(parent);
  case INT:
    return new AvroConverters.FieldIntegerConverter(parent);
  case LONG:
    return new AvroConverters.FieldLongConverter(parent);
  case MAP:
    return new MapConverter(parent, type.asGroupType(), schema, model);
  case RECORD:
    return new AvroIndexedRecordConverter(parent, type.asGroupType(), schema, model);
  case STRING:
    return new AvroConverters.FieldStringConverter(parent);
  case UNION:
    return new AvroUnionConverter(parent, type, schema, model);
  case NULL: // fall through
  default:
    throw new UnsupportedOperationException(String.format("Cannot convert Avro type: %s" +
        " (Parquet type: %s) ", schema, type));
  }
}
 

ListProperty<Class<? extends Conversion<?>>> getCustomConversions(); 

AvroExtension customConversion(Class<? extends Conversion<?>> conversionClass);