Java Code Examples for org.apache.avro.Conversions#DecimalConversion

The following examples show how to use org.apache.avro.Conversions#DecimalConversion . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: TestGenericLogicalTypes.java    From parquet-mr with Apache License 2.0 6 votes vote down vote up 
@Test
public void testReadDecimalFixed() throws IOException {
  Schema fixedSchema = Schema.createFixed("aFixed", null, null, 4);
  Schema fixedRecord = record("R", field("dec", fixedSchema));
  Schema decimalSchema = DECIMAL_9_2.addToSchema(
      Schema.createFixed("aFixed", null, null, 4));
  Schema decimalRecord = record("R", field("dec", decimalSchema));

  GenericRecord r1 = instance(decimalRecord, "dec", D1);
  GenericRecord r2 = instance(decimalRecord, "dec", D2);
  List<GenericRecord> expected = Arrays.asList(r1, r2);

  Conversion<BigDecimal> conversion = new Conversions.DecimalConversion();

  // use the conversion directly instead of relying on the write side
  GenericRecord r1fixed = instance(fixedRecord, "dec",
      conversion.toFixed(D1, fixedSchema, DECIMAL_9_2));
  GenericRecord r2fixed = instance(fixedRecord, "dec",
      conversion.toFixed(D2, fixedSchema, DECIMAL_9_2));

  File test = write(fixedRecord, r1fixed, r2fixed);
  Assert.assertEquals("Should convert fixed to BigDecimals",
      expected, read(GENERIC, decimalRecord, test));
}
Example 2
Source File: TestGenericLogicalTypes.java    From parquet-mr with Apache License 2.0 6 votes vote down vote up 
@Test
public void testWriteDecimalFixed() throws IOException {
  Schema fixedSchema = Schema.createFixed("aFixed", null, null, 4);
  Schema fixedRecord = record("R", field("dec", fixedSchema));
  Schema decimalSchema = DECIMAL_9_2.addToSchema(
      Schema.createFixed("aFixed", null, null, 4));
  Schema decimalRecord = record("R", field("dec", decimalSchema));

  GenericRecord r1 = instance(decimalRecord, "dec", D1);
  GenericRecord r2 = instance(decimalRecord, "dec", D2);

  Conversion<BigDecimal> conversion = new Conversions.DecimalConversion();

  // use the conversion directly instead of relying on the write side
  GenericRecord r1fixed = instance(fixedRecord, "dec",
      conversion.toFixed(D1, fixedSchema, DECIMAL_9_2));
  GenericRecord r2fixed = instance(fixedRecord, "dec",
      conversion.toFixed(D2, fixedSchema, DECIMAL_9_2));
  List<GenericRecord> expected = Arrays.asList(r1fixed, r2fixed);

  File test = write(GENERIC, decimalRecord, r1, r2);
  Assert.assertEquals("Should read BigDecimals as fixed",
      expected, read(GENERIC, fixedRecord, test));
}
Example 3
Source File: TestGenericLogicalTypes.java    From parquet-mr with Apache License 2.0 6 votes vote down vote up 
@Test
public void testReadDecimalBytes() throws IOException {
  Schema bytesSchema = Schema.create(Schema.Type.BYTES);
  Schema bytesRecord = record("R", field("dec", bytesSchema));
  Schema decimalSchema = DECIMAL_9_2.addToSchema(Schema.create(Schema.Type.BYTES));
  Schema decimalRecord = record("R", field("dec", decimalSchema));

  GenericRecord r1 = instance(decimalRecord, "dec", D1);
  GenericRecord r2 = instance(decimalRecord, "dec", D2);
  List<GenericRecord> expected = Arrays.asList(r1, r2);

  Conversion<BigDecimal> conversion = new Conversions.DecimalConversion();

  // use the conversion directly instead of relying on the write side
  GenericRecord r1bytes = instance(bytesRecord, "dec",
      conversion.toBytes(D1, bytesSchema, DECIMAL_9_2));
  GenericRecord r2bytes = instance(bytesRecord, "dec",
      conversion.toBytes(D2, bytesSchema, DECIMAL_9_2));

  File test = write(bytesRecord, r1bytes, r2bytes);
  Assert.assertEquals("Should convert bytes to BigDecimals",
      expected, read(GENERIC, decimalRecord, test));
}
Example 4
Source File: TestGenericLogicalTypes.java    From parquet-mr with Apache License 2.0 6 votes vote down vote up 
@Test
public void testWriteDecimalBytes() throws IOException {
  Schema bytesSchema = Schema.create(Schema.Type.BYTES);
  Schema bytesRecord = record("R", field("dec", bytesSchema));
  Schema decimalSchema = DECIMAL_9_2.addToSchema(Schema.create(Schema.Type.BYTES));
  Schema decimalRecord = record("R", field("dec", decimalSchema));

  GenericRecord r1 = instance(decimalRecord, "dec", D1);
  GenericRecord r2 = instance(decimalRecord, "dec", D2);

  Conversion<BigDecimal> conversion = new Conversions.DecimalConversion();

  // use the conversion directly instead of relying on the write side
  GenericRecord r1bytes = instance(bytesRecord, "dec",
      conversion.toBytes(D1, bytesSchema, DECIMAL_9_2));
  GenericRecord r2bytes = instance(bytesRecord, "dec",
      conversion.toBytes(D2, bytesSchema, DECIMAL_9_2));

  List<GenericRecord> expected = Arrays.asList(r1bytes, r2bytes);

  File test = write(GENERIC, decimalRecord, r1, r2);
  Assert.assertEquals("Should read BigDecimals as bytes",
      expected, read(GENERIC, bytesRecord, test));
}
Example 5
Source File: FetchParquetTest.java    From nifi with Apache License 2.0 6 votes vote down vote up 
private void writeParquetUsersWithDecimal(final File parquetFile, int numUsers) throws IOException {
    if (parquetFile.exists()) {
        Assert.assertTrue(parquetFile.delete());
    }

    final BigDecimal initialAmount = new BigDecimal("1234567.0123456789");
    final AvroParquetWriter.Builder<GenericRecord> writerBuilder = createAvroParquetWriter(parquetFile, schemaWithDecimal);

    final List<Schema> amountSchemaUnion = schemaWithDecimal.getField("amount").schema().getTypes();
    final Schema amountSchema = amountSchemaUnion.stream().filter(s -> s.getType() == Schema.Type.FIXED).findFirst().orElse(null);
    Assert.assertNotNull(amountSchema);

    final Conversions.DecimalConversion decimalConversion = new Conversions.DecimalConversion();

    try (final ParquetWriter<GenericRecord> writer = writerBuilder.build()) {
        for (int i=0; i < numUsers; i++) {
            final BigDecimal incrementedAmount = initialAmount.add(new BigDecimal("1"));
            final GenericRecord user = new GenericData.Record(schemaWithDecimal);
            user.put("name", "Bob" + i);
            user.put("amount", decimalConversion.toFixed(incrementedAmount, amountSchema, amountSchema.getLogicalType()));

            writer.write(user);
        }
    }

}
Example 6
Source File: AvroToPdiConverter.java    From pentaho-hadoop-shims with Apache License 2.0 5 votes vote down vote up 
private Object convertToPentahoType( int pentahoType, ByteBuffer avroData, Schema field ) {
  Object pentahoData = null;
  if ( avroData != null ) {
    try {
      switch ( pentahoType ) {
        case ValueMetaInterface.TYPE_BIGNUMBER:
          Conversions.DecimalConversion converter = new Conversions.DecimalConversion();
          Schema schema = field;
          if ( schema.getType().equals( Schema.Type.UNION ) ) {
            List<Schema> schemas = field.getTypes();
            for ( Schema s : schemas ) {
              if ( !s.getName().equalsIgnoreCase( "null" ) ) {
                schema = s;
                break;
              }
            }
          }
          Object precision = schema.getObjectProp( AvroSpec.DECIMAL_PRECISION );
          Object scale = schema.getObjectProp( AvroSpec.DECIMAL_SCALE );
          LogicalTypes.Decimal decimalType =
            LogicalTypes.decimal( Integer.parseInt( precision.toString() ), Integer.parseInt( scale.toString() ) );
          pentahoData = converter.fromBytes( avroData, avroSchema, decimalType );
          break;
        case ValueMetaInterface.TYPE_BINARY:
          pentahoData = new byte[ avroData.remaining() ];
          avroData.get( (byte[]) pentahoData );
          break;
      }
    } catch ( Exception e ) {
      // If unable to do the type conversion just ignore. null will be returned.
    }
  }
  return pentahoData;
}
Example 7
Source File: AvroNestedReader.java    From pentaho-hadoop-shims with Apache License 2.0 5 votes vote down vote up 
/**
 * @param pentahoType
 * @param avroData
 * @param fieldSchema
 * @return
 */
public Object convertToKettleValue( AvroInputField pentahoType, ByteBuffer avroData, Schema fieldSchema ) {
  Object pentahoData = null;
  if ( avroData != null ) {
    try {
      switch ( pentahoType.getPentahoType() ) {
        case ValueMetaInterface.TYPE_BIGNUMBER:
          Conversions.DecimalConversion converter = new Conversions.DecimalConversion();
          Schema schema = fieldSchema;
          if ( schema.getType().equals( Schema.Type.UNION ) ) {
            List<Schema> schemas = schema.getTypes();
            for ( Schema s : schemas ) {
              if ( !s.getName().equalsIgnoreCase( "null" ) ) {
                schema = s;
                break;
              }
            }
          }
          Object precision = schema.getObjectProp( AvroSpec.DECIMAL_PRECISION );
          Object scale = schema.getObjectProp( AvroSpec.DECIMAL_SCALE );
          LogicalTypes.Decimal decimalType =
            LogicalTypes.decimal( Integer.parseInt( precision.toString() ), Integer.parseInt( scale.toString() ) );
          pentahoData = converter.fromBytes( avroData, m_schemaToUse, decimalType );
          break;
        case ValueMetaInterface.TYPE_BINARY:
          pentahoData = new byte[ avroData.remaining() ];
          avroData.get( (byte[]) pentahoData );
          break;
      }
    } catch ( Exception e ) {
      // If unable to do the type conversion just ignore. null will be returned.
    }
  }
  return pentahoData;
}
Example 8
Source File: PentahoAvroRecordReader.java    From pentaho-hadoop-shims with Apache License 2.0 5 votes vote down vote up 
private Object convertToPentahoType( int pentahoType, ByteBuffer avroData, Schema.Field field ) {
  Object pentahoData = null;
  if ( avroData != null ) {
    try {
      switch ( pentahoType ) {
        case ValueMetaInterface.TYPE_BIGNUMBER:
          Conversions.DecimalConversion converter = new Conversions.DecimalConversion();
          Schema schema = field.schema();
          if ( schema.getType().equals( Schema.Type.UNION ) ) {
            List<Schema> schemas = field.schema().getTypes();
            for ( Schema s : schemas ) {
              if ( !s.getName().equalsIgnoreCase( "null" ) ) {
                schema = s;
                break;
              }
            }
          }
          Object precision = schema.getObjectProp( AvroSpec.DECIMAL_PRECISION );
          Object scale = schema.getObjectProp( AvroSpec.DECIMAL_SCALE );
          LogicalTypes.Decimal decimalType =
            LogicalTypes.decimal( Integer.parseInt( precision.toString() ), Integer.parseInt( scale.toString() ) );
          pentahoData = converter.fromBytes( avroData, avroSchema, decimalType );
          break;
        case ValueMetaInterface.TYPE_BINARY:
          pentahoData = new byte[ avroData.remaining() ];
          avroData.get( (byte[]) pentahoData );
          break;
      }
    } catch ( Exception e ) {
      // If unable to do the type conversion just ignore. null will be returned.
    }
  }
  return pentahoData;
}
Example 9
Source File: HoodieTestDataGenerator.java    From hudi with Apache License 2.0 4 votes vote down vote up 
public static GenericRecord generateGenericRecord(String rowKey, String riderName, String driverName,
                                                  double timestamp, boolean isDeleteRecord,
                                                  boolean isFlattened) {
  GenericRecord rec = new GenericData.Record(isFlattened ? FLATTENED_AVRO_SCHEMA : AVRO_SCHEMA);
  rec.put("_row_key", rowKey);
  rec.put("timestamp", timestamp);
  rec.put("rider", riderName);
  rec.put("driver", driverName);
  rec.put("begin_lat", RAND.nextDouble());
  rec.put("begin_lon", RAND.nextDouble());
  rec.put("end_lat", RAND.nextDouble());
  rec.put("end_lon", RAND.nextDouble());

  if (isFlattened) {
    rec.put("fare", RAND.nextDouble() * 100);
    rec.put("currency", "USD");
  } else {
    rec.put("distance_in_meters", RAND.nextInt());
    rec.put("seconds_since_epoch", RAND.nextLong());
    rec.put("weight", RAND.nextFloat());
    byte[] bytes = "Canada".getBytes();
    rec.put("nation", ByteBuffer.wrap(bytes));
    long currentTimeMillis = System.currentTimeMillis();
    Date date = new Date(currentTimeMillis);
    rec.put("current_date", (int) date.toLocalDate().toEpochDay());
    rec.put("current_ts", currentTimeMillis);

    BigDecimal bigDecimal = new BigDecimal(String.format("%5f", RAND.nextFloat()));
    Schema decimalSchema = AVRO_SCHEMA.getField("height").schema();
    Conversions.DecimalConversion decimalConversions = new Conversions.DecimalConversion();
    GenericFixed genericFixed = decimalConversions.toFixed(bigDecimal, decimalSchema, LogicalTypes.decimal(10, 6));
    rec.put("height", genericFixed);

    rec.put("city_to_state", Collections.singletonMap("LA", "CA"));

    GenericRecord fareRecord = new GenericData.Record(AVRO_SCHEMA.getField("fare").schema());
    fareRecord.put("amount", RAND.nextDouble() * 100);
    fareRecord.put("currency", "USD");
    rec.put("fare", fareRecord);

    GenericArray<GenericRecord> tipHistoryArray = new GenericData.Array<>(1, AVRO_SCHEMA.getField("tip_history").schema());
    Schema tipSchema = new Schema.Parser().parse(AVRO_SCHEMA.getField("tip_history").schema().toString()).getElementType();
    GenericRecord tipRecord = new GenericData.Record(tipSchema);
    tipRecord.put("amount", RAND.nextDouble() * 100);
    tipRecord.put("currency", "USD");
    tipHistoryArray.add(tipRecord);
    rec.put("tip_history", tipHistoryArray);
  }

  if (isDeleteRecord) {
    rec.put("_hoodie_is_deleted", true);
  } else {
    rec.put("_hoodie_is_deleted", false);
  }
  return rec;
}