Java Code Examples for org.apache.avro.specific.SpecificDatumWriter#write()

The following examples show how to use org.apache.avro.specific.SpecificDatumWriter#write() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: GenericAvroSerializer.java    From geowave with Apache License 2.0 6 votes vote down vote up 
public static synchronized <T> byte[] serialize(final T avroObject, final Schema avroSchema) {

    try {
      final ByteArrayOutputStream os = new ByteArrayOutputStream();
      final BinaryEncoder encoder = ef.binaryEncoder(os, null);

      final String schemaName = getSchemaName(avroSchema);
      if (!writers.containsKey(schemaName)) {
        writers.put(schemaName, new SpecificDatumWriter<T>(avroSchema));
      }

      final SpecificDatumWriter<T> writer = writers.get(schemaName);
      writer.write(avroObject, encoder);
      encoder.flush();
      return os.toByteArray();
    } catch (final IOException e) {
      LOGGER.error("Unable to serialize Avro record to byte[]: " + e.getMessage(), e);
      return null;
    }
  }
Example 2
Source File: TypeUtils.java    From geowave with Apache License 2.0 6 votes vote down vote up 
private static <T> byte[] deserialize(
    final T avroObject,
    final Schema avroSchema,
    final Class<T> avroClass) throws IOException {

  final ByteArrayOutputStream os = new ByteArrayOutputStream();
  final BinaryEncoder encoder = ef.binaryEncoder(os, null);
  if (!writers.containsKey(avroClass.toString())) {
    writers.put(avroClass.toString(), new SpecificDatumWriter<T>(avroSchema));
  }

  final SpecificDatumWriter<T> writer = writers.get(avroClass.toString());
  writer.write(avroObject, encoder);
  encoder.flush();
  return os.toByteArray();
}
Example 3
Source File: AvroLWM2MDataPublish.java    From SDA with BSD 2-Clause "Simplified" License 6 votes vote down vote up 
/**
 * 데이타 전송
 * @param event
 * @throws Exception
 * @return void
 */
public void send(COL_LWM2M event) throws Exception {
	EncoderFactory avroEncoderFactory = EncoderFactory.get();
	SpecificDatumWriter<COL_LWM2M> avroEventWriter = new SpecificDatumWriter<COL_LWM2M>(COL_LWM2M.SCHEMA$);
	
	ByteArrayOutputStream stream = new ByteArrayOutputStream();
	BinaryEncoder binaryEncoder = avroEncoderFactory.binaryEncoder(stream,null);

	try {
		avroEventWriter.write(event, binaryEncoder);
		binaryEncoder.flush();
	} catch (IOException e) {
		e.printStackTrace();
		throw e;
	}
	IOUtils.closeQuietly(stream);

	KeyedMessage<String, byte[]> data = new KeyedMessage<String, byte[]>(
			TOPIC, stream.toByteArray());

	producer.send(data);
}
Example 4
Source File: AvroOneM2MDataPublish.java    From SDA with BSD 2-Clause "Simplified" License 6 votes vote down vote up 
/**
 * 데이타 전송
 * @param event
 * @throws Exception
 * @return void
 */
public void send(COL_ONEM2M event) throws Exception {
	EncoderFactory avroEncoderFactory = EncoderFactory.get();
	SpecificDatumWriter<COL_ONEM2M> avroEventWriter = new SpecificDatumWriter<COL_ONEM2M>(COL_ONEM2M.SCHEMA$);
	
	ByteArrayOutputStream stream = new ByteArrayOutputStream();
	BinaryEncoder binaryEncoder = avroEncoderFactory.binaryEncoder(stream,null);

	try {
		avroEventWriter.write(event, binaryEncoder);
		binaryEncoder.flush();
	} catch (IOException e) {
		e.printStackTrace();
		throw e;
	}
	IOUtils.closeQuietly(stream);

	KeyedMessage<String, byte[]> data = new KeyedMessage<String, byte[]>(
			TOPIC, stream.toByteArray());

	producer.send(data);
}
Example 5
Source File: AvroLWM2MDataPublish.java    From SDA with BSD 2-Clause "Simplified" License 6 votes vote down vote up 
/**
 * 데이타 전송
 * @param event
 * @throws Exception
 * @return void
 */
public void send(COL_LWM2M event) throws Exception {
	EncoderFactory avroEncoderFactory = EncoderFactory.get();
	SpecificDatumWriter<COL_LWM2M> avroEventWriter = new SpecificDatumWriter<COL_LWM2M>(COL_LWM2M.SCHEMA$);
	
	ByteArrayOutputStream stream = new ByteArrayOutputStream();
	BinaryEncoder binaryEncoder = avroEncoderFactory.binaryEncoder(stream,null);

	try {
		avroEventWriter.write(event, binaryEncoder);
		binaryEncoder.flush();
	} catch (IOException e) {
		e.printStackTrace();
		throw e;
	}
	IOUtils.closeQuietly(stream);

	KeyedMessage<String, byte[]> data = new KeyedMessage<String, byte[]>(
			TOPIC, stream.toByteArray());

	producer.send(data);
}
Example 6
Source File: AvroOneM2MDataPublish.java    From SDA with BSD 2-Clause "Simplified" License 6 votes vote down vote up 
/**
 * 데이타 전송
 * @param event
 * @throws Exception
 * @return void
 */
public void send(COL_ONEM2M event) throws Exception {
	EncoderFactory avroEncoderFactory = EncoderFactory.get();
	SpecificDatumWriter<COL_ONEM2M> avroEventWriter = new SpecificDatumWriter<COL_ONEM2M>(COL_ONEM2M.SCHEMA$);
	
	ByteArrayOutputStream stream = new ByteArrayOutputStream();
	BinaryEncoder binaryEncoder = avroEncoderFactory.binaryEncoder(stream,null);

	try {
		avroEventWriter.write(event, binaryEncoder);
		binaryEncoder.flush();
	} catch (IOException e) {
		e.printStackTrace();
		throw e;
	}
	IOUtils.closeQuietly(stream);

	KeyedMessage<String, byte[]> data = new KeyedMessage<String, byte[]>(
			TOPIC, stream.toByteArray());

	producer.send(data);
}
Example 7
Source File: JavaToAvroSerializer.java    From Decision with Apache License 2.0 6 votes vote down vote up 
private byte[] getInsertMessageBytes(InsertMessage insertMessage){

        byte[] result = null;

        ByteArrayOutputStream out = new ByteArrayOutputStream();
        BinaryEncoder encoder = EncoderFactory.get().binaryEncoder(out, null);
        SpecificDatumWriter writer = new SpecificDatumWriter<InsertMessage>(InsertMessage.getClassSchema());

        try {
            writer.write(insertMessage, encoder);
            encoder.flush();
            out.close();
            result = out.toByteArray();
        }catch (IOException e){
            return null;
        }

        return result;

    }
Example 8
Source File: HardwareEmulatorMain.java    From Decision with Apache License 2.0 6 votes vote down vote up 
private byte[] getInsertMessageBytes(InsertMessage insertMessage){

            byte[] result = null;

            ByteArrayOutputStream out = new ByteArrayOutputStream();
            BinaryEncoder encoder = EncoderFactory.get().binaryEncoder(out, null);
            SpecificDatumWriter writer = new SpecificDatumWriter<InsertMessage>(InsertMessage.getClassSchema());

            try {
                writer.write(insertMessage, encoder);
                encoder.flush();
                out.close();
                result = out.toByteArray();
            }catch (IOException e){
                return null;
            }

            return result;

        }
Example 9
Source File: AvroRowDataDeSerializationSchemaTest.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Test
public void testSpecificType() throws Exception {
	JodaTimeRecord record = new JodaTimeRecord();
	record.setTypeTimestampMillis(DateTime.parse("2010-06-30T01:20:20"));
	record.setTypeDate(LocalDate.parse("2014-03-01"));
	record.setTypeTimeMillis(LocalTime.parse("12:12:12"));
	SpecificDatumWriter<JodaTimeRecord> datumWriter = new SpecificDatumWriter<>(JodaTimeRecord.class);
	ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
	Encoder encoder = EncoderFactory.get().binaryEncoder(byteArrayOutputStream, null);
	datumWriter.write(record, encoder);
	encoder.flush();
	byte[] input = byteArrayOutputStream.toByteArray();

	DataType dataType = ROW(
			FIELD("type_timestamp_millis", TIMESTAMP(3)),
			FIELD("type_date", DATE()),
			FIELD("type_time_millis", TIME(3)));
	final RowType rowType = (RowType) dataType.getLogicalType();
	final TypeInformation<RowData> typeInfo = new RowDataTypeInfo(rowType);
	AvroRowDataSerializationSchema serializationSchema = new AvroRowDataSerializationSchema(rowType);
	serializationSchema.open(null);
	AvroRowDataDeserializationSchema deserializationSchema =
			new AvroRowDataDeserializationSchema(rowType, typeInfo);
	deserializationSchema.open(null);

	RowData rowData = deserializationSchema.deserialize(input);
	byte[] output = serializationSchema.serialize(rowData);
	RowData rowData2 = deserializationSchema.deserialize(output);
	Assert.assertEquals(rowData, rowData2);
	Assert.assertEquals("2010-06-30T01:20:20", rowData.getTimestamp(0, 3).toString());
	Assert.assertEquals("2014-03-01", DataFormatConverters.LocalDateConverter.INSTANCE.toExternal(
			rowData.getInt(1)).toString());
	Assert.assertEquals("12:12:12", DataFormatConverters.LocalTimeConverter.INSTANCE.toExternal(
			rowData.getInt(2)).toString());
}
Example 10
Source File: RecordTest.java    From gradle-avro-plugin with Apache License 2.0 5 votes vote down vote up 
@Disabled
@ParameterizedTest
@MethodSource
<T extends SpecificRecord> void buildAndWriteRecord(T record) throws Exception {
    Schema schema = record.getSchema();
    SpecificDatumWriter<T> writer = new SpecificDatumWriter<>();
    OutputStream outputStream = new ByteArrayOutputStream();
    Encoder jsonEncoder = encoderFactory.jsonEncoder(schema, outputStream, true);
    Encoder validatingEncoder = encoderFactory.validatingEncoder(schema, jsonEncoder);
    writer.write(record, validatingEncoder);
}
Example 11
Source File: WatcherAvroUtil.java    From reef with Apache License 2.0 5 votes vote down vote up 
public static String toString(final SpecificRecord record) {
  final String jsonEncodedRecord;
  try {
    final Schema schema = record.getSchema();
    final ByteArrayOutputStream bos = new ByteArrayOutputStream();
    final Encoder encoder = EncoderFactory.get().jsonEncoder(schema, bos);
    final SpecificDatumWriter datumWriter = new SpecificDatumWriter(record.getClass());
    datumWriter.write(record, encoder);
    encoder.flush();
    jsonEncodedRecord = new String(bos.toByteArray(), Charset.forName("UTF-8"));
  } catch (final IOException e) {
    throw new RuntimeException(e);
  }
  return jsonEncodedRecord;
}
Example 12
Source File: AvroSerializer.java    From secor with Apache License 2.0 5 votes vote down vote up 
public static byte[] serialize(SpecificDatumWriter<GenericRecord> writer, GenericRecord record) throws IOException {
    ByteArrayOutputStream out = new ByteArrayOutputStream();
    Encoder encoder = EncoderFactory.get().directBinaryEncoder(out, null);
    writer.write(record, encoder);
    encoder.flush();
    ByteBuffer serialized = ByteBuffer.allocate(out.toByteArray().length);
    serialized.put(out.toByteArray());
    return serialized.array();
}
Example 13
Source File: AvroCompatibilityHelperGeneratedEnumClassesTest.java    From avro-util with BSD 2-Clause "Simplified" License 5 votes vote down vote up 
private void roundtrip(Object thingie) throws Exception {
  Schema schema = SpecificData.get().getSchema(thingie.getClass());
  ByteArrayOutputStream os = new ByteArrayOutputStream();
  BinaryEncoder binaryEncoder = AvroCompatibilityHelper.newBinaryEncoder(os, false, null);
  SpecificDatumWriter<Object> writer = new SpecificDatumWriter<>(schema);
  writer.write(thingie, binaryEncoder);
  binaryEncoder.flush();
  byte[] serialized = os.toByteArray();
  ByteArrayInputStream is = new ByteArrayInputStream(serialized);
  BinaryDecoder binaryDecoder = AvroCompatibilityHelper.newBinaryDecoder(is, false, null);
  SpecificDatumReader<Object> reader = new SpecificDatumReader<>(schema);
  Object deserialize = reader.read(null, binaryDecoder);
  Assert.assertEquals(deserialize, thingie);
}
Example 14
Source File: FastSerdeTestsSupport.java    From avro-fastserde with Apache License 2.0 5 votes vote down vote up 
public static <T> Decoder serializeSpecific(T record, Schema schema) {
    ByteArrayOutputStream baos = new ByteArrayOutputStream();
    BinaryEncoder binaryEncoder = EncoderFactory.get().directBinaryEncoder(baos, null);

    try {
        SpecificDatumWriter<T> writer = new SpecificDatumWriter<>(schema);
        writer.write(record, binaryEncoder);
        binaryEncoder.flush();

    } catch (Exception e) {
        throw new RuntimeException(e);
    }

    return DecoderFactory.get().binaryDecoder(baos.toByteArray(), null);
}
Example 15
Source File: FastSerdeTestsSupport.java    From avro-util with BSD 2-Clause "Simplified" License 5 votes vote down vote up 
public static <T> Decoder specificDataAsDecoder(T record, Schema schema) {
  ByteArrayOutputStream baos = new ByteArrayOutputStream();
  Encoder binaryEncoder = AvroCompatibilityHelper.newBinaryEncoder(baos, true, null);

  try {
    SpecificDatumWriter<T> writer = new SpecificDatumWriter<>(schema);
    writer.write(record, binaryEncoder);
    binaryEncoder.flush();
  } catch (Exception e) {
    throw new RuntimeException(e);
  }

  return DecoderFactory.defaultFactory().createBinaryDecoder(baos.toByteArray(), null);
}
Example 16
Source File: AvroCompatibilityHelperGeneratedFixedClassesTest.java    From avro-util with BSD 2-Clause "Simplified" License 5 votes vote down vote up 
private void roundtrip(Object thingie) throws Exception {
  Schema schema = SpecificData.get().getSchema(thingie.getClass());
  ByteArrayOutputStream os = new ByteArrayOutputStream();
  BinaryEncoder binaryEncoder = AvroCompatibilityHelper.newBinaryEncoder(os, false, null);
  SpecificDatumWriter<Object> writer = new SpecificDatumWriter<>(schema);
  writer.write(thingie, binaryEncoder);
  binaryEncoder.flush();
  byte[] serialized = os.toByteArray();
  ByteArrayInputStream is = new ByteArrayInputStream(serialized);
  BinaryDecoder binaryDecoder = AvroCompatibilityHelper.newBinaryDecoder(is, false, null);
  SpecificDatumReader<Object> reader = new SpecificDatumReader<>(schema);
  Object deserialize = reader.read(null, binaryDecoder);
  Assert.assertEquals(deserialize, thingie);
}
Example 17
Source File: HBaseWriter.java    From hiped2 with Apache License 2.0 4 votes vote down vote up 
/**
 * The MapReduce driver - setup and launch the job.
 *
 * @param args the command-line arguments
 * @return the process exit code
 * @throws Exception if something goes wrong
 */
public int run(final String[] args) throws Exception {

  Cli cli = Cli.builder().setArgs(args).addOptions(CliCommonOpts.InputFileOption.values()).build();
  int result = cli.runCmd();

  if (result != 0) {
    return result;
  }

  File inputFile = new File(cli.getArgValueAsString(CliCommonOpts.InputFileOption.INPUT));

  Configuration conf = HBaseConfiguration.create();

  createTableAndColumn(conf, STOCKS_TABLE_NAME,
      STOCK_DETAILS_COLUMN_FAMILY_AS_BYTES);

  HTable htable = new HTable(conf, STOCKS_TABLE_NAME);
  htable.setAutoFlush(false);
  htable.setWriteBufferSize(1024 * 1024 * 12);


  SpecificDatumWriter<Stock> writer =
      new SpecificDatumWriter<Stock>();
  writer.setSchema(Stock.SCHEMA$);

  ByteArrayOutputStream bao = new ByteArrayOutputStream();
  BinaryEncoder encoder =
      EncoderFactory.get().directBinaryEncoder(bao, null);

  for (Stock stock: AvroStockUtils.fromCsvFile(inputFile)) {
    writer.write(stock, encoder);
    encoder.flush();

    byte[] rowkey = Bytes.add(
        Bytes.toBytes(stock.getSymbol().toString()),
        Bytes.toBytes(stock.getDate().toString()));

    byte[] stockAsAvroBytes = bao.toByteArray();

    Put put = new Put(rowkey);
    put.add(STOCK_DETAILS_COLUMN_FAMILY_AS_BYTES,
        STOCK_COLUMN_QUALIFIER_AS_BYTES,
        stockAsAvroBytes);

    htable.put(put);

    bao.reset();
  }

  htable.flushCommits();
  htable.close();
  System.out.println("done");

  return 0;
}