org.apache.avro.io.DatumWriter#write

Source File: JsonUtils.java From localization_nifi with Apache License 2.0

8 votes

/**
 * Writes provided {@link GenericRecord} into the provided
 * {@link OutputStream} as JSON.
 */
public static void write(GenericRecord record, OutputStream out) {
    try {
        DatumWriter<GenericRecord> writer = new GenericDatumWriter<GenericRecord>(record.getSchema());
        JsonEncoder encoder = EncoderFactory.get().jsonEncoder(record.getSchema(), out);
        writer.write(record, encoder);
        encoder.flush();
    } catch (Exception e) {
        throw new IllegalStateException("Failed to read GenericRecord", e);
    }
}

Source File: DataSerializer.java From jMetalSP with MIT License

6 votes

public byte[] serializeMessage(S clazz, String path) {

        byte[] result = null;
        try {

            File file = new File(path);

            Schema schema = new Schema.Parser().parse(file);

            ByteArrayOutputStream out = new ByteArrayOutputStream();
            BinaryEncoder encoder = EncoderFactory.get().binaryEncoder(out, null);
            DatumWriter<S> dataFileWriter = new SpecificDatumWriter<S>(schema);
            dataFileWriter.write(clazz, encoder);
            encoder.flush();
            result=out.toByteArray();
            out.close();
        }catch (Exception ex){
            ex.printStackTrace();
        }
        return result;
    }

Source File: AvroSerializationSchema.java From pulsar with Apache License 2.0

6 votes

@Override
public byte[] serialize(T t) {
    if (null == t) {
        return null;
    }

    // Writer to serialize Avro record into a byte array.
    DatumWriter<T> writer = new SpecificDatumWriter<>(t.getSchema());
    // Output stream to serialize records into byte array.
    ByteArrayOutputStream arrayOutputStream = new ByteArrayOutputStream();
    // Low-level class for serialization of Avro values.
    Encoder encoder = EncoderFactory.get().binaryEncoder(arrayOutputStream, null);
    arrayOutputStream.reset();
    try {
        writer.write(t, encoder);
        encoder.flush();
    } catch (IOException e) {
        throw new RuntimeException("Error while serializing the record to Avro", e);
    }

    return arrayOutputStream.toByteArray();
}

Source File: TransformTest.java From schema-registry-transfer-smt with Apache License 2.0

6 votes

private ByteArrayOutputStream encodeAvroObject(org.apache.avro.Schema schema, int sourceId, Object datum) throws IOException {
    ByteArrayOutputStream out = new ByteArrayOutputStream();

    out.write(MAGIC_BYTE);
    out.write(ByteBuffer.allocate(ID_SIZE).putInt(sourceId).array());

    EncoderFactory encoderFactory = EncoderFactory.get();
    BinaryEncoder encoder = encoderFactory.directBinaryEncoder(out, null);
    Object
            value =
            datum instanceof NonRecordContainer ? ((NonRecordContainer) datum).getValue()
                    : datum;
    DatumWriter<Object> writer = new GenericDatumWriter<>(schema);
    writer.write(value, encoder);
    encoder.flush();

    return out;
}

Source File: TruckParkAppIntegrationTest.java From data-highway with Apache License 2.0

5 votes

@Override
public byte[] serialize(String topic, Record record) {
  try (ByteArrayOutputStream baos = new ByteArrayOutputStream()) {
    baos.write(DataDeserializer.MAGIC_BYTE);
    baos.write(Ints.toByteArray(VERSION));
    DatumWriter<Object> writer = new GenericDatumWriter<>(SCHEMA);
    BinaryEncoder encoder = EncoderFactory.get().directBinaryEncoder(baos, null);
    writer.write(record, encoder);
    encoder.flush();
    return baos.toByteArray();
  } catch (IOException unreachable) {
    throw new RuntimeException(unreachable);
  }
}

Source File: AvroCodec.java From schema-evolution-samples with Apache License 2.0

5 votes

@Override
public void encode(Object object, OutputStream outputStream) throws IOException {
	Schema schema = getSchema(object);
	Integer id = schemaRegistryClient.register(schema);
	DatumWriter writer = getDatumWriter(object.getClass(),schema);
	Encoder encoder = EncoderFactory.get().binaryEncoder(outputStream, null);
	outputStream.write(ByteBuffer.allocate(4).putInt(id).array());
	writer.write(object,encoder);
	encoder.flush();
}

Source File: AvroSerializer.java From spring-kafka with MIT License

5 votes

@Override
public byte[] serialize(String topic, T data) {
  try {
    byte[] result = null;

    if (data != null) {
      LOGGER.debug("data='{}'", data);

      ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
      BinaryEncoder binaryEncoder =
          EncoderFactory.get().binaryEncoder(byteArrayOutputStream, null);

      DatumWriter<GenericRecord> datumWriter = new GenericDatumWriter<>(data.getSchema());
      datumWriter.write(data, binaryEncoder);

      binaryEncoder.flush();
      byteArrayOutputStream.close();

      result = byteArrayOutputStream.toByteArray();
      LOGGER.debug("serialized data='{}'", DatatypeConverter.printHexBinary(result));
    }
    return result;
  } catch (IOException ex) {
    throw new SerializationException(
        "Can't serialize data='" + data + "' for topic='" + topic + "'", ex);
  }
}

Source File: AvroUtils.java From java-11-examples with Apache License 2.0

5 votes

public static byte[] serializeEmployee(Employee employee) throws IOException {
    DatumWriter<Employee> employeeDatumWriter = new SpecificDatumWriter<>(Employee.class);
    ByteArrayOutputStream os = new ByteArrayOutputStream();
    BinaryEncoder binaryEncoder = EncoderFactory.get().directBinaryEncoder(os, null);
    employeeDatumWriter.setSchema(Employee.getClassSchema());
    employeeDatumWriter.write(employee, binaryEncoder);
    os.flush();
    return os.toByteArray();
}

Source File: FastDeserializerGeneratorForReuseTest.java From avro-util with BSD 2-Clause "Simplified" License

5 votes

private static byte[] serialize(GenericRecord record, Schema schema) throws IOException {
  ByteArrayOutputStream baos = new ByteArrayOutputStream();
  Encoder encoder = AvroCompatibilityHelper.newBinaryEncoder(baos, true, null);
  DatumWriter datumWriter = new GenericDatumWriter(schema);
  datumWriter.write(record, encoder);
  encoder.flush();

  return baos.toByteArray();
}

Source File: GenericRecordBinaryEncoder.java From gcp-ingestion with Mozilla Public License 2.0

5 votes

/**
 * Encode a GenericRecord into an avro encoded payload.
 */
public byte[] encodeRecord(GenericRecord record, Schema schema) {
  try (ByteArrayOutputStream baos = new ByteArrayOutputStream()) {
    DatumWriter<GenericRecord> writer = new GenericDatumWriter<>(schema);
    Encoder encoder = EncoderFactory.get().binaryEncoder(baos, null);
    writer.write(record, encoder);
    encoder.flush();
    return baos.toByteArray();
  } catch (IOException e) {
    throw new UncheckedIOException(e);
  }
}

Source File: AvroUtils.java From localization_nifi with Apache License 2.0

5 votes

/**
 * Writes provided {@link GenericRecord} into the provided
 * {@link OutputStream}.
 */
public static void write(GenericRecord record, OutputStream out) {
    BinaryEncoder encoder = EncoderFactory.get().binaryEncoder(out, null);
    DatumWriter<GenericRecord> writer = new GenericDatumWriter<>(record.getSchema());
    try {
        writer.write(record, encoder);
        encoder.flush();
    } catch (Exception e) {
        throw new IllegalStateException("Failed to write AVRO record", e);
    }
}

Source File: TestConvertAvroToJSON.java From localization_nifi with Apache License 2.0

5 votes

@Test
public void testSingleSchemalessAvroMessage_wrapSingleMessage_noContainer() throws IOException {
    final TestRunner runner = TestRunners.newTestRunner(new ConvertAvroToJSON());
    runner.setProperty(ConvertAvroToJSON.CONTAINER_OPTIONS, ConvertAvroToJSON.CONTAINER_NONE);
    runner.setProperty(ConvertAvroToJSON.WRAP_SINGLE_RECORD, Boolean.toString(true));
    Schema schema = new Schema.Parser().parse(new File("src/test/resources/user.avsc"));
    String stringSchema = schema.toString();
    runner.setProperty(ConvertAvroToJSON.SCHEMA, stringSchema);

    final GenericRecord user1 = new GenericData.Record(schema);
    user1.put("name", "Alyssa");
    user1.put("favorite_number", 256);

    final ByteArrayOutputStream out1 = new ByteArrayOutputStream();
    final BinaryEncoder encoder = EncoderFactory.get().binaryEncoder(out1, null);
    final DatumWriter<GenericRecord> datumWriter = new GenericDatumWriter<>(schema);
    datumWriter.write(user1, encoder);

    encoder.flush();
    out1.flush();
    byte[] test = out1.toByteArray();
    runner.enqueue(test);

    runner.run();

    runner.assertAllFlowFilesTransferred(ConvertAvroToJSON.REL_SUCCESS, 1);
    final MockFlowFile out = runner.getFlowFilesForRelationship(ConvertAvroToJSON.REL_SUCCESS).get(0);
    out.assertContentEquals("{\"name\": \"Alyssa\", \"favorite_number\": 256, \"favorite_color\": null}");
}

Source File: TestConvertAvroToJSON.java From localization_nifi with Apache License 2.0

5 votes

@Test
public void testSingleSchemalessAvroMessage_wrapSingleMessage() throws IOException {
    final TestRunner runner = TestRunners.newTestRunner(new ConvertAvroToJSON());
    runner.setProperty(ConvertAvroToJSON.CONTAINER_OPTIONS, ConvertAvroToJSON.CONTAINER_ARRAY);
    runner.setProperty(ConvertAvroToJSON.WRAP_SINGLE_RECORD, Boolean.toString(true));
    Schema schema = new Schema.Parser().parse(new File("src/test/resources/user.avsc"));
    String stringSchema = schema.toString();
    runner.setProperty(ConvertAvroToJSON.SCHEMA, stringSchema);

    final GenericRecord user1 = new GenericData.Record(schema);
    user1.put("name", "Alyssa");
    user1.put("favorite_number", 256);

    final ByteArrayOutputStream out1 = new ByteArrayOutputStream();
    final BinaryEncoder encoder = EncoderFactory.get().binaryEncoder(out1, null);
    final DatumWriter<GenericRecord> datumWriter = new GenericDatumWriter<>(schema);
    datumWriter.write(user1, encoder);

    encoder.flush();
    out1.flush();
    byte[] test = out1.toByteArray();
    runner.enqueue(test);

    runner.run();

    runner.assertAllFlowFilesTransferred(ConvertAvroToJSON.REL_SUCCESS, 1);
    final MockFlowFile out = runner.getFlowFilesForRelationship(ConvertAvroToJSON.REL_SUCCESS).get(0);
    out.assertContentEquals("[{\"name\": \"Alyssa\", \"favorite_number\": 256, \"favorite_color\": null}]");
}

Source File: TestConvertAvroToJSON.java From localization_nifi with Apache License 2.0

5 votes

@Test
public void testSingleSchemalessAvroMessage_noContainer() throws IOException {
    final TestRunner runner = TestRunners.newTestRunner(new ConvertAvroToJSON());
    runner.setProperty(ConvertAvroToJSON.CONTAINER_OPTIONS, ConvertAvroToJSON.CONTAINER_NONE);
    Schema schema = new Schema.Parser().parse(new File("src/test/resources/user.avsc"));
    String stringSchema = schema.toString();
    runner.setProperty(ConvertAvroToJSON.SCHEMA, stringSchema);

    final GenericRecord user1 = new GenericData.Record(schema);
    user1.put("name", "Alyssa");
    user1.put("favorite_number", 256);

    final ByteArrayOutputStream out1 = new ByteArrayOutputStream();
    final BinaryEncoder encoder = EncoderFactory.get().binaryEncoder(out1, null);
    final DatumWriter<GenericRecord> datumWriter = new GenericDatumWriter<>(schema);
    datumWriter.write(user1, encoder);

    encoder.flush();
    out1.flush();
    byte[] test = out1.toByteArray();
    runner.enqueue(test);

    runner.run();

    runner.assertAllFlowFilesTransferred(ConvertAvroToJSON.REL_SUCCESS, 1);
    final MockFlowFile out = runner.getFlowFilesForRelationship(ConvertAvroToJSON.REL_SUCCESS).get(0);
    out.assertContentEquals("{\"name\": \"Alyssa\", \"favorite_number\": 256, \"favorite_color\": null}");
}

Source File: TestConvertAvroToJSON.java From localization_nifi with Apache License 2.0

5 votes

@Test
public void testSingleSchemalessAvroMessage() throws IOException {
    final TestRunner runner = TestRunners.newTestRunner(new ConvertAvroToJSON());
    Schema schema = new Schema.Parser().parse(new File("src/test/resources/user.avsc"));
    String stringSchema = schema.toString();
    runner.setProperty(ConvertAvroToJSON.SCHEMA, stringSchema);

    final GenericRecord user1 = new GenericData.Record(schema);
    user1.put("name", "Alyssa");
    user1.put("favorite_number", 256);

    final ByteArrayOutputStream out1 = new ByteArrayOutputStream();
    final BinaryEncoder encoder = EncoderFactory.get().binaryEncoder(out1, null);
    final DatumWriter<GenericRecord> datumWriter = new GenericDatumWriter<>(schema);
    datumWriter.write(user1, encoder);

    encoder.flush();
    out1.flush();
    byte[] test = out1.toByteArray();
    runner.enqueue(test);

    runner.run();

    runner.assertAllFlowFilesTransferred(ConvertAvroToJSON.REL_SUCCESS, 1);
    final MockFlowFile out = runner.getFlowFilesForRelationship(ConvertAvroToJSON.REL_SUCCESS).get(0);
    out.assertContentEquals("{\"name\": \"Alyssa\", \"favorite_number\": 256, \"favorite_color\": null}");
}

Source File: OfframpIntegrationTest.java From data-highway with Apache License 2.0

5 votes

private static byte[] encode(Record record) throws IOException {
  try (ByteArrayOutputStream buffer = new ByteArrayOutputStream()) {
    Encoder encoder = EncoderFactory.get().directBinaryEncoder(buffer, null);
    DatumWriter<Object> writer = new GenericDatumWriter<>(record.getSchema());
    buffer.write(0x00);
    buffer.write(Ints.toByteArray(1));
    writer.write(record, encoder);
    encoder.flush();
    return buffer.toByteArray();
  }
}

Source File: DataDeserializerTest.java From data-highway with Apache License 2.0

5 votes

private byte[] toAvroBinary(Schema schema, Object value, int version) {
  try (ByteArrayOutputStream baos = new ByteArrayOutputStream()) {
    baos.write(0x0);
    baos.write(Ints.toByteArray(version));
    DatumWriter<Object> writer = new GenericDatumWriter<>(schema);
    Encoder encoder = EncoderFactory.get().directBinaryEncoder(baos, null);
    writer.write(value, encoder);
    encoder.flush();
    return baos.toByteArray();
  } catch (IOException e) {
    throw new RuntimeException(e);
  }
}

Source File: DataDeserializerGdprTest.java From data-highway with Apache License 2.0

5 votes

private byte[] toAvroBinary(Schema schema, Object value, int version) {
  try (ByteArrayOutputStream baos = new ByteArrayOutputStream()) {
    baos.write(0x0);
    baos.write(Ints.toByteArray(version));
    DatumWriter<Object> writer = new GenericDatumWriter<>(schema);
    Encoder encoder = EncoderFactory.get().directBinaryEncoder(baos, null);
    writer.write(value, encoder);
    encoder.flush();
    return baos.toByteArray();
  } catch (IOException e) {
    throw new RuntimeException(e);
  }
}

Source File: AvroKafkaSerializer.java From apicurio-registry with Apache License 2.0

5 votes

@Override
protected void serializeData(Schema schema, U data, OutputStream out) throws IOException {
    BinaryEncoder encoder = encoderFactory.directBinaryEncoder(out, null);

    // I guess this can happen if generics are lost with reflection ...
    if (data instanceof NonRecordContainer) {
        //noinspection unchecked
        data = (U) NonRecordContainer.class.cast(data).getValue();
    }

    DatumWriter<U> writer = avroDatumProvider.createDatumWriter(data, schema);
    writer.write(data, encoder);
    encoder.flush();
}

Source File: TestAvroTranslator.java From envelope with Apache License 2.0

4 votes

private void performAvroTranslation(Config config) throws Exception {
  Schema schema = getSchema();

  Record record = new Record(schema);
  record.put("field1", "hello");
  record.put("field2", true);
  record.put("field3", ByteBuffer.wrap("world".getBytes()));
  record.put("field4", 1.0d);
  record.put("field5", 1);
  record.put("field6", 1.0f);
  record.put("field7", 1L);
  for (int i = 8; i <= 14; i++) {
    record.put("field" + i, null);
  }

  ByteArrayOutputStream out = new ByteArrayOutputStream();
  BinaryEncoder encoder = EncoderFactory.get().binaryEncoder(out, null);
  DatumWriter<Record> writer = new GenericDatumWriter<>(schema);
  writer.write(record, encoder);
  encoder.flush();
  byte[] a = out.toByteArray();
  out.close();

  AvroTranslator t = new AvroTranslator();
  assertNoValidationFailures(t, config);
  t.configure(config);

  Row translated = t.translate(TestingMessageFactory.get(a, DataTypes.BinaryType)).iterator().next();

  Row validated = Contexts.getSparkSession().createDataFrame(
      Lists.newArrayList(translated), translated.schema()).collectAsList().get(0);

  assertEquals("hello", validated.getAs("field1"));
  assertEquals(true, validated.getAs("field2"));
  assertEquals("world", new String(validated.<byte[]>getAs("field3")));
  assertEquals(1.0d, validated.getAs("field4"));
  assertEquals(1, validated.getAs("field5"));
  assertEquals(1.0f, validated.getAs("field6"));
  assertEquals(1L, validated.getAs("field7"));
  for (int i = 8; i < 14; i++) {
    assertNull(validated.getAs("field" + i));
  }
}

Java Code Examples for org.apache.avro.io.DatumWriter#write()