org.apache.avro.io.BinaryEncoder Java Examples

The following examples show how to use org.apache.avro.io.BinaryEncoder. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: AzureBlobAvroWriter.java    From samza with Apache License 2.0 7 votes vote down vote up
@VisibleForTesting
byte[] encodeRecord(IndexedRecord record) {
  ByteArrayOutputStream out = new ByteArrayOutputStream();
  Schema schema = record.getSchema();
  try {
    EncoderFactory encoderfactory = new EncoderFactory();
    BinaryEncoder encoder = encoderfactory.binaryEncoder(out, null);
    DatumWriter<IndexedRecord> writer;
    if (record instanceof SpecificRecord) {
      writer = new SpecificDatumWriter<>(schema);
    } else {
      writer = new GenericDatumWriter<>(schema);
    }
    writer.write(record, encoder);
    encoder.flush(); //encoder may buffer
  } catch (Exception e) {
    throw new SamzaException("Unable to serialize Avro record using schema within the record: " + schema.toString(), e);
  }
  return out.toByteArray();
}
 
Example #2
Source File: AvroGenericRecordAccessorTest.java    From incubator-gobblin with Apache License 2.0 6 votes vote down vote up
@AfterMethod
public void serializeRecord(ITestResult result)
    throws IOException {
  if (result.isSuccess() && result.getThrowable() == null) {
  /* Serialize the GenericRecord; this can catch issues in set() that the underlying GenericRecord
   * may not catch until serialize time
   */
    DatumWriter<GenericRecord> datumWriter = new GenericDatumWriter<>(recordSchema);
    ByteArrayOutputStream bOs = new ByteArrayOutputStream();

    BinaryEncoder encoder = EncoderFactory.get().binaryEncoder(bOs, null);
    datumWriter.write(record, encoder);
    encoder.flush();
    bOs.flush();

    Assert.assertTrue(bOs.toByteArray().length > 0);
  }
}
 
Example #3
Source File: TransformTest.java    From schema-registry-transfer-smt with Apache License 2.0 6 votes vote down vote up
private ByteArrayOutputStream encodeAvroObject(org.apache.avro.Schema schema, int sourceId, Object datum) throws IOException {
    ByteArrayOutputStream out = new ByteArrayOutputStream();

    out.write(MAGIC_BYTE);
    out.write(ByteBuffer.allocate(ID_SIZE).putInt(sourceId).array());

    EncoderFactory encoderFactory = EncoderFactory.get();
    BinaryEncoder encoder = encoderFactory.directBinaryEncoder(out, null);
    Object
            value =
            datum instanceof NonRecordContainer ? ((NonRecordContainer) datum).getValue()
                    : datum;
    DatumWriter<Object> writer = new GenericDatumWriter<>(schema);
    writer.write(value, encoder);
    encoder.flush();

    return out;
}
 
Example #4
Source File: KafkaValueSerializer.java    From kareldb with Apache License 2.0 6 votes vote down vote up
@Override
public byte[] serialize(String topic, NavigableMap<Long, VersionedValue> object) {
    if (object == null) {
        return null;
    }
    try {
        ByteArrayOutputStream out = new ByteArrayOutputStream();
        out.write(MAGIC_BYTE);
        out.write(ByteBuffer.allocate(VERSION_SIZE).putInt(version).array());
        BinaryEncoder encoder = encoderFactory.directBinaryEncoder(out, null);
        writer.write(toArray(object), encoder);
        encoder.flush();
        byte[] bytes = out.toByteArray();
        out.close();
        return bytes;
    } catch (IOException | RuntimeException e) {
        // avro serialization can throw AvroRuntimeException, NullPointerException,
        // ClassCastException, etc
        LOG.error("Error serializing Avro value " + e.getMessage());
        throw new SerializationException("Error serializing Avro value", e);
    }
}
 
Example #5
Source File: AvroCompatibilityHelperBinaryCodecsTest.java    From avro-util with BSD 2-Clause "Simplified" License 6 votes vote down vote up
@Test
public void testBinaryCodecs() throws Exception {
  AtomicReference<BinaryEncoder> bufferedEncoderRef = new AtomicReference<>(null);
  AtomicReference<BinaryEncoder> directEncoderRef = new AtomicReference<>(null);
  AtomicReference<BinaryDecoder> bufferedDecoderRef = new AtomicReference<>(null);
  AtomicReference<BinaryDecoder> directDecoderRef = new AtomicReference<>(null);

  for (boolean reuseEncoder : Arrays.asList(false, true)) { //false 1st
    for (boolean reuseDecoder : Arrays.asList(false, true)) { //false 1st
      for (boolean useBufferedEncoder : Arrays.asList(true, false)) {
        for (boolean useBufferedDecoder : Arrays.asList(true, false)) {

          runBinaryEncodeDecodeCycle(
              bufferedEncoderRef, directEncoderRef, bufferedDecoderRef, directDecoderRef,
              reuseEncoder, reuseDecoder, useBufferedEncoder, useBufferedDecoder
          );

        }
      }
    }
  }
}
 
Example #6
Source File: JavaToAvroSerializer.java    From Decision with Apache License 2.0 6 votes vote down vote up
private byte[] getInsertMessageBytes(InsertMessage insertMessage){

        byte[] result = null;

        ByteArrayOutputStream out = new ByteArrayOutputStream();
        BinaryEncoder encoder = EncoderFactory.get().binaryEncoder(out, null);
        SpecificDatumWriter writer = new SpecificDatumWriter<InsertMessage>(InsertMessage.getClassSchema());

        try {
            writer.write(insertMessage, encoder);
            encoder.flush();
            out.close();
            result = out.toByteArray();
        }catch (IOException e){
            return null;
        }

        return result;

    }
 
Example #7
Source File: FastStringableTest.java    From avro-fastserde with Apache License 2.0 6 votes vote down vote up
private <T> Decoder serializeSpecificFast(T data, Schema schema) {
    ByteArrayOutputStream baos = new ByteArrayOutputStream();
    BinaryEncoder binaryEncoder = EncoderFactory.get().directBinaryEncoder(baos, null);

    try {
        FastSpecificSerializerGenerator<T> fastSpecificSerializerGenerator = new FastSpecificSerializerGenerator<>(
                schema, tempDir, classLoader, null);
        FastSerializer<T> fastSerializer = fastSpecificSerializerGenerator.generateSerializer();
        fastSerializer.serialize(data, binaryEncoder);
        binaryEncoder.flush();

    } catch (Exception e) {
        throw new RuntimeException(e);
    }

    return DecoderFactory.get().binaryDecoder(baos.toByteArray(), null);
}
 
Example #8
Source File: KafkaAvroPublisher.java    From doctorkafka with Apache License 2.0 6 votes vote down vote up
public void publish(BrokerStats brokerStats) throws IOException {
  try {
    ByteArrayOutputStream stream = new ByteArrayOutputStream();
    BinaryEncoder binaryEncoder = avroEncoderFactory.binaryEncoder(stream, null);

    avroEventWriter.write(brokerStats, binaryEncoder);
    binaryEncoder.flush();
    IOUtils.closeQuietly(stream);

    String key = brokerStats.getName() + "_" + System.currentTimeMillis();
    int numPartitions = kafkaProducer.partitionsFor(destTopic).size();
    int partition = brokerStats.getId() % numPartitions;

    Future<RecordMetadata> future = kafkaProducer.send(
        new ProducerRecord<>(destTopic, partition, key.getBytes(), stream.toByteArray()));
    future.get();

    OpenTsdbMetricConverter.incr("kafka.stats.collector.success", 1, "host=" + HOSTNAME);
  } catch (Exception e) {
    LOG.error("Failure in publish stats", e);
    OpenTsdbMetricConverter.incr("kafka.stats.collector.failure", 1, "host=" + HOSTNAME);
    throw new RuntimeException("Avro serialization failure", e);
  }
}
 
Example #9
Source File: DoctorKafkaActionReporter.java    From doctorkafka with Apache License 2.0 6 votes vote down vote up
public synchronized void sendMessage(String clusterName, String message) {
  int numRetries = 0;
  while (numRetries < MAX_RETRIES) {
    try {
      long timestamp = System.currentTimeMillis();
      OperatorAction operatorAction = new OperatorAction(timestamp, clusterName, message);

      ByteArrayOutputStream stream = new ByteArrayOutputStream();
      BinaryEncoder binaryEncoder = avroEncoderFactory.binaryEncoder(stream, null);
      avroWriter.write(operatorAction, binaryEncoder);
      binaryEncoder.flush();
      IOUtils.closeQuietly(stream);

      String key = Long.toString(System.currentTimeMillis());
      ProducerRecord<byte[], byte[]>  producerRecord = 
          new ProducerRecord<>(topic, key.getBytes(), stream.toByteArray());
      Future<RecordMetadata> future = kafkaProducer.send(producerRecord);
      future.get();
      LOG.info("Send an message {} to action report : ", message);
      break;
    } catch (Exception e) {
      LOG.error("Failed to publish report message {}: {}", clusterName, message, e);
      numRetries++;
    }
  }
}
 
Example #10
Source File: DataSerializer.java    From jMetalSP with MIT License 6 votes vote down vote up
public byte[] serializeMessage(S clazz, String path) {

        byte[] result = null;
        try {

            File file = new File(path);

            Schema schema = new Schema.Parser().parse(file);

            ByteArrayOutputStream out = new ByteArrayOutputStream();
            BinaryEncoder encoder = EncoderFactory.get().binaryEncoder(out, null);
            DatumWriter<S> dataFileWriter = new SpecificDatumWriter<S>(schema);
            dataFileWriter.write(clazz, encoder);
            encoder.flush();
            result=out.toByteArray();
            out.close();
        }catch (Exception ex){
            ex.printStackTrace();
        }
        return result;
    }
 
Example #11
Source File: AvroLWM2MDataPublish.java    From SDA with BSD 2-Clause "Simplified" License 6 votes vote down vote up
/**
 * 데이타 전송
 * @param event
 * @throws Exception
 * @return void
 */
public void send(COL_LWM2M event) throws Exception {
	EncoderFactory avroEncoderFactory = EncoderFactory.get();
	SpecificDatumWriter<COL_LWM2M> avroEventWriter = new SpecificDatumWriter<COL_LWM2M>(COL_LWM2M.SCHEMA$);
	
	ByteArrayOutputStream stream = new ByteArrayOutputStream();
	BinaryEncoder binaryEncoder = avroEncoderFactory.binaryEncoder(stream,null);

	try {
		avroEventWriter.write(event, binaryEncoder);
		binaryEncoder.flush();
	} catch (IOException e) {
		e.printStackTrace();
		throw e;
	}
	IOUtils.closeQuietly(stream);

	KeyedMessage<String, byte[]> data = new KeyedMessage<String, byte[]>(
			TOPIC, stream.toByteArray());

	producer.send(data);
}
 
Example #12
Source File: AvroSerializer.java    From envelope with Apache License 2.0 6 votes vote down vote up
@Override
public byte[] serialize(String topic, Row data) {
  if (data == null) {
    return null;
  }
  
  GenericRecord record = recordForRow(data, schema);
  
  ByteArrayOutputStream out = new ByteArrayOutputStream();
  BinaryEncoder encoder = EncoderFactory.get().binaryEncoder(out, null);
  try {
    datumWriter.write(record, encoder);
    encoder.flush();
    out.close();
  } catch (IOException e) {
    throw new RuntimeException("Avro serializer for Kafka output could not serialize row", e);
  }
  
  return out.toByteArray();
}
 
Example #13
Source File: WriteAvroResultWithExternalSchema.java    From nifi with Apache License 2.0 6 votes vote down vote up
public WriteAvroResultWithExternalSchema(final Schema avroSchema, final RecordSchema recordSchema, final SchemaAccessWriter schemaAccessWriter,
                                         final OutputStream out, final BlockingQueue<BinaryEncoder> recycleQueue, final ComponentLog logger) {
    super(out);
    this.recordSchema = recordSchema;
    this.schemaAccessWriter = schemaAccessWriter;
    this.avroSchema = avroSchema;
    this.buffered = new BufferedOutputStream(out);
    this.recycleQueue = recycleQueue;

    BinaryEncoder reusableEncoder = recycleQueue.poll();
    if (reusableEncoder == null) {
        logger.debug("Was not able to obtain a BinaryEncoder from reuse pool. This is normal for the first X number of iterations (where X is equal to the max size of the pool), " +
            "but if this continues, it indicates that increasing the size of the pool will likely yield better performance for this Avro Writer.");
    }

    encoder = EncoderFactory.get().blockingBinaryEncoder(buffered, reusableEncoder);

    datumWriter = new GenericDatumWriter<>(avroSchema);
}
 
Example #14
Source File: FastGenericSerializerGeneratorTest.java    From avro-fastserde with Apache License 2.0 6 votes vote down vote up
private <T> Decoder serializeGenericFast(T data, Schema schema) {
    ByteArrayOutputStream baos = new ByteArrayOutputStream();
    BinaryEncoder binaryEncoder = EncoderFactory.get().directBinaryEncoder(baos, null);

    try {
        FastGenericSerializerGenerator<T> fastGenericSerializerGenerator = new FastGenericSerializerGenerator<>(
                schema, tempDir, classLoader, null);
        FastSerializer<T> fastSerializer = fastGenericSerializerGenerator.generateSerializer();
        fastSerializer.serialize(data, binaryEncoder);
        binaryEncoder.flush();

    } catch (Exception e) {
        throw new RuntimeException(e);
    }

    return DecoderFactory.get().binaryDecoder(baos.toByteArray(), null);
}
 
Example #15
Source File: AvroToBytesConverter.java    From incubator-gobblin with Apache License 2.0 6 votes vote down vote up
@Override
public Iterable<byte[]> convertRecord(String outputSchema, GenericRecord inputRecord, WorkUnitState workUnit)
    throws DataConversionException {
  try {
    ByteArrayOutputStream bytesOut = new ByteArrayOutputStream();

    BinaryEncoder encoder = EncoderFactory.get().binaryEncoder(bytesOut, encoderCache.get());
    encoderCache.set(encoder);
    writer.write(inputRecord, encoder);
    encoder.flush();

    return Collections.singleton(bytesOut.toByteArray());
  } catch (IOException e) {
    throw new DataConversionException("Error serializing record", e);
  }
}
 
Example #16
Source File: LiAvroSerializerBase.java    From incubator-gobblin with Apache License 2.0 6 votes vote down vote up
public byte[] serialize(String topic, GenericRecord data)
    throws SerializationException {
  Schema schema = data.getSchema();
  MD5Digest schemaId = null;
  try {
    schemaId = schemaRegistry.register(topic, schema);
    ByteArrayOutputStream out = new ByteArrayOutputStream();
    // MAGIC_BYTE | schemaId-bytes | avro_payload
    out.write(LiAvroSerDeHelper.MAGIC_BYTE);
    out.write(schemaId.asBytes());
    BinaryEncoder encoder = encoderFactory.directBinaryEncoder(out, null);
    DatumWriter<GenericRecord> writer = new GenericDatumWriter<>(schema);
    writer.write(data, encoder);
    encoder.flush();
    byte[] bytes = out.toByteArray();
    out.close();
    return bytes;
  } catch (IOException | SchemaRegistryException e) {
    throw new SerializationException(e);
  }
}
 
Example #17
Source File: TypeUtils.java    From geowave with Apache License 2.0 6 votes vote down vote up
private static <T> byte[] deserialize(
    final T avroObject,
    final Schema avroSchema,
    final Class<T> avroClass) throws IOException {

  final ByteArrayOutputStream os = new ByteArrayOutputStream();
  final BinaryEncoder encoder = ef.binaryEncoder(os, null);
  if (!writers.containsKey(avroClass.toString())) {
    writers.put(avroClass.toString(), new SpecificDatumWriter<T>(avroSchema));
  }

  final SpecificDatumWriter<T> writer = writers.get(avroClass.toString());
  writer.write(avroObject, encoder);
  encoder.flush();
  return os.toByteArray();
}
 
Example #18
Source File: KafkaOutputPTransformRuntime.java    From components with Apache License 2.0 6 votes vote down vote up
@ProcessElement
public void processElement(ProcessContext c) {
    try {
        ByteArrayOutputStream out = new ByteArrayOutputStream();
        BinaryEncoder encoder = EncoderFactory.get().binaryEncoder(out, null);
        if (helper.isUseCustomAvroSchema()) {
            helper.getKafkaIndexedRecordWrapper().setIndexedRecord(c.element());
            helper.getDatumWriter().write(helper.getKafkaIndexedRecordWrapper(), encoder);
        } else {
            if (helper.getDatumWriter() == null) {
                // set the datumWriter for the first time with the incoming record schema
                helper.setDatumWriter(new GenericDatumWriter(c.element().getSchema()));
            }
            helper.getDatumWriter().write(c.element(), encoder);
        }
        encoder.flush();
        byte[] result = out.toByteArray();
        out.close();
        c.output(result);
    } catch (IOException e) {
        throw TalendRuntimeException.createUnexpectedException(e);
    }
}
 
Example #19
Source File: KafkaOutputPTransformRuntime.java    From components with Apache License 2.0 6 votes vote down vote up
@ProcessElement
public void processElement(ProcessContext c) {
    try {
        ByteArrayOutputStream out = new ByteArrayOutputStream();
        BinaryEncoder encoder = EncoderFactory.get().binaryEncoder(out, null);
        if (helper.isUseCustomAvroSchema()) {
            helper.getKafkaIndexedRecordWrapper().setIndexedRecord(c.element().getValue());
            helper.getDatumWriter().write(helper.getKafkaIndexedRecordWrapper(), encoder);
        } else {
            if (helper.getDatumWriter() == null) {
                // set the datumWriter for the first time with the incoming record schema
                helper.setDatumWriter(new GenericDatumWriter(c.element().getValue().getSchema()));
            }
            helper.getDatumWriter().write(c.element().getValue(), encoder);
        }
        encoder.flush();
        byte[] result = out.toByteArray();
        out.close();
        c.output(KV.of(c.element().getKey(), result));
    } catch (IOException e) {
        throw TalendRuntimeException.createUnexpectedException(e);
    }
}
 
Example #20
Source File: AvroOneM2MDataPublish.java    From SDA with BSD 2-Clause "Simplified" License 6 votes vote down vote up
/**
 * 데이타 전송
 * @param event
 * @throws Exception
 * @return void
 */
public void send(COL_ONEM2M event) throws Exception {
	EncoderFactory avroEncoderFactory = EncoderFactory.get();
	SpecificDatumWriter<COL_ONEM2M> avroEventWriter = new SpecificDatumWriter<COL_ONEM2M>(COL_ONEM2M.SCHEMA$);
	
	ByteArrayOutputStream stream = new ByteArrayOutputStream();
	BinaryEncoder binaryEncoder = avroEncoderFactory.binaryEncoder(stream,null);

	try {
		avroEventWriter.write(event, binaryEncoder);
		binaryEncoder.flush();
	} catch (IOException e) {
		e.printStackTrace();
		throw e;
	}
	IOUtils.closeQuietly(stream);

	KeyedMessage<String, byte[]> data = new KeyedMessage<String, byte[]>(
			TOPIC, stream.toByteArray());

	producer.send(data);
}
 
Example #21
Source File: AvroMessageEncoderUtil.java    From brooklin with BSD 2-Clause "Simplified" License 6 votes vote down vote up
/**
 * generates the md5 hash of the schemaId and appends it to the given byte array.
 * the byte array representing the payload of a BrooklinEnvelope
 *
 * This is done so when the client decodes the payload, it will contain a schemaId which
 * can be used to retrieve the schema from the Schema Registry
 *
 * This method also converts an IndexedRecord into a byte array first
 */
public static byte[] encode(String schemaId, IndexedRecord record) throws AvroEncodingException {
  Validate.notNull(record, "cannot encode null Record, schemaId: " + schemaId);
  ByteArrayOutputStream out = new ByteArrayOutputStream();
  out.write(MAGIC_BYTE);
  byte[] md5Bytes = hexToMd5(schemaId);

  try {
    out.write(md5Bytes);
    BinaryEncoder encoder = EncoderFactory.get().binaryEncoder(out, null);
    DatumWriter<org.apache.avro.generic.IndexedRecord> writer;
    if (record instanceof SpecificRecord) {
      writer = new SpecificDatumWriter<>(record.getSchema());
    } else {
      writer = new GenericDatumWriter<>(record.getSchema());
    }
    writer.write(record, encoder);
    encoder.flush(); //encoder may buffer
  } catch (IOException e) {
    throw new AvroEncodingException(e);
  }

  return out.toByteArray();
}
 
Example #22
Source File: HardwareEmulatorMain.java    From Decision with Apache License 2.0 6 votes vote down vote up
private byte[] getInsertMessageBytes(InsertMessage insertMessage){

            byte[] result = null;

            ByteArrayOutputStream out = new ByteArrayOutputStream();
            BinaryEncoder encoder = EncoderFactory.get().binaryEncoder(out, null);
            SpecificDatumWriter writer = new SpecificDatumWriter<InsertMessage>(InsertMessage.getClassSchema());

            try {
                writer.write(insertMessage, encoder);
                encoder.flush();
                out.close();
                result = out.toByteArray();
            }catch (IOException e){
                return null;
            }

            return result;

        }
 
Example #23
Source File: GenericAvroSerializer.java    From geowave with Apache License 2.0 6 votes vote down vote up
public static synchronized <T> byte[] serialize(final T avroObject, final Schema avroSchema) {

    try {
      final ByteArrayOutputStream os = new ByteArrayOutputStream();
      final BinaryEncoder encoder = ef.binaryEncoder(os, null);

      final String schemaName = getSchemaName(avroSchema);
      if (!writers.containsKey(schemaName)) {
        writers.put(schemaName, new SpecificDatumWriter<T>(avroSchema));
      }

      final SpecificDatumWriter<T> writer = writers.get(schemaName);
      writer.write(avroObject, encoder);
      encoder.flush();
      return os.toByteArray();
    } catch (final IOException e) {
      LOGGER.error("Unable to serialize Avro record to byte[]: " + e.getMessage(), e);
      return null;
    }
  }
 
Example #24
Source File: TestAzureBlobAvroWriter.java    From samza with Apache License 2.0 5 votes vote down vote up
private byte[] encodeRecord(IndexedRecord record) throws Exception {
  ByteArrayOutputStream out = new ByteArrayOutputStream();
  Schema schema = record.getSchema();
  EncoderFactory encoderfactory = new EncoderFactory();
  BinaryEncoder encoder = encoderfactory.binaryEncoder(out, null);
  DatumWriter<IndexedRecord> writer;
  if (record instanceof SpecificRecord) {
    writer = new SpecificDatumWriter<>(schema);
  } else {
    writer = new GenericDatumWriter<>(schema);
  }
  writer.write(record, encoder);
  encoder.flush(); //encoder may buffer
  return out.toByteArray();
}
 
Example #25
Source File: AvroTypeUtil.java    From datacollector with Apache License 2.0 5 votes vote down vote up
public static byte[] getBinaryEncodedAvroRecord(GenericRecord datum) throws IOException {
  final ByteArrayOutputStream baos = new ByteArrayOutputStream(1024);
  final DatumWriter<GenericRecord> outputDatumWriter = new GenericDatumWriter<>(datum.getSchema());
  final BinaryEncoder encoder = EncoderFactory.get().binaryEncoder(baos, null);

  outputDatumWriter.write(datum, encoder);
  encoder.flush();
  baos.flush();
  baos.close();

  return baos.toByteArray();
}
 
Example #26
Source File: IcebergEncoder.java    From iceberg with Apache License 2.0 5 votes vote down vote up
@Override
public void encode(D datum, OutputStream stream) throws IOException {
  BinaryEncoder encoder = EncoderFactory.get().directBinaryEncoder(stream, ENCODER.get());
  ENCODER.set(encoder);
  writer.write(datum, encoder);
  encoder.flush();
}
 
Example #27
Source File: MessageSerializerImpl.java    From reef with Apache License 2.0 5 votes vote down vote up
/**
 * Deserialize messages of type TMessage from input outputStream.
 * @param outputStream A ByteArrayOutputStream where the message to
 *                     be serialized will be written.
 * @param message An Avro message class which implements the Avro SpcificRecord interface.
 * @param sequence The numerical position of the message in the outgoing message stream.
 * @throws IOException An error occurred writing the message to the outputStream.
 */
public void serialize(final ByteArrayOutputStream outputStream,
                      final SpecificRecord message, final long sequence) throws IOException {
  // Binary encoder for both the header and message.
  final BinaryEncoder encoder = EncoderFactory.get().binaryEncoder(outputStream, null);
  // Write the header and the message.
  headerWriter.write(new Header(sequence, msgMetaClassName), encoder);
  messageWriter.write((TMessage)message, encoder);
  encoder.flush();
}
 
Example #28
Source File: TestConvertAvroToJSON.java    From localization_nifi with Apache License 2.0 5 votes vote down vote up
@Test
public void testSingleSchemalessAvroMessage_noContainer() throws IOException {
    final TestRunner runner = TestRunners.newTestRunner(new ConvertAvroToJSON());
    runner.setProperty(ConvertAvroToJSON.CONTAINER_OPTIONS, ConvertAvroToJSON.CONTAINER_NONE);
    Schema schema = new Schema.Parser().parse(new File("src/test/resources/user.avsc"));
    String stringSchema = schema.toString();
    runner.setProperty(ConvertAvroToJSON.SCHEMA, stringSchema);

    final GenericRecord user1 = new GenericData.Record(schema);
    user1.put("name", "Alyssa");
    user1.put("favorite_number", 256);

    final ByteArrayOutputStream out1 = new ByteArrayOutputStream();
    final BinaryEncoder encoder = EncoderFactory.get().binaryEncoder(out1, null);
    final DatumWriter<GenericRecord> datumWriter = new GenericDatumWriter<>(schema);
    datumWriter.write(user1, encoder);

    encoder.flush();
    out1.flush();
    byte[] test = out1.toByteArray();
    runner.enqueue(test);

    runner.run();

    runner.assertAllFlowFilesTransferred(ConvertAvroToJSON.REL_SUCCESS, 1);
    final MockFlowFile out = runner.getFlowFilesForRelationship(ConvertAvroToJSON.REL_SUCCESS).get(0);
    out.assertContentEquals("{\"name\": \"Alyssa\", \"favorite_number\": 256, \"favorite_color\": null}");
}
 
Example #29
Source File: AvroUtils.java    From localization_nifi with Apache License 2.0 5 votes vote down vote up
/**
 * Writes provided {@link GenericRecord} into the provided
 * {@link OutputStream}.
 */
public static void write(GenericRecord record, OutputStream out) {
    BinaryEncoder encoder = EncoderFactory.get().binaryEncoder(out, null);
    DatumWriter<GenericRecord> writer = new GenericDatumWriter<>(record.getSchema());
    try {
        writer.write(record, encoder);
        encoder.flush();
    } catch (Exception e) {
        throw new IllegalStateException("Failed to write AVRO record", e);
    }
}
 
Example #30
Source File: TestConvertAvroToJSON.java    From nifi with Apache License 2.0 5 votes vote down vote up
@Test
public void testSingleSchemalessAvroMessage_wrapSingleMessage_noContainer() throws IOException {
    final TestRunner runner = TestRunners.newTestRunner(new ConvertAvroToJSON());
    runner.setProperty(ConvertAvroToJSON.CONTAINER_OPTIONS, ConvertAvroToJSON.CONTAINER_NONE);
    runner.setProperty(ConvertAvroToJSON.WRAP_SINGLE_RECORD, Boolean.toString(true));
    Schema schema = new Schema.Parser().parse(new File("src/test/resources/user.avsc"));
    String stringSchema = schema.toString();
    runner.setProperty(ConvertAvroToJSON.SCHEMA, stringSchema);

    final GenericRecord user1 = new GenericData.Record(schema);
    user1.put("name", "Alyssa");
    user1.put("favorite_number", 256);

    final ByteArrayOutputStream out1 = new ByteArrayOutputStream();
    final BinaryEncoder encoder = EncoderFactory.get().binaryEncoder(out1, null);
    final DatumWriter<GenericRecord> datumWriter = new GenericDatumWriter<>(schema);
    datumWriter.write(user1, encoder);

    encoder.flush();
    out1.flush();
    byte[] test = out1.toByteArray();
    runner.enqueue(test);

    runner.run();

    runner.assertAllFlowFilesTransferred(ConvertAvroToJSON.REL_SUCCESS, 1);
    final MockFlowFile out = runner.getFlowFilesForRelationship(ConvertAvroToJSON.REL_SUCCESS).get(0);
    out.assertContentEquals("{\"name\": \"Alyssa\", \"favorite_number\": 256, \"favorite_color\": null}");
}