Java Code Examples for org.apache.avro.specific.SpecificDatumReader#read()

private void roundtrip(Object thingie) throws Exception {
  Schema schema = SpecificData.get().getSchema(thingie.getClass());
  ByteArrayOutputStream os = new ByteArrayOutputStream();
  BinaryEncoder binaryEncoder = AvroCompatibilityHelper.newBinaryEncoder(os, false, null);
  SpecificDatumWriter<Object> writer = new SpecificDatumWriter<>(schema);
  writer.write(thingie, binaryEncoder);
  binaryEncoder.flush();
  byte[] serialized = os.toByteArray();
  ByteArrayInputStream is = new ByteArrayInputStream(serialized);
  BinaryDecoder binaryDecoder = AvroCompatibilityHelper.newBinaryDecoder(is, false, null);
  SpecificDatumReader<Object> reader = new SpecificDatumReader<>(schema);
  Object deserialize = reader.read(null, binaryDecoder);
  Assert.assertEquals(deserialize, thingie);
}
 

/**
 * Convert bytes to AvroHttpRequest.
 */
public AvroHttpRequest fromBytes(final byte[] theBytes) {
  try {
    final BinaryDecoder decoder = DecoderFactory.get().binaryDecoder(theBytes, null);
    final SpecificDatumReader<AvroHttpRequest> reader = new SpecificDatumReader<>(AvroHttpRequest.class);
    return reader.read(null, decoder);
  } catch (final IOException e) {
    throw new RuntimeException(e);
  }
}
 

static AvroYarnJobSubmissionParameters readYarnJobSubmissionParametersFromInputStream(
    final InputStream inputStream) throws IOException {
  final JsonDecoder decoder = DecoderFactory.get().jsonDecoder(
      AvroYarnJobSubmissionParameters.getClassSchema(), inputStream);
  final SpecificDatumReader<AvroYarnJobSubmissionParameters> reader = new SpecificDatumReader<>(
      AvroYarnJobSubmissionParameters.class);
  return reader.read(null, decoder);
}
 

static AvroYarnAppSubmissionParameters readYarnAppSubmissionParametersFromInputStream(
    final InputStream inputStream) throws IOException {
  final JsonDecoder decoder = DecoderFactory.get().jsonDecoder(
      AvroYarnAppSubmissionParameters.getClassSchema(), inputStream);
  final SpecificDatumReader<AvroYarnAppSubmissionParameters> reader = new SpecificDatumReader<>(
      AvroYarnAppSubmissionParameters.class);
  return reader.read(null, decoder);
}
 

private static AvroAzureBatchJobSubmissionParameters readAvroJobSubmissionParameters(
    final File paramsFile) throws IOException {
  final AvroAzureBatchJobSubmissionParameters avroAzureBatchJobSubmissionParameters;
  try (FileInputStream fileInputStream = new FileInputStream(paramsFile)) {
    final JsonDecoder decoder = DecoderFactory.get().jsonDecoder(
        AvroAzureBatchJobSubmissionParameters.getClassSchema(), fileInputStream);
    final SpecificDatumReader<AvroAzureBatchJobSubmissionParameters> reader =
        new SpecificDatumReader<>(AvroAzureBatchJobSubmissionParameters.class);
    avroAzureBatchJobSubmissionParameters = reader.read(null, decoder);
  }
  return avroAzureBatchJobSubmissionParameters;
}
 

/**
 * Reads avro object from input stream.
 *
 * @param inputStream The input stream to read from
 * @return Avro object
 * @throws IOException
 */
AvroYarnJobSubmissionParameters fromInputStream(final InputStream inputStream) throws IOException {
  final JsonDecoder decoder = DecoderFactory.get().jsonDecoder(
          AvroYarnJobSubmissionParameters.getClassSchema(), inputStream);
  final SpecificDatumReader<AvroYarnJobSubmissionParameters> reader = new SpecificDatumReader<>(
          AvroYarnJobSubmissionParameters.class);
  return reader.read(null, decoder);
}
 

static <T> T fromBytes(final byte[] theBytes, final Class<T> theClass) {
  final BinaryDecoder decoder = DecoderFactory.get().binaryDecoder(theBytes, null);
  final SpecificDatumReader<T> reader = new SpecificDatumReader<>(theClass);
  try {
    return reader.read(null, decoder);
  } catch (final IOException e) {
    throw new RuntimeException("Failed to deserialize an avro object", e);
  }
}
 

@Override
public T deserialize(byte[] message) {
    ensureInitialized();
    try {
        //decoder = DecoderFactory.get().binaryDecoder(message, decoder);
        File file = new File(path);
        Schema schema = new Schema.Parser().parse(file);
        SpecificDatumReader<T> reader = new SpecificDatumReader<>(schema);
        decoder = DecoderFactory.get().binaryDecoder(message,null);
        return reader.read(null, decoder);
    } catch (Exception ex) {
        throw new RuntimeException(ex);
    }
}
 

public static synchronized <T> T deserialize(final byte[] avroData, final Schema avroSchema) {
  try {
    final BinaryDecoder decoder = df.binaryDecoder(avroData, null);

    final String schemaName = getSchemaName(avroSchema);
    if (!readers.containsKey(schemaName)) {
      readers.put(schemaName, new SpecificDatumReader<T>(avroSchema));
    }
    final SpecificDatumReader<T> reader = readers.get(schemaName);
    return reader.read(null, decoder);
  } catch (final IOException e) {
    LOGGER.error("Unable to deserialize byte[] to Avro object: " + e.getMessage(), e);
    return null;
  }
}
 

private byte[] generateFailedTaskSerializedAvro() throws IOException {
  AvroFailedTask avroFailedTask = null;

  if (jfailedTask.getData() != null && jfailedTask.getData().isPresent()) {
    // Deserialize what was passed in from C#.
    try (ByteArrayInputStream fileInputStream = new ByteArrayInputStream(jfailedTask.getData().get())) {
      final JsonDecoder decoder = DecoderFactory.get().jsonDecoder(
          AvroFailedTask.getClassSchema(), fileInputStream);
      final SpecificDatumReader<AvroFailedTask> reader =
          new SpecificDatumReader<>(AvroFailedTask.class);
      avroFailedTask = reader.read(null, decoder);
    }
  } else {
    // This may result from a failed Evaluator.
    avroFailedTask = AvroFailedTask.newBuilder()
        .setIdentifier(jfailedTask.getId())
        .setCause(ByteBuffer.wrap(new byte[0]))
        .setData(ByteBuffer.wrap(new byte[0]))
        .setMessage("")
        .build();
  }

  // Overwrite the message if Java provides a message and C# does not.
  // Typically the case for failed Evaluators.
  if (StringUtils.isNoneBlank(jfailedTask.getMessage()) &&
      StringUtils.isBlank(avroFailedTask.getMessage().toString())) {
    avroFailedTask.setMessage(jfailedTask.getMessage());
  }

  final DatumWriter<AvroFailedTask> datumWriter = new SpecificDatumWriter<>(AvroFailedTask.class);

  try (ByteArrayOutputStream outputStream = new ByteArrayOutputStream()) {
    final JsonEncoder encoder = EncoderFactory.get().jsonEncoder(avroFailedTask.getSchema(), outputStream);
    datumWriter.write(avroFailedTask, encoder);
    encoder.flush();
    outputStream.flush();
    return outputStream.toByteArray();
  }
}
 

public static void main(String[] args) throws Exception {
  CommandLine commandLine = parseCommandLine(args);
  String zkUrl = commandLine.getOptionValue(ZOOKEEPER);
  String statsTopic = commandLine.getOptionValue(STATS_TOPIC);

  String bootstrapBrokers = OperatorUtil.getBrokers(zkUrl, SecurityProtocol.PLAINTEXT);
  Properties props = new Properties();
  props.put(KafkaUtils.BOOTSTRAP_SERVERS, bootstrapBrokers);
  props.put("group.id", "broker_statsreader_group");
  props.put("enable.auto.commit", "false");
  props.put("auto.commit.interval.ms", "1000");
  props.put("key.deserializer", "org.apache.kafka.common.serialization.ByteArrayDeserializer");
  props.put("value.deserializer", "org.apache.kafka.common.serialization.ByteArrayDeserializer");

  Schema schema = BrokerStats.getClassSchema();
  KafkaConsumer<byte[], byte[]> consumer = new KafkaConsumer<>(props);
  consumer.subscribe(Arrays.asList(statsTopic));
  while (true) {
    ConsumerRecords<byte[], byte[]> records = consumer.poll(100);
    for (ConsumerRecord<byte[], byte[]> record : records) {
      System.out.printf("offset = %d, key.size = %d, value.size = %s%n",
          record.offset(), record.key().length, record.value().length);
      try {
        BinaryDecoder binaryDecoder = avroDecoderFactory.binaryDecoder(record.value(), null);
        SpecificDatumReader<BrokerStats> reader = new SpecificDatumReader<>(schema);
        BrokerStats result = new BrokerStats();
        reader.read(result, binaryDecoder);
        System.out.println(result);
      } catch (Exception e) {
        LOG.error("Fail to decode an message", e);
      }
    }
  }
}
 

private static <T> T deserialize(
    final T avroObject,
    final byte[] avroData,
    final Class<T> avroClass,
    final Schema avroSchema) throws IOException {
  final BinaryDecoder decoder = df.binaryDecoder(avroData, null);
  if (!readers.containsKey(avroClass.toString())) {
    readers.put(avroClass.toString(), new SpecificDatumReader(avroSchema));
  }
  final SpecificDatumReader<T> reader = readers.get(avroClass.toString());
  return reader.read(avroObject, decoder);
}
 

public static <T extends SpecificRecord> T toSpecificRecord(GenericData.Record record) throws IOException {
    GenericDatumWriter<GenericData.Record> datumWriter = new GenericDatumWriter<>(record.getSchema());
    ByteArrayOutputStream baos = new ByteArrayOutputStream();

    Encoder binaryEncoder = EncoderFactory.get().binaryEncoder(baos, null);
    datumWriter.write(record, binaryEncoder);
    binaryEncoder.flush();

    SpecificDatumReader<T> datumReader = new SpecificDatumReader<>(record.getSchema());
    return datumReader.read(null, DecoderFactory.get().binaryDecoder(baos.toByteArray(), null));
}
 

/**
 * Deserializes avro definition.
 * @param serializedRuntimeDefinition serialized definition
 * @return Avro object
 * @throws IOException
 */
public AvroMultiRuntimeDefinition fromString(final String serializedRuntimeDefinition) throws
        IOException{
  final JsonDecoder decoder = DecoderFactory.get().
          jsonDecoder(AvroMultiRuntimeDefinition.getClassSchema(), serializedRuntimeDefinition);
  final SpecificDatumReader<AvroMultiRuntimeDefinition> reader = new SpecificDatumReader<>(AvroMultiRuntimeDefinition
          .class);
  final AvroMultiRuntimeDefinition rd = reader.read(null, decoder);
  return rd;
}
 

public static <T> T specificDataFromDecoder(Schema writerSchema, Decoder decoder) {
  SpecificDatumReader<T> datumReader = new SpecificDatumReader<>(writerSchema);
  try {
    return datumReader.read(null, decoder);
  } catch (Exception e) {
    throw new RuntimeException(e);
  }
}
 

private void roundtrip(Object thingie) throws Exception {
  Schema schema = SpecificData.get().getSchema(thingie.getClass());
  ByteArrayOutputStream os = new ByteArrayOutputStream();
  BinaryEncoder binaryEncoder = AvroCompatibilityHelper.newBinaryEncoder(os, false, null);
  SpecificDatumWriter<Object> writer = new SpecificDatumWriter<>(schema);
  writer.write(thingie, binaryEncoder);
  binaryEncoder.flush();
  byte[] serialized = os.toByteArray();
  ByteArrayInputStream is = new ByteArrayInputStream(serialized);
  BinaryDecoder binaryDecoder = AvroCompatibilityHelper.newBinaryDecoder(is, false, null);
  SpecificDatumReader<Object> reader = new SpecificDatumReader<>(schema);
  Object deserialize = reader.read(null, binaryDecoder);
  Assert.assertEquals(deserialize, thingie);
}
 

public static <T extends SpecificRecord> T deserialize(byte[] data, Schema schema) throws IOException {
    SpecificDatumReader<T> reader = new SpecificDatumReader<>(schema);
    Decoder decoder = DecoderFactory.get().binaryDecoder(data, null);
    return reader.read(null, decoder);
}
 

public static <T extends SpecificRecord> T deserialize(byte[] data, Schema schema) throws IOException {
    SpecificDatumReader<T> reader = new SpecificDatumReader<>(schema);
    Decoder decoder = DecoderFactory.get().binaryDecoder(data, null);
    return reader.read(null, decoder);
}
 

private static AvroConfiguration avroFromBytes(final byte[] theBytes) throws IOException {
  final BinaryDecoder decoder = DecoderFactory.get().binaryDecoder(theBytes, null);
  final SpecificDatumReader<AvroConfiguration> reader = new SpecificDatumReader<>(AvroConfiguration.class);
  return reader.read(null, decoder);
}
 

/**
 * Decode and deserialize the byte array into an instance of an Avro record
 * @param clazz class of the instance to decode. It must be a valid Avro Record.
 * @param bytes bytes to decode
 * @param reuse existing instance of T that may be used to populate with the decoded result. There is no guarantee it
 *              will actually be used.
 * @param <T> type of the instance to decode
 * @return decoded instance of T
 */
public static <T extends SpecificRecord> T decodeAvroSpecificRecord(Class<T> clazz, byte[] bytes, T reuse)
    throws IOException {
  SpecificDatumReader<T> reader = new SpecificDatumReader<>(clazz);
  Decoder decoder = DecoderFactory.defaultFactory().createBinaryDecoder(bytes, null);
  return reader.read(reuse, decoder);
}