Java Code Examples for org.apache.avro.io.DatumReader#read()

@ElFunction(
    prefix = "avro",
    name = "decode",
    description = "Returns an avro record using the specified schema and the byte array")
public static GenericRecord decode(
    @ElParam("schema") String schema, @ElParam("bytes") byte[] avroBytes
) throws IOException {

  Schema.Parser parser = new Schema.Parser();
  Schema avroSchema = parser.parse(schema);
  DatumReader<GenericRecord> datumReader = new GenericDatumReader<>(avroSchema);
  BinaryDecoder decoder = DecoderFactory.get().binaryDecoder(new ByteArrayInputStream(avroBytes), null);
  GenericData.Record avroRecord = new GenericData.Record(avroSchema);

  return datumReader.read(avroRecord, decoder);
}
 

/**
 * Change the schema of an Avro record.
 * @param record The Avro record whose schema is to be changed.
 * @param newSchema The target schema. It must be compatible as reader schema with record.getSchema() as writer schema.
 * @return a new Avro record with the new schema.
 * @throws IOException if conversion failed.
 */
public static GenericRecord convertRecordSchema(GenericRecord record, Schema newSchema) throws IOException {
  if (record.getSchema().equals(newSchema)) {
    return record;
  }

  try {
    BinaryDecoder decoder = new DecoderFactory().binaryDecoder(recordToByteArray(record), null);
    DatumReader<GenericRecord> reader = new GenericDatumReader<>(record.getSchema(), newSchema);
    return reader.read(null, decoder);
  } catch (IOException e) {
    throw new IOException(
        String.format("Cannot convert avro record to new schema. Original schema = %s, new schema = %s",
            record.getSchema(), newSchema),
        e);
  }
}
 

@Override
public Record deserialize(String topic, byte[] data) {
  ByteBuffer buffer = ByteBuffer.wrap(data);
  if (buffer.get() != MAGIC_BYTE) {
    throw new RuntimeException("Unknown magic byte!");
  }
  int version = buffer.getInt();
  Schema schema = schemaLookup.getSchema(version);

  int offset = buffer.position();
  int length = buffer.remaining();

  DatumReader<Record> reader = new GenericDatumReader<>(schema, schema, genericData);
  Decoder decoder = DecoderFactory.get().binaryDecoder(data, offset, length, null);
  try {
    return reader.read(null, decoder);
  } catch (IOException e) {
    throw new RuntimeException("Unable to decode record.", e);
  }
}
 

private void checkBareRecordsSplitSize(final List<MockFlowFile> flowFiles, final int expectedRecordsPerSplit, final boolean checkMetadata) throws IOException {
    for (final MockFlowFile flowFile : flowFiles) {
        try (final ByteArrayInputStream in = new ByteArrayInputStream(flowFile.toByteArray())) {
            final DatumReader<GenericRecord> reader = new GenericDatumReader<>(schema);
            final Decoder decoder = DecoderFactory.get().binaryDecoder(in, null);

            int count = 0;
            GenericRecord record = reader.read(null, decoder);
            try {
                while (record != null) {
                    Assert.assertNotNull(record.get("name"));
                    Assert.assertNotNull(record.get("favorite_number"));
                    count++;
                    record = reader.read(record, decoder);
                }
            } catch (EOFException eof) {
                // expected
            }
            assertEquals(expectedRecordsPerSplit, count);
        }

        if (checkMetadata) {
            Assert.assertTrue(flowFile.getAttributes().containsKey(META_KEY1));
            Assert.assertTrue(flowFile.getAttributes().containsKey(META_KEY2));
            Assert.assertTrue(flowFile.getAttributes().containsKey(META_KEY3));
        }
    }
}
 

@Override
public FixFile convert(TestRun tr, FixFile ff) throws Exception {
    Schema schema = new Schema.Parser().parse(schemaCreator.create(tr).getContent());
    ByteArrayOutputStream baos = new ByteArrayOutputStream();
    InputStream input = ff.getContent();
    DataFileWriter<Object> writer;;
    try {
        DatumReader<Object> reader = new GenericDatumReader<>(schema);
        DataInputStream din = new DataInputStream(input);
        writer = new DataFileWriter<>(new GenericDatumWriter<>());
        writer.create(schema, baos);
        Decoder decoder = DecoderFactory.get().jsonDecoder(schema, din);
        Object datum;
        while (true) {
            try {
                datum = reader.read(null, decoder);
            } catch (EOFException eofe) {
                break;
            }
            writer.append(datum);
        }
        writer.flush();
    } finally {
        input.close();
    }
    return new FixFile(new ByteArrayInputStream(baos.toByteArray()));
}
 

public static Person desFromAvroBytes(byte[] record) throws IOException {
    DatumReader<GenericRecord> datumReader = new GenericDatumReader<GenericRecord>(schema);
    BinaryDecoder decoder = null;
    decoder = DecoderFactory.get().binaryDecoder(record, decoder);
    GenericRecord avroValue = datumReader.read(null, decoder);
    return fromAvroRecord(avroValue);
}
 

public AvroHttpRequest deSerealizeAvroHttpRequestJSON(byte[] data) {
    DatumReader<AvroHttpRequest> reader = new SpecificDatumReader<>(AvroHttpRequest.class);
    Decoder decoder = null;
    try {
        decoder = DecoderFactory.get()
            .jsonDecoder(AvroHttpRequest.getClassSchema(), new String(data));
        return reader.read(null, decoder);
    } catch (IOException e) {
        logger.error("Deserialization error" + e.getMessage());
    }
    return null;
}
 

@Override
public <T> T decode(byte[] bytes, Class<T> type) throws IOException {
	Assert.notNull(bytes, "'bytes' cannot be null");
	Assert.notNull(bytes, "Class can not be null");
	ByteBuffer buf = ByteBuffer.wrap(bytes);
	byte[] payload = new byte[bytes.length-4];
	Integer schemaId = buf.getInt();
	buf.get(payload);
	Schema schema = schemaRegistryClient.fetch(schemaId);
	DatumReader reader = getDatumReader(type,schema);
	Decoder decoder = DecoderFactory.get().binaryDecoder(payload,null);
	return (T) reader.read(null,decoder);
}
 

/**
     * 解析被generic schema封装的实际数据
     *
     * @param schema  schema对象
     * @param payload 实际数据
     * @return List<GenericRecord>
     * @throws Exception
     */
    public List<GenericRecord> decode(Schema schema, byte[] payload) throws Exception {
        logger.trace("Schema:" + schema.toString() + " schema payload:" + new String(payload, "utf-8"));
//        List<GenericRecord> list = new LinkedList<>();
//        DatumReader<GenericRecord> reader = new GenericDatumReader<>(schema);
//        BinaryDecoder decoder = getBinaryDecoder(payload);
//        while (!decoder.isEnd()) {
//            list.add(reader.read(null, decoder));
//        }
//        return list;

        ByteBuffer buffer = ByteBuffer.wrap(payload);

        if (buffer.get() != Constants.MAGIC_BYTE) {
            logger.error("Unknown magic byte!");
        }

        int id = buffer.getInt();

        try {
            schema = schemaRegistry.getById(id);
        } catch (RestClientException e) {
            logger.error("Schema Registry RestClientException: " + e);
        }

        int length = buffer.limit() - 5;
        int start = buffer.position() + buffer.arrayOffset();

        logger.debug("Schema:" + schema.toString() + " schema payload:" + new String(payload, "utf-8"));
        List<GenericRecord> list = new LinkedList<>();
        DatumReader<GenericRecord> reader = new GenericDatumReader<>(schema);
        BinaryDecoder decoder = DecoderFactory.get().binaryDecoder(buffer.array(), start, length, null);

        GenericRecord genericRecord = reader.read(null, decoder);
        list.add((GenericRecord) genericRecord);

        return list;
    }
 

private <T> T readWithSlowAvro(Schema readerSchema, Schema writerSchema, Decoder decoder, boolean specific) {
  DatumReader<T> datumReader;
  if (specific) {
    datumReader = new SpecificDatumReader<>(writerSchema, readerSchema);
  } else {
    datumReader = new GenericDatumReader<>(writerSchema, readerSchema);
  }
  try {
    return datumReader.read(null, decoder);
  } catch (IOException e) {
    e.printStackTrace();
    return null;
  }
}
 

@Test(groups = {"deserializationTest"})
public void testFastGenericDeserializerGenerator() throws Exception {
  FastSerdeCache cache = FastSerdeCache.getDefaultInstance();
  FastDeserializer<GenericRecord> deserializer =
      (FastDeserializer<GenericRecord>) cache.buildFastGenericDeserializer(COMPLICATE_SCHEMA, COMPLICATE_SCHEMA);

  // Generate a record
  GenericRecord record = newComplicateRecord('0');
  byte[] serializedBytes = serialize(record, COMPLICATE_SCHEMA);
  // Generate a different record
  GenericRecord reuseRecord = newComplicateRecord('1');


  GenericRecord deserializedRecordWithFastAvro = deserializer.deserialize(getDecoder(serializedBytes));
  GenericRecord deserializedRecordWithFastAvroWithReuse =
      deserializer.deserialize(deserializedRecordWithFastAvro, getDecoder(serializedBytes));
  compareTwoRecords(deserializedRecordWithFastAvro, deserializedRecordWithFastAvroWithReuse);

  DatumReader datumReader = new GenericDatumReader(COMPLICATE_SCHEMA);
  GenericRecord deserializedRecordWithRegularAvro =
      (GenericRecord) datumReader.read(null, getDecoder(serializedBytes));
  // regenerate the reuse record since the original one is modified in the last step.
  reuseRecord = newComplicateRecord('1');
  GenericRecord deserializedRecordWithRegularAvroWithReuse =
      (GenericRecord) datumReader.read(reuseRecord, getDecoder(serializedBytes));
  compareTwoRecords(deserializedRecordWithFastAvro, deserializedRecordWithRegularAvro);
  compareTwoRecords(deserializedRecordWithFastAvro, deserializedRecordWithRegularAvroWithReuse);
}
 

public static Person desFromAvroBytes(byte[] record) throws IOException {
    DatumReader<GenericRecord> datumReader = new GenericDatumReader<GenericRecord>(schema);
    BinaryDecoder decoder = null;
    decoder = DecoderFactory.get().binaryDecoder(record, decoder);
    GenericRecord avroValue = datumReader.read(null, decoder);
    return fromAvroRecord(avroValue);
}
 

@Override
public GenericRecord formatRecord(PubsubMessage element, Schema schema) {
  InputStream in = new ByteArrayInputStream(element.getPayload());
  DatumReader<GenericRecord> reader = new GenericDatumReader<>(schema);
  Decoder decoder = DecoderFactory.get().binaryDecoder(in, null);
  try {
    return reader.read(null, decoder);
  } catch (IOException e) {
    throw new UncheckedIOException(e);
  }
}
 

/**
 * Reads provided {@link InputStream} as ISON into Avro
 * {@link GenericRecord} applying provided {@link Schema} returning the
 * resulting GenericRecord.
 */
public static GenericRecord read(InputStream jsonIs, Schema schema) {
    DataInputStream din = new DataInputStream(jsonIs);
    try {
        Decoder decoder = DecoderFactory.get().jsonDecoder(schema, din);
        DatumReader<GenericData.Record> reader = new GenericDatumReader<>(schema);
        return reader.read(null, decoder);
    } catch (Exception e) {
        throw new IllegalStateException("Failed to parse incoming Json input stream into Avro GenericRecord. "
                + "Possible reason: the value may not be a valid JSON or incompatible schema is provided. Schema was '"
                + schema.toString(true) + "'.", e);
    }
}
 

public static Person desFromAvroBytes(byte[] record) throws IOException {
    DatumReader<GenericRecord> datumReader = new GenericDatumReader<GenericRecord>(schema);
    BinaryDecoder decoder = null;
    decoder = DecoderFactory.get().binaryDecoder(record, decoder);
    GenericRecord avroValue = datumReader.read(null, decoder);
    return fromAvroRecord(avroValue);
}
 

public AvroHttpRequest deSerealizeAvroHttpRequestBinary(byte[] data) {
    DatumReader<AvroHttpRequest> employeeReader = new SpecificDatumReader<>(AvroHttpRequest.class);
    Decoder decoder = DecoderFactory.get()
        .binaryDecoder(data, null);
    try {
        return employeeReader.read(null, decoder);
    } catch (IOException e) {
        logger.error("Deserialization error" + e.getMessage());
    }
    return null;
}
 

@Override
protected U readData(Schema schema, ByteBuffer buffer, int start, int length) {
    try {
        DatumReader<U> reader = avroDatumProvider.createDatumReader(schema);
        return reader.read(null, decoderFactory.binaryDecoder(buffer.array(), start, length, null));
    } catch (IOException e) {
        throw new UncheckedIOException(e);
    }
}
 

public static Employee deserializeEmployee(byte[] data) throws IOException {
    DatumReader<Employee> employeeDatumReader = new SpecificDatumReader<>(Employee.class);
    BinaryDecoder binaryDecoder = DecoderFactory.get().binaryDecoder(data, null);
    employeeDatumReader.setSchema(Employee.getClassSchema());
    return employeeDatumReader.read(null, binaryDecoder);
}
 

@Test
public void testWriteAvroRecordsDropSchemaSingleEvent() throws InterruptedException, StageException, IOException {

  DataGeneratorFormatConfig dataGeneratorFormatConfig = new DataGeneratorFormatConfig();
  dataGeneratorFormatConfig.avroSchema = SdcAvroTestUtil.AVRO_SCHEMA1;
  dataGeneratorFormatConfig.avroSchemaSource = INLINE;
  dataGeneratorFormatConfig.includeSchema = false;
  dataGeneratorFormatConfig.avroCompression = AvroCompression.NULL;
  FlumeTarget flumeTarget = FlumeTestUtil.createFlumeTarget(
    FlumeTestUtil.createDefaultFlumeConfig(port, true),
    DataFormat.AVRO,
    dataGeneratorFormatConfig
  );
  TargetRunner targetRunner = new TargetRunner.Builder(FlumeDTarget.class, flumeTarget).build();

  targetRunner.runInit();
  List<Record> records = SdcAvroTestUtil.getRecords1();
  targetRunner.runWrite(records);
  targetRunner.runDestroy();

  List<GenericRecord> genericRecords = new ArrayList<>();
  DatumReader<GenericRecord> datumReader = new GenericDatumReader<>(); //Reader schema argument is optional
  datumReader.setSchema(new Schema.Parser().parse(SdcAvroTestUtil.AVRO_SCHEMA1));

  int eventCounter = 0;
  Transaction transaction = ch.getTransaction();
  transaction.begin();
  Event event = ch.take();
  while(event != null) {
    eventCounter++;
    BinaryDecoder decoder = DecoderFactory.get().binaryDecoder(event.getBody(), null);
    GenericRecord read = datumReader.read(null, decoder);
    while(read != null) {
      genericRecords.add(read);
      try {
        read = datumReader.read(null, decoder);
      } catch (EOFException e) {
        break;
      }
    }
    event = ch.take();
  }
  transaction.commit();
  transaction.close();

  Assert.assertEquals(1, eventCounter);
  Assert.assertEquals(3, genericRecords.size());
  SdcAvroTestUtil.compare1(genericRecords);
}
 

@Test
public void testSerializeAvro() throws Exception {


  Schema schema = new Schema.Parser().parse("{\n" +
      "\t\"type\":\"record\",\n" +
      "\t\"name\":\"test_schema_1\",\n" +
      "\t\"fields\" : [ {\n" +
      "\t\t\"name\":\"a\",\n" +
      "\t\t\"type\":\"int\"\n" +
      "\t\t},\n" +
      "\t\t{\n" +
      "\t\t\"name\":\"b\",\n" +
      "\t\t\"type\":\"string\"\n" +
      "\t\t}\n" +
      "\t\t]\n" +
      "}");

  GenericRecord event = new GenericData.Record(schema);

  event.put("a", 1);
  event.put("b", "static string");

  byte[] m = BaseProducer.serializeAvro(schema,event);

  DatumReader<GenericData.Record> reader = new GenericDatumReader<GenericData.Record>(schema);
  BinaryDecoder binaryDecoder = DecoderFactory.get().binaryDecoder(m,null);

  GenericRecord record = reader.read(null,binaryDecoder);

  Assert.assertEquals(record,event);




}