org.apache.avro.io.Decoder Java Examples

public IndexedRecord deserializerecord0(Object reuse, Decoder decoder)
    throws IOException
{
    IndexedRecord record;
    if ((((reuse)!= null)&&((reuse) instanceof IndexedRecord))&&(((IndexedRecord)(reuse)).getSchema() == mapMapValueSchema0)) {
        record = ((IndexedRecord)(reuse));
    } else {
        record = new org.apache.avro.generic.GenericData.Record(mapMapValueSchema0);
    }
    int unionIndex0 = (decoder.readIndex());
    if (unionIndex0 == 0) {
        decoder.readNull();
    }
    if (unionIndex0 == 1) {
        if (record.get(0) instanceof Utf8) {
            record.put(0, (decoder).readString(((Utf8) record.get(0))));
        } else {
            record.put(0, (decoder).readString(null));
        }
    }
    return record;
}
 

public List<Double> deserialize(List<Double> reuse, Decoder decoder)
    throws IOException
{
    PrimitiveDoubleList array0 = null;
    long chunkLen0 = (decoder.readArrayStart());
    if (chunkLen0 > 0) {
        if ((reuse) instanceof PrimitiveDoubleList) {
            array0 = ((PrimitiveDoubleList)(reuse));
            array0 .clear();
        } else {
            array0 = new PrimitiveDoubleArrayList(((int) chunkLen0));
        }
        do {
            for (int counter0 = 0; (counter0 <chunkLen0); counter0 ++) {
                array0 .addPrimitive((decoder.readDouble()));
            }
            chunkLen0 = (decoder.arrayNext());
        } while (chunkLen0 > 0);
    } else {
        array0 = new PrimitiveDoubleArrayList(((int) chunkLen0));
    }
    return array0;
}
 

public <T> Decoder dataAsBinaryDecoder(T data, Schema schema) {
  ByteArrayOutputStream baos = new ByteArrayOutputStream();
  Encoder binaryEncoder = AvroCompatibilityHelper.newBinaryEncoder(baos, true, null); //new BinaryEncoder(baos);

  try {
    FastGenericSerializerGenerator<T> fastGenericSerializerGenerator =
        new FastGenericSerializerGenerator<>(schema, tempDir, classLoader, null);
    FastSerializer<T> fastSerializer = fastGenericSerializerGenerator.generateSerializer();
    fastSerializer.serialize(data, binaryEncoder);
    binaryEncoder.flush();
  } catch (Exception e) {
    throw new RuntimeException(e);
  }

  return DecoderFactory.defaultFactory().createBinaryDecoder(baos.toByteArray(), null);
}
 

@Override
public GenericArrayData read(Decoder decoder, Object reuse) throws IOException {
  reusedList.clear();
  long chunkLength = decoder.readArrayStart();

  while (chunkLength > 0) {
    for (int i = 0; i < chunkLength; i += 1) {
      reusedList.add(elementReader.read(decoder, null));
    }

    chunkLength = decoder.arrayNext();
  }

  // this will convert the list to an array so it is okay to reuse the list
  return new GenericArrayData(reusedList.toArray());
}
 

@Override
public ArrayBasedMapData read(Decoder decoder, Object reuse) throws IOException {
  reusedKeyList.clear();
  reusedValueList.clear();

  long chunkLength = decoder.readArrayStart();

  while (chunkLength > 0) {
    for (int i = 0; i < chunkLength; i += 1) {
      reusedKeyList.add(keyReader.read(decoder, null));
      reusedValueList.add(valueReader.read(decoder, null));
    }

    chunkLength = decoder.arrayNext();
  }

  return new ArrayBasedMapData(
      new GenericArrayData(reusedKeyList.toArray()),
      new GenericArrayData(reusedValueList.toArray()));
}
 

public List<Integer> deserialize(List<Integer> reuse, Decoder decoder)
    throws IOException
{
    PrimitiveIntList array0 = null;
    long chunkLen0 = (decoder.readArrayStart());
    if (chunkLen0 > 0) {
        if ((reuse) instanceof PrimitiveIntList) {
            array0 = ((PrimitiveIntList)(reuse));
            array0 .clear();
        } else {
            array0 = new PrimitiveIntArrayList(((int) chunkLen0));
        }
        do {
            for (int counter0 = 0; (counter0 <chunkLen0); counter0 ++) {
                array0 .addPrimitive((decoder.readInt()));
            }
            chunkLen0 = (decoder.arrayNext());
        } while (chunkLen0 > 0);
    } else {
        array0 = new PrimitiveIntArrayList(((int) chunkLen0));
    }
    return array0;
}
 

public IndexedRecord deserializerecord0(Object reuse, Decoder decoder)
    throws IOException
{
    IndexedRecord record;
    if ((((reuse)!= null)&&((reuse) instanceof IndexedRecord))&&(((IndexedRecord)(reuse)).getSchema() == mapValueOptionSchema0)) {
        record = ((IndexedRecord)(reuse));
    } else {
        record = new org.apache.avro.generic.GenericData.Record(mapValueOptionSchema0);
    }
    int unionIndex1 = (decoder.readIndex());
    if (unionIndex1 == 0) {
        decoder.readNull();
    }
    if (unionIndex1 == 1) {
        if (record.get(0) instanceof Utf8) {
            record.put(0, (decoder).readString(((Utf8) record.get(0))));
        } else {
            record.put(0, (decoder).readString(null));
        }
    }
    return record;
}
 

public List<Boolean> deserialize(List<Boolean> reuse, Decoder decoder)
    throws IOException
{
    PrimitiveBooleanList array0 = null;
    long chunkLen0 = (decoder.readArrayStart());
    if (chunkLen0 > 0) {
        if ((reuse) instanceof PrimitiveBooleanList) {
            array0 = ((PrimitiveBooleanList)(reuse));
            array0 .clear();
        } else {
            array0 = new PrimitiveBooleanArrayList(((int) chunkLen0));
        }
        do {
            for (int counter0 = 0; (counter0 <chunkLen0); counter0 ++) {
                array0 .addPrimitive((decoder.readBoolean()));
            }
            chunkLen0 = (decoder.arrayNext());
        } while (chunkLen0 > 0);
    } else {
        array0 = new PrimitiveBooleanArrayList(((int) chunkLen0));
    }
    return array0;
}
 

public List<Integer> deserialize(List<Integer> reuse, Decoder decoder)
    throws IOException
{
    PrimitiveIntList array0 = null;
    long chunkLen0 = (decoder.readArrayStart());
    if (chunkLen0 > 0) {
        if ((reuse) instanceof PrimitiveIntList) {
            array0 = ((PrimitiveIntList)(reuse));
            array0 .clear();
        } else {
            array0 = new PrimitiveIntArrayList(((int) chunkLen0));
        }
        do {
            for (int counter0 = 0; (counter0 <chunkLen0); counter0 ++) {
                array0 .addPrimitive((decoder.readInt()));
            }
            chunkLen0 = (decoder.arrayNext());
        } while (chunkLen0 > 0);
    } else {
        array0 = new PrimitiveIntArrayList(((int) chunkLen0));
    }
    return array0;
}
 

private JMethod createMethod(final Schema schema, boolean read) {
  if (!Schema.Type.RECORD.equals(schema.getType())) {
    throw new FastDeserializerGeneratorException("Methods are defined only for records, not for " + schema.getType());
  }
  if (methodAlreadyDefined(schema, read)) {
    throw new FastDeserializerGeneratorException("Method already exists for: " + schema.getFullName());
  }

  JClass schemaClass = schemaAssistant.classFromSchema(schema);
  JMethod method = generatedClass.method(JMod.PUBLIC, read ? schemaClass : codeModel.VOID,
      getUniqueName("deserialize" + schema.getName()));

  method._throws(IOException.class);
  method.param(Object.class, VAR_NAME_FOR_REUSE);
  method.param(Decoder.class, DECODER);

  (read ? deserializeMethodMap : skipMethodMap).put(schema.getFullName(), method);

  return method;
}
 

public <T> Decoder dataAsDecoder(T data, Schema schema) {
  ByteArrayOutputStream baos = new ByteArrayOutputStream();
  Encoder binaryEncoder = AvroCompatibilityHelper.newBinaryEncoder(baos, true, null);

  try {
    FastSpecificSerializerGenerator<T> fastSpecificSerializerGenerator =
        new FastSpecificSerializerGenerator<>(schema, tempDir, classLoader, null);
    FastSerializer<T> fastSerializer = fastSpecificSerializerGenerator.generateSerializer();
    fastSerializer.serialize(data, binaryEncoder);
    binaryEncoder.flush();
  } catch (Exception e) {
    throw new RuntimeException(e);
  }

  return DecoderFactory.defaultFactory().createBinaryDecoder(baos.toByteArray(), null);
}
 

public <T> Decoder writeWithFastAvro(T data, Schema schema, boolean specific) {
  ByteArrayOutputStream baos = new ByteArrayOutputStream();
  Encoder binaryEncoder = AvroCompatibilityHelper.newBinaryEncoder(baos, true, null);

  try {
    FastSerializer<T> fastSerializer;
    if (specific) {
      FastSpecificSerializerGenerator<T> fastSpecificSerializerGenerator = new FastSpecificSerializerGenerator<>(schema, tempDir, classLoader, null);
      fastSerializer = fastSpecificSerializerGenerator.generateSerializer();
    } else {
      FastGenericSerializerGenerator<T> fastGenericSerializerGenerator = new FastGenericSerializerGenerator<>(schema, tempDir, classLoader, null);
      fastSerializer = fastGenericSerializerGenerator.generateSerializer();
    }
    fastSerializer.serialize(data, binaryEncoder);
    binaryEncoder.flush();
  } catch (Exception e) {
    throw new RuntimeException(e);
  }

  return DecoderFactory.defaultFactory().createBinaryDecoder(baos.toByteArray(), null);
}
 

public List<Long> deserialize(List<Long> reuse, Decoder decoder)
    throws IOException
{
    PrimitiveLongList array0 = null;
    long chunkLen0 = (decoder.readArrayStart());
    if (chunkLen0 > 0) {
        if ((reuse) instanceof PrimitiveLongList) {
            array0 = ((PrimitiveLongList)(reuse));
            array0 .clear();
        } else {
            array0 = new PrimitiveLongArrayList(((int) chunkLen0));
        }
        do {
            for (int counter0 = 0; (counter0 <chunkLen0); counter0 ++) {
                array0 .addPrimitive((decoder.readLong()));
            }
            chunkLen0 = (decoder.arrayNext());
        } while (chunkLen0 > 0);
    } else {
        array0 = new PrimitiveLongArrayList(((int) chunkLen0));
    }
    return array0;
}
 

public List<Boolean> deserialize(List<Boolean> reuse, Decoder decoder)
    throws IOException
{
    PrimitiveBooleanList array0 = null;
    long chunkLen0 = (decoder.readArrayStart());
    if (chunkLen0 > 0) {
        if ((reuse) instanceof PrimitiveBooleanList) {
            array0 = ((PrimitiveBooleanList)(reuse));
            array0 .clear();
        } else {
            array0 = new PrimitiveBooleanArrayList(((int) chunkLen0));
        }
        do {
            for (int counter0 = 0; (counter0 <chunkLen0); counter0 ++) {
                array0 .addPrimitive((decoder.readBoolean()));
            }
            chunkLen0 = (decoder.arrayNext());
        } while (chunkLen0 > 0);
    } else {
        array0 = new PrimitiveBooleanArrayList(((int) chunkLen0));
    }
    return array0;
}
 

/**
 * Converts the avro binary data to the json format
 */
@Override
public XContentBuilder serialize(Event event) {
    XContentBuilder builder = null;
    try {
        if (datumReader != null) {
            Decoder decoder = new DecoderFactory().binaryDecoder(event.getBody(), null);
            GenericRecord data = datumReader.read(null, decoder);
            logger.trace("Record in event " + data);
            XContentParser parser = XContentFactory
                    .xContent(XContentType.JSON)
                    .createParser(NamedXContentRegistry.EMPTY,
                            DeprecationHandler.THROW_UNSUPPORTED_OPERATION,
                            data.toString());
            builder = jsonBuilder().copyCurrentStructure(parser);
            parser.close();
        } else {
            logger.error("Schema File is not configured");
        }
    } catch (IOException e) {
        logger.error("Exception in parsing avro format data but continuing serialization to process further records",
                e.getMessage(), e);
    }
    return builder;
}
 

public List<Integer> deserialize(List<Integer> reuse, Decoder decoder)
    throws IOException
{
    PrimitiveIntList array0 = null;
    long chunkLen0 = (decoder.readArrayStart());
    if (chunkLen0 > 0) {
        if ((reuse) instanceof PrimitiveIntList) {
            array0 = ((PrimitiveIntList)(reuse));
            array0 .clear();
        } else {
            array0 = new PrimitiveIntArrayList(((int) chunkLen0));
        }
        do {
            for (int counter0 = 0; (counter0 <chunkLen0); counter0 ++) {
                array0 .addPrimitive((decoder.readInt()));
            }
            chunkLen0 = (decoder.arrayNext());
        } while (chunkLen0 > 0);
    } else {
        array0 = new PrimitiveIntArrayList(((int) chunkLen0));
    }
    return array0;
}
 

public IndexedRecord deserializerecord0(Object reuse, Decoder decoder)
    throws IOException
{
    IndexedRecord record;
    if ((((reuse)!= null)&&((reuse) instanceof IndexedRecord))&&(((IndexedRecord)(reuse)).getSchema() == mapValueOptionSchema0)) {
        record = ((IndexedRecord)(reuse));
    } else {
        record = new org.apache.avro.generic.GenericData.Record(mapValueOptionSchema0);
    }
    int unionIndex1 = (decoder.readIndex());
    if (unionIndex1 == 0) {
        decoder.readNull();
    }
    if (unionIndex1 == 1) {
        if (record.get(0) instanceof Utf8) {
            record.put(0, (decoder).readString(((Utf8) record.get(0))));
        } else {
            record.put(0, (decoder).readString(null));
        }
    }
    return record;
}
 

/**
 * Reads provided {@link InputStream} as ISON into Avro
 * {@link GenericRecord} applying provided {@link Schema} returning the
 * resulting GenericRecord.
 */
public static GenericRecord read(InputStream jsonIs, Schema schema) {
    DataInputStream din = new DataInputStream(jsonIs);
    try {
        Decoder decoder = DecoderFactory.get().jsonDecoder(schema, din);
        DatumReader<GenericData.Record> reader = new GenericDatumReader<>(schema);
        return reader.read(null, decoder);
    } catch (Exception e) {
        throw new IllegalStateException("Failed to parse incoming Json input stream into Avro GenericRecord. "
                + "Possible reason: the value may not be a valid JSON or incompatible schema is provided. Schema was '"
                + schema.toString(true) + "'.", e);
    }
}
 

@Test
public void testGetDataBinary() throws java.io.IOException {
    // given
    UiSpecsPropertiesDto propertiesDto = new UiSpecsPropertiesDto();
    propertiesDto.setProperties(getFileAsObjectNode("jdbc_data_set_properties_with_schema.json"));
    propertiesDto.setDependencies(singletonList(getJdbcDataStoreProperties()));

    String dataSetDefinitionName = "JDBCDataset";

    // when
    Response schemaResponse = given().body(propertiesDto).contentType(APPLICATION_JSON_UTF8_VALUE) //
                                     .accept(APPLICATION_JSON_UTF8_VALUE) //
                                     .post(getVersionPrefix() + "/runtimes/schema");
    schemaResponse.then().statusCode(200).log().ifError();

    Schema schema = new Schema.Parser().parse(schemaResponse.asInputStream());

    Response response = given().body(propertiesDto).contentType(APPLICATION_JSON_UTF8_VALUE) //
            .accept(RuntimesController.AVRO_BINARY_MIME_TYPE_OFFICIAL_INVALID).post(getVersionPrefix() + "/runtimes/data"); //
    response.then().statusCode(200).log().ifError();

    // then
    GenericDatumReader<GenericRecord> reader = new GenericDatumReader<>(schema);
    DecoderFactory decoderFactory = DecoderFactory.get();
    Decoder decoder = decoderFactory.binaryDecoder(response.asInputStream(), null);
    assertRecordsEqualsToTestValues(reader, decoder);
}
 

public Map<Utf8, IndexedRecord> deserialize(Map<Utf8, IndexedRecord> reuse, Decoder decoder)
    throws IOException
{
    Map<Utf8, IndexedRecord> map0 = null;
    long chunkLen0 = (decoder.readMapStart());
    if (chunkLen0 > 0) {
        Map<Utf8, IndexedRecord> mapReuse0 = null;
        if ((reuse) instanceof Map) {
            mapReuse0 = ((Map)(reuse));
        }
        if (mapReuse0 != (null)) {
            mapReuse0 .clear();
            map0 = mapReuse0;
        } else {
            map0 = new HashMap<Utf8, IndexedRecord>();
        }
        do {
            for (int counter0 = 0; (counter0 <chunkLen0); counter0 ++) {
                Utf8 key0 = (decoder.readString(null));
                map0 .put(key0, deserializerecord0(null, (decoder)));
            }
            chunkLen0 = (decoder.mapNext());
        } while (chunkLen0 > 0);
    } else {
        map0 = Collections.emptyMap();
    }
    return map0;
}
 

@Override
public T read(T reuse, Decoder decoder) throws IOException {
  ResolvingDecoder resolver = resolve(decoder);
  T value = reader.read(resolver, reuse);
  resolver.drain();
  return value;
}
 

@Override
public GenericRecord formatRecord(PubsubMessage element, Schema schema) {
  InputStream in = new ByteArrayInputStream(element.getPayload());
  DatumReader<GenericRecord> reader = new GenericDatumReader<>(schema);
  Decoder decoder = DecoderFactory.get().binaryDecoder(in, null);
  try {
    return reader.read(null, decoder);
  } catch (IOException e) {
    throw new UncheckedIOException(e);
  }
}
 

@SuppressWarnings("unchecked")
private static <T> T decodeRecordSlow(Schema writerSchema, Schema readerSchema, Decoder decoder) {
  org.apache.avro.io.DatumReader<GenericData> datumReader = new GenericDatumReader<>(writerSchema, readerSchema);
  try {
    return (T) datumReader.read(null, decoder);
  } catch (IOException e) {
    e.printStackTrace();
    return null;
  }
}
 

@Override
public byte[] read(Decoder decoder, Object reuse) throws IOException {
  if (reuse instanceof byte[]) {
    byte[] reusedBytes = (byte[]) reuse;
    if (reusedBytes.length == length) {
      decoder.readFixed(reusedBytes, 0, length);
      return reusedBytes;
    }
  }

  byte[] bytes = new byte[length];
  decoder.readFixed(bytes, 0, length);
  return bytes;
}
 

@Override
    public byte[] read(Decoder decoder, Object reuse) throws IOException {
      // use the decoder's readBytes method because it may be a resolving decoder
      // the only time the previous value could be reused is when its length matches the next array,
      // but there is no way to know this with the readBytes call, which uses a ByteBuffer. it is
      // possible to wrap the reused array in a ByteBuffer, but this may still result in allocating
      // a new buffer. since the usual case requires an allocation anyway to get the size right,
      // just allocate every time.
      return decoder.readBytes(null).array();
//      int length = decoder.readInt();
//      byte[] bytes = new byte[length];
//      decoder.readFixed(bytes, 0, length);
//      return bytes;
    }
 

@SuppressWarnings("unchecked")
private GenericRecord decodeGenericSlow(Schema readerSchema, Schema writerSchema, Decoder decoder) {
  org.apache.avro.io.DatumReader<GenericData> datumReader = new GenericDatumReader<>(writerSchema, readerSchema);
  try {
    return (GenericRecord) datumReader.read(null, decoder);
  } catch (IOException e) {
    e.printStackTrace();
    return null;
  }
}
 

public static <T> Decoder serializeGeneric(T data, Schema schema) {
    ByteArrayOutputStream baos = new ByteArrayOutputStream();
    BinaryEncoder binaryEncoder = EncoderFactory.get().directBinaryEncoder(baos, null);

    try {
        GenericDatumWriter<T> writer = new GenericDatumWriter<>(schema);
        writer.write(data, binaryEncoder);
        binaryEncoder.flush();

    } catch (Exception e) {
        throw new RuntimeException(e);
    }

    return DecoderFactory.get().binaryDecoder(baos.toByteArray(), null);
}
 

public Map<Utf8, IndexedRecord> deserialize(Map<Utf8, IndexedRecord> reuse, Decoder decoder)
    throws IOException
{
    Map<Utf8, IndexedRecord> map0 = null;
    long chunkLen0 = (decoder.readMapStart());
    if (chunkLen0 > 0) {
        Map<Utf8, IndexedRecord> mapReuse0 = null;
        if ((reuse) instanceof Map) {
            mapReuse0 = ((Map)(reuse));
        }
        if (mapReuse0 != (null)) {
            mapReuse0 .clear();
            map0 = mapReuse0;
        } else {
            map0 = new HashMap<Utf8, IndexedRecord>();
        }
        do {
            for (int counter0 = 0; (counter0 <chunkLen0); counter0 ++) {
                Utf8 key0 = (decoder.readString(null));
                int unionIndex0 = (decoder.readIndex());
                if (unionIndex0 == 0) {
                    decoder.readNull();
                }
                if (unionIndex0 == 1) {
                    map0 .put(key0, deserializerecord0(null, (decoder)));
                }
            }
            chunkLen0 = (decoder.mapNext());
        } while (chunkLen0 > 0);
    } else {
        map0 = Collections.emptyMap();
    }
    return map0;
}
 

/**
 * Called to read a fixed value. Overridden to read a pig byte array.
 */
@Override
protected Object readFixed(Object old, Schema expected, Decoder in) throws IOException {
    GenericFixed fixed = (GenericFixed) super.readFixed(old, expected, in);
    DataByteArray byteArray = new DataByteArray(fixed.bytes());
    return byteArray;
 }
 

public <T> T decodeRecord(Schema schema, Decoder decoder) {
  GenericDatumReader<T> datumReader = new GenericDatumReader<>(schema);
  try {
    return datumReader.read(null, decoder);
  } catch (Exception e) {
    throw new RuntimeException(e);
  }
}