io.protostuff.ProtobufIOUtil Java Examples

@JsonIgnore
@SuppressWarnings({ "rawtypes", "unchecked" })
public FileConfig asFileConfig() {
  buffer.clear();
  FileConfig fc = new FileConfig();
  fc.setType(getFileType());
  fc.setName(name);
  fc.setOwner(owner);
  fc.setCtime(ctime);
  fc.setType(getFileType());
  fc.setTag(getVersion());
  fc.setLocation(location);
  byte[] bytes = ProtobufIOUtil.toByteArray(this, (Schema) getPrivateSchema(), buffer);
  fc.setExtendedConfig(ByteString.copyFrom(bytes));
  fc.setFullPathList(fullPath);
  return fc;
}
 

private static FileFormat get(FileConfig fileConfig) {
  // TODO (Amit H) Remove after defining classes for tsv, csv, and psv
  FileType fileType = fileConfig.getType();
  if (fileType == FileType.CSV || fileType == FileType.TSV || fileType == FileType.PSV) {
    fileType = FileType.TEXT;
  }
  final Class<? extends FileFormat> fileFormatClass = FileFormatDefinitions.CLASS_TYPES.get(fileType);
  final Schema<FileFormat> schema = (Schema<FileFormat>) FileFormatDefinitions.SCHEMAS.get(fileFormatClass);

  final FileFormat fileFormat = schema.newMessage();
  if (fileConfig.getExtendedConfig() != null) {
    ProtobufIOUtil.mergeFrom(fileConfig.getExtendedConfig().toByteArray(), fileFormat, schema);
  }

  fileFormat.setCtime(fileConfig.getCtime());
  fileFormat.setName(fileConfig.getName());
  fileFormat.setOwner(fileConfig.getOwner());
  fileFormat.setFullPath(fileConfig.getFullPathList());
  fileFormat.setVersion(fileConfig.getTag());
  fileFormat.setLocation(fileConfig.getLocation());
  return fileFormat;
}
 

/**
 * Decode a message from the passed input stream, and compute and verify its CRC. This method reads
 * data written by the method {@link EntryEncodingUtil#encodeWithLengthAndCrc}.
 *
 * @param inputStream Input stream, opened for reading and positioned just before the length-prepended header
 * @return The deserialized, constructed, validated message
 * @throws IOException                if a problem is encountered while reading or parsing
 * @throws EntryEncodingUtil.CrcError if the recorded CRC of the message does not match its computed CRC.
 */
public static <T> T decodeAndCheckCrc(InputStream inputStream, Schema<T> schema)
    throws IOException, CrcError {
  // TODO this should check the length first and compare it with a passed-in maximum length
  final T message = schema.newMessage();
  final CrcInputStream crcStream = new CrcInputStream(inputStream, new Adler32());
  ProtobufIOUtil.mergeDelimitedFrom(crcStream, message, schema);

  final long computedCrc = crcStream.getValue();
  final long diskCrc = readCrc(inputStream);
  if (diskCrc != computedCrc) {
    throw new CrcError("CRC mismatch on deserialized message " + message.toString());
  }

  return message;
}
 

public void testProtobuf() throws Exception
{
    Schema<Zoo> schema = RuntimeSchema.getSchema(Zoo.class);
    Zoo p = filledZoo();

    byte[] data = ProtobufIOUtil.toByteArray(p, schema, buf());
    // System.err.println("protobuf: " + data.length);

    Zoo p2 = new Zoo();
    ProtobufIOUtil.mergeFrom(data, p2, schema);

    assertEquals(p, p2);

    List<Zoo> list = new ArrayList<Zoo>();
    list.add(p);
    list.add(p2);

    ByteArrayOutputStream out = new ByteArrayOutputStream();
    ProtobufIOUtil.writeListTo(out, list, schema, buf());
    byte[] listData = out.toByteArray();

    ByteArrayInputStream in = new ByteArrayInputStream(listData);
    List<Zoo> parsedList = ProtobufIOUtil.parseListFrom(in, schema);

    assertEquals(list, parsedList);
}
 

@Override
public byte[] serialize(String topic, Company data) {
    if (data == null) {
        return null;
    }
    Schema schema = RuntimeSchema.getSchema(data.getClass());
    LinkedBuffer buffer = LinkedBuffer.allocate(LinkedBuffer.DEFAULT_BUFFER_SIZE);
    byte[] protostuff = null;
    try {
        protostuff = ProtobufIOUtil.toByteArray(data, schema, buffer);
    } catch (Exception e) {
        throw new IllegalStateException(e.getMessage(), e);
    } finally {
        buffer.clear();
    }
    return protostuff;
}
 

public void testInheritanceProtobuf() throws IOException
{
    Schema<InputSystem> schema = getSchema(InputSystem.class);
    InputSystem sys = new InputSystem();
    KeyBoard kb = new KeyBoard();
    Mouse ms = new Mouse();
    kb.setName("Test");
    kb.setNumberOfKeys(10);
    ms.setName("Test1");
    ms.setNumberOfButtons(2);
    List<InputDevice> devices = new ArrayList<InputDevice>();
    devices.add(ms);
    devices.add(kb);
    sys.setInputDevices(devices);
    ByteArrayOutputStream out = new ByteArrayOutputStream();
    ProtobufIOUtil.writeTo(out, sys, schema, buf());
    byte[] listData = out.toByteArray();
    InputSystem deserSystem = new InputSystem();
    ByteArrayInputStream in = new ByteArrayInputStream(listData);
    ProtobufIOUtil.mergeFrom(in, deserSystem, schema, buf());

    assertEquals(sys, deserSystem);
}
 

private void communicateChange(SourceConfig config, RpcType rpcType) {

    final Set<NodeEndpoint> endpoints = new HashSet<>();
    endpoints.add(context.get().getEndpoint());

    List<RpcFuture<Ack>> futures = new ArrayList<>();
    SourceWrapper wrapper = SourceWrapper.newBuilder().setBytes(ByteString.copyFrom(ProtobufIOUtil.toByteArray(config, SourceConfig.getSchema(), LinkedBuffer.allocate()))).build();
    for(NodeEndpoint e : Iterables.concat(this.context.get().getCoordinators(), this.context.get().getExecutors())) {
      if(!endpoints.add(e)) {
        continue;
      }

      SendSource send = new SendSource(wrapper, rpcType);
      tunnelFactory.getCommandRunner(e.getAddress(), e.getFabricPort()).runCommand(send);
      logger.trace("Sending [{}] to {}:{}", config.getName(), e.getAddress(), e.getUserPort());
      futures.add(send.getFuture());
    }

    try {
      Futures.successfulAsList(futures).get(CHANGE_COMMUNICATION_WAIT, TimeUnit.MILLISECONDS);
    } catch (InterruptedException | ExecutionException | TimeoutException e1) {
      logger.warn("Failure while communicating source change [{}].", config.getName(), e1);
    }
  }
 

@Override
protected <T> void roundTrip(T message, Schema<T> schema,
        Pipe.Schema<T> pipeSchema) throws Exception
{
    byte[] protobuf = ProtobufIOUtil.toByteArray(message, schema, buf());

    ByteArrayInputStream protobufStream = new ByteArrayInputStream(protobuf);

    byte[] protostuff = ProtostuffIOUtil.toByteArray(
            ProtobufIOUtil.newPipe(protobuf, 0, protobuf.length),
            pipeSchema, buf());

    byte[] protostuffFromStream = ProtostuffIOUtil.toByteArray(
            ProtobufIOUtil.newPipe(protobufStream), pipeSchema, buf());

    assertTrue(Arrays.equals(protostuff, protostuffFromStream));

    T parsedMessage = schema.newMessage();
    ProtostuffIOUtil.mergeFrom(protostuff, parsedMessage, schema);
    SerializableObjects.assertEquals(message, parsedMessage);

    ByteArrayInputStream protostuffStream = new ByteArrayInputStream(
            protostuff);

    byte[] protobufRoundTrip = ProtobufIOUtil.toByteArray(
            ProtostuffIOUtil.newPipe(protostuff, 0, protostuff.length),
            pipeSchema, buf());

    byte[] protobufRoundTripFromStream = ProtobufIOUtil.toByteArray(
            ProtostuffIOUtil.newPipe(protostuffStream), pipeSchema, buf());

    assertTrue(Arrays.equals(protobufRoundTrip, protobufRoundTripFromStream));

    assertTrue(Arrays.equals(protobufRoundTrip, protobuf));
}
 

@Override
public final T clone() {
  byte[] bytes = toBytes();
  T message = schema.newMessage();
  ProtobufIOUtil.mergeFrom(bytes, message, schema);
  return message;
}
 

@SuppressWarnings("unchecked")
@Override
public void writeTo(Object message, Class<?> clazz, Type type, Annotation[] annotations, MediaType media, MultivaluedMap<String,Object> httpHeaders,
                OutputStream out) throws IOException, WebApplicationException {
    
    // TODO: Figure out a method to add the proto file location in the response headers.
    // This map must be mofified before any data is written to out,
    // since at that time the response headers will be flushed.
    
    Schema<Object> schema = null;
    if (message instanceof Message) {
        Message<Object> msg = (Message<Object>) message;
        schema = msg.cachedSchema();
    } else {
        schema = (Schema<Object>) RuntimeSchema.getSchema(clazz);
    }
    
    try {
        if (MediaType.APPLICATION_XML_TYPE.equals(media) || MediaType.TEXT_XML_TYPE.equals(media)) {
            XmlIOUtil.writeTo(out, message, schema);
        } else if ("text/yaml".equals(media.toString()) || "text/x-yaml".equals(media.toString()) || "application/x-yaml".equals(media.toString())) {
            YamlIOUtil.writeTo(out, message, schema, buffer);
        } else if ("application/x-protobuf".equals(media.toString())) {
            ProtobufIOUtil.writeTo(out, message, schema, buffer);
        } else if ("application/x-protostuff".equals(media.toString())) {
            ProtostuffIOUtil.writeTo(out, message, schema, buffer);
        } else if (MediaType.APPLICATION_JSON_TYPE.equals(media)) {
            IOContext ctx = new IOContext(JsonIOUtil.DEFAULT_JSON_FACTORY._getBufferRecycler(), out, false);
            UTF8JsonGenerator generator = new UTF8JsonGenerator(ctx, JsonIOUtil.DEFAULT_JSON_FACTORY.getGeneratorFeatures(),
                            JsonIOUtil.DEFAULT_JSON_FACTORY.getCodec(), out);
            try {
                JsonIOUtil.writeTo(generator, message, schema, false);
            } finally {
                generator.close();
            }
        }
    } finally {
        buffer.clear();
    }
}
 

@Test
void unknownEnumValue_toString() {
    TestUnknownEnumValue2 source = TestUnknownEnumValue2.newBuilder()
            .setField(TestUnknownEnumValue2.E2.C)
            .build();
    byte[] data = ProtobufIOUtil.toByteArray(source, TestUnknownEnumValue2.getSchema(), LinkedBuffer.allocate());
    Schema<TestUnknownEnumValue1> schema = TestUnknownEnumValue1.getSchema();
    TestUnknownEnumValue1 message = schema.newMessage();
    ProtobufIOUtil.mergeFrom(data, message, schema);
    Assertions.assertEquals("TestUnknownEnumValue1{field=UNRECOGNIZED(2)}", message.toString());
}
 

private static Object readContent(final EventType eventType, final byte[] content) {
    final Class<?> targetClass = eventType.getTargetClass();
    if (targetClass == null) {
        LOG.error("Cannot read clazz header for given EventType value {}, missing mapping", eventType.getValue());
        throw new MessageConversionException("Missing mapping of EventType for value " + eventType.getValue());
    }
    @SuppressWarnings("unchecked")
    final Schema<Object> schema = (Schema<Object>) RuntimeSchema.getSchema(targetClass);
    final Object deserializeEvent = schema.newMessage();
    ProtobufIOUtil.mergeFrom(content, deserializeEvent, schema);
    return deserializeEvent;
}
 

private static byte[] writeContent(final Object payload) {
    final Class<?> serializeClass = payload.getClass();
    @SuppressWarnings("unchecked")
    final Schema<Object> schema = (Schema<Object>) RuntimeSchema.getSchema(serializeClass);
    final LinkedBuffer buffer = LinkedBuffer.allocate();
    return ProtobufIOUtil.toByteArray(payload, schema, buffer);
}
 

@Benchmark
public GeneratedInt10 generated_deserialize_10_int_field() throws Exception
{
    GeneratedInt10 int10 = new GeneratedInt10();
    ProtobufIOUtil.mergeFrom(data_10_int, int10, GeneratedInt10.getSchema());
    return int10;
}
 

@Override
protected <T> void roundTrip(T message, Schema<T> schema,
        Pipe.Schema<T> pipeSchema) throws Exception
{
    byte[] protobuf = ProtobufIOUtil.toByteArray(message, schema, buf());

    ByteArrayInputStream protobufStream = new ByteArrayInputStream(protobuf);

    byte[] protostuff = ProtostuffIOUtil.toByteArray(
            ProtobufIOUtil.newPipe(protobuf, 0, protobuf.length),
            pipeSchema, buf());

    byte[] protostuffFromStream = ProtostuffIOUtil.toByteArray(
            ProtobufIOUtil.newPipe(protobufStream), pipeSchema, buf());

    assertTrue(Arrays.equals(protostuff, protostuffFromStream));

    T parsedMessage = schema.newMessage();
    ProtostuffIOUtil.mergeFrom(protostuff, parsedMessage, schema);
    SerializableObjects.assertEquals(message, parsedMessage);

    ByteArrayInputStream protostuffStream = new ByteArrayInputStream(
            protostuff);

    byte[] protobufRoundTrip = ProtobufIOUtil.toByteArray(
            ProtostuffIOUtil.newPipe(protostuff, 0, protostuff.length),
            pipeSchema, buf());

    byte[] protobufRoundTripFromStream = ProtobufIOUtil.toByteArray(
            ProtostuffIOUtil.newPipe(protostuffStream), pipeSchema, buf());

    assertTrue(Arrays.equals(protobufRoundTrip, protobufRoundTripFromStream));

    assertTrue(Arrays.equals(protobufRoundTrip, protobuf));
}
 

public void testProtobuf() throws Exception
{
    Schema<Payment> schema = RuntimeSchema.getSchema(Payment.class);
    Payment p = filledPayment();

    byte[] data = ProtobufIOUtil.toByteArray(p, schema, buf());

    Payment p2 = new Payment();
    ProtobufIOUtil.mergeFrom(data, p2, schema);
    /*
     * System.err.println(p2.getId()); System.err.println(p2.getBd()); System.err.println(p2.getBi());
     * System.err.println(p2.getBdList()); System.err.println(p2.getBiList());
     */

    assertEquals(p, p2);

    List<Payment> list = new ArrayList<Payment>();
    list.add(p);
    list.add(p2);

    ByteArrayOutputStream out = new ByteArrayOutputStream();
    ProtobufIOUtil.writeListTo(out, list, schema, buf());
    byte[] listData = out.toByteArray();

    ByteArrayInputStream in = new ByteArrayInputStream(listData);
    List<Payment> parsedList = ProtobufIOUtil.parseListFrom(in, schema);

    assertEquals(list, parsedList);
}
 

@Benchmark
public SparseInt10 runtime_sparse_deserialize_10_int_field() throws Exception
{
    SparseInt10 int10 = new SparseInt10();
    ProtobufIOUtil.mergeFrom(data_10_int, int10, sparseInt10RuntimeSchema);
    return int10;
}
 

@Override
public <T> T deSerialize(byte[] bytes, Class<T> clazz) {
    RuntimeSchema<T> runtimeSchema = RuntimeSchema.createFrom (clazz);
    T object = runtimeSchema.newMessage ();
    ProtobufIOUtil.mergeFrom (bytes, object, runtimeSchema);
    return object;
}
 

@Override
protected <T> void roundTrip(T message, Schema<T> schema,
        Pipe.Schema<T> pipeSchema) throws Exception
{
    byte[] protobuf = ProtobufIOUtil.toByteArray(message, schema, buf());

    ByteArrayInputStream protobufStream = new ByteArrayInputStream(protobuf);

    byte[] protostuff = ProtostuffIOUtil.toByteArray(
            ProtobufIOUtil.newPipe(protobuf, 0, protobuf.length),
            pipeSchema, buf());

    byte[] protostuffFromStream = ProtostuffIOUtil.toByteArray(
            ProtobufIOUtil.newPipe(protobufStream), pipeSchema, buf());

    assertTrue(Arrays.equals(protostuff, protostuffFromStream));

    T parsedMessage = schema.newMessage();
    ProtostuffIOUtil.mergeFrom(protostuff, parsedMessage, schema);
    SerializableObjects.assertEquals(message, parsedMessage);

    ByteArrayInputStream protostuffStream = new ByteArrayInputStream(
            protostuff);

    byte[] protobufRoundTrip = ProtobufIOUtil.toByteArray(
            ProtostuffIOUtil.newPipe(protostuff, 0, protostuff.length),
            pipeSchema, buf());

    byte[] protobufRoundTripFromStream = ProtobufIOUtil.toByteArray(
            ProtostuffIOUtil.newPipe(protostuffStream), pipeSchema, buf());

    assertTrue(Arrays.equals(protobufRoundTrip, protobufRoundTripFromStream));

    assertTrue(Arrays.equals(protobufRoundTrip, protobuf));
}
 

@Benchmark
public void runtime_sparse_serialize_10_int_fields() throws Exception
{
    try
    {
        ProtobufIOUtil.writeTo(buffer, sparseInt10, sparseInt10RuntimeSchema);
    }
    finally
    {
        buffer.clear();
    }
}
 

@Override
protected <T> void roundTrip(T message, Schema<T> schema,
        Pipe.Schema<T> pipeSchema) throws IOException
{
    byte[] protobuf = ProtobufIOUtil.toByteArray(message, schema, buf());

    ByteArrayInputStream protobufStream = new ByteArrayInputStream(protobuf);

    byte[] protostuff = ProtostuffIOUtil.toByteArray(
            ProtobufIOUtil.newPipe(protobuf, 0, protobuf.length),
            pipeSchema, buf());

    byte[] protostuffFromStream = ProtostuffIOUtil.toByteArray(
            ProtobufIOUtil.newPipe(protobufStream), pipeSchema, buf());

    assertTrue(protostuff.length == protostuffFromStream.length);
    assertEquals(STRING.deser(protostuff),
            STRING.deser(protostuffFromStream));

    ByteArrayInputStream protostuffStream = new ByteArrayInputStream(
            protostuff);

    byte[] protobufRoundTrip = ProtobufIOUtil.toByteArray(
            ProtostuffIOUtil.newPipe(protostuff, 0, protostuff.length),
            pipeSchema, buf());

    byte[] protobufRoundTripFromStream = ProtobufIOUtil.toByteArray(
            ProtostuffIOUtil.newPipe(protostuffStream), pipeSchema, buf());

    assertTrue(protobufRoundTrip.length == protobufRoundTripFromStream.length);

    String strProtobufRoundTrip = STRING.deser(protobufRoundTrip);

    assertEquals(strProtobufRoundTrip,
            STRING.deser(protobufRoundTripFromStream));

    assertTrue(protobufRoundTrip.length == protobuf.length);

    assertEquals(strProtobufRoundTrip, STRING.deser(protobuf));
}
 

@Override
protected <T> void roundTrip(T message, Schema<T> schema,
        Pipe.Schema<T> pipeSchema) throws Exception
{
    byte[] protobuf = ProtobufIOUtil.toByteArray(message, schema, buf());

    ByteArrayInputStream protobufStream = new ByteArrayInputStream(protobuf);

    byte[] protostuff = ProtostuffIOUtil.toByteArray(
            ProtobufIOUtil.newPipe(protobuf, 0, protobuf.length),
            pipeSchema, buf());

    byte[] protostuffFromStream = ProtostuffIOUtil.toByteArray(
            ProtobufIOUtil.newPipe(protobufStream), pipeSchema, buf());

    assertTrue(Arrays.equals(protostuff, protostuffFromStream));

    T parsedMessage = schema.newMessage();
    ProtostuffIOUtil.mergeFrom(protostuff, parsedMessage, schema);
    SerializableObjects.assertEquals(message, parsedMessage);

    ByteArrayInputStream protostuffStream = new ByteArrayInputStream(
            protostuff);

    byte[] protobufRoundTrip = ProtobufIOUtil.toByteArray(
            ProtostuffIOUtil.newPipe(protostuff, 0, protostuff.length),
            pipeSchema, buf());

    byte[] protobufRoundTripFromStream = ProtobufIOUtil.toByteArray(
            ProtostuffIOUtil.newPipe(protostuffStream), pipeSchema, buf());

    assertTrue(Arrays.equals(protobufRoundTrip, protobufRoundTripFromStream));

    assertTrue(Arrays.equals(protobufRoundTrip, protobuf));
}
 

@Override
protected <T> void roundTrip(T message, Schema<T> schema,
        Pipe.Schema<T> pipeSchema) throws IOException
{
    byte[] protobuf = ProtobufIOUtil.toByteArray(message, schema, buf());

    ByteArrayInputStream protobufStream = new ByteArrayInputStream(protobuf);

    byte[] protostuff = ProtostuffIOUtil.toByteArray(
            ProtobufIOUtil.newPipe(protobuf, 0, protobuf.length),
            pipeSchema, buf());

    byte[] protostuffFromStream = ProtostuffIOUtil.toByteArray(
            ProtobufIOUtil.newPipe(protobufStream), pipeSchema, buf());

    assertTrue(protostuff.length == protostuffFromStream.length);
    assertEquals(STRING.deser(protostuff),
            STRING.deser(protostuffFromStream));

    ByteArrayInputStream protostuffStream = new ByteArrayInputStream(
            protostuff);

    byte[] protobufRoundTrip = ProtobufIOUtil.toByteArray(
            ProtostuffIOUtil.newPipe(protostuff, 0, protostuff.length),
            pipeSchema, buf());

    byte[] protobufRoundTripFromStream = ProtobufIOUtil.toByteArray(
            ProtostuffIOUtil.newPipe(protostuffStream), pipeSchema, buf());

    assertTrue(protobufRoundTrip.length == protobufRoundTripFromStream.length);

    String strProtobufRoundTrip = STRING.deser(protobufRoundTrip);

    assertEquals(strProtobufRoundTrip,
            STRING.deser(protobufRoundTripFromStream));

    assertTrue(protobufRoundTrip.length == protobuf.length);

    assertEquals(strProtobufRoundTrip, STRING.deser(protobuf));
}
 

@Benchmark
public void generated_serialize_1_int_field() throws Exception
{
    try
    {
        ProtobufIOUtil.writeTo(buffer, generatedInt1, generatedInt1Schema);
    }
    finally
    {
        buffer.clear();
    }
}
 

@Benchmark
public Int1 runtime_deserialize_1_int_field() throws Exception
{
    Int1 int1 = new Int1();
    ProtobufIOUtil.mergeFrom(data_1_int, int1, int1RuntimeSchema);
    return int1;
}
 

@Benchmark
public void runtime_serialize_1_int_field() throws Exception
{
    try
    {
        ProtobufIOUtil.writeTo(buffer, int1, int1RuntimeSchema);
    }
    finally
    {
        buffer.clear();
    }
}
 

@Benchmark
public Int10 runtime_deserialize_10_int_field() throws Exception
{
    Int10 int10 = new Int10();
    ProtobufIOUtil.mergeFrom(data_10_int, int10, int10RuntimeSchema);
    return int10;
}
 

@Benchmark
public void runtime_serialize_10_int_fields() throws Exception
{
    try
    {
        ProtobufIOUtil.writeTo(buffer, int10, int10RuntimeSchema);
    }
    finally
    {
        buffer.clear();
    }
}
 

@Benchmark
public SparseInt1 runtime_sparse_deserialize_1_int_field() throws Exception
{
    SparseInt1 int1 = new SparseInt1();
    ProtobufIOUtil.mergeFrom(data_1_int, int1, sparseInt1RuntimeSchema);
    return int1;
}
 

@Benchmark
public void runtime_sparse_serialize_1_int_field() throws Exception
{
    try
    {
        ProtobufIOUtil.writeTo(buffer, sparseInt1, sparseInt1RuntimeSchema);
    }
    finally
    {
        buffer.clear();
    }
}