Java Code Examples for com.google.protobuf.ByteString#newOutput()

/**
 * Serialize the given value to byte string using the given mapper, employing the given codec algorithm.
 *
 * @param mapper object mapper
 * @param value  value to serialize
 * @param codec  codec
 * @return serialized bytes
 * @throws JsonGenerationException in case of serialization errors
 */
public static ByteString writeValue(ObjectMapper mapper, Object value, Codec codec)
    throws JsonGenerationException {
  final Output output = ByteString.newOutput();

  try {
    final OutputStream os = codec.compress(output);
    try {
      mapper.writer()
          .without(SerializationFeature.INDENT_OUTPUT)
          .writeValue(os, value);
    } finally {
      os.close();
    }
  } catch (IOException e) {
    // Should not happen but...
    throw new JsonGenerationException(e, null);
  }

  // Javadoc says data is copied, but it's more of a transfer of ownership!
  return output.toByteString();
}
 

@Override
public void persistState(StateCheckpointWriter checkpointBuilder) throws IOException {
	if (!contents.isEmpty()) {
		// serialize the coder.
		byte[] coder = InstantiationUtil.serializeObject(elemCoder);

		checkpointBuilder.addListUpdatesBuilder()
				.setTag(stateKey)
				.setData(coder)
				.writeInt(contents.size());

		for (T item : contents) {
			// encode the element
			ByteString.Output stream = ByteString.newOutput();
			elemCoder.encode(item, stream, Coder.Context.OUTER);
			ByteString data = stream.toByteString();

			// add the data to the checkpoint.
			checkpointBuilder.setData(data);
		}
	}
}
 

@Override
public void persistState(StateCheckpointWriter checkpointBuilder) throws IOException {
	if (!isClear) {
		// serialize the coder.
		byte[] coder = InstantiationUtil.serializeObject(accumCoder);

		// serialize the combiner.
		byte[] combiner = InstantiationUtil.serializeObject(combineFn);

		// encode the accumulator into a ByteString
		ByteString.Output stream = ByteString.newOutput();
		accumCoder.encode(accum, stream, Coder.Context.OUTER);
		ByteString data = stream.toByteString();

		// put the flag that the next serialized element is an accumulator
		checkpointBuilder.addAccumulatorBuilder()
			.setTag(stateKey)
			.setData(coder)
			.setData(combiner)
			.setData(data);
	}
}
 

@InterfaceStability.Evolving
@InterfaceAudience.LimitedPrivate({"hive, pig"})
public static MRRuntimeProtos.MRSplitProto createSplitProto(
    org.apache.hadoop.mapred.InputSplit oldSplit) throws IOException {
  MRRuntimeProtos.MRSplitProto.Builder builder = MRRuntimeProtos.MRSplitProto.newBuilder();

  builder.setSplitClassName(oldSplit.getClass().getName());

  ByteString.Output os = ByteString
      .newOutput(SPLIT_SERIALIZED_LENGTH_ESTIMATE);
  oldSplit.write(new NonSyncDataOutputStream(os));
  ByteString splitBs = os.toByteString();
  builder.setSplitBytes(splitBs);

  return builder.build();
}
 

MemoryWriteOutputStream(
    ContentAddressableStorage storage,
    Digest digest,
    ListenableFuture<ByteString> writtenFuture) {
  this.storage = storage;
  this.digest = digest;
  this.writtenFuture = writtenFuture;
  if (digest.getSizeBytes() > Integer.MAX_VALUE) {
    throw new IllegalArgumentException(
        String.format(
            "content size %d exceeds maximum of %d", digest.getSizeBytes(), Integer.MAX_VALUE));
  }
  out = ByteString.newOutput((int) digest.getSizeBytes());
  hashOut = DigestUtil.forDigest(digest).newHashingOutputStream(out);
  addListener(
      () -> {
        future.set(null);
        try {
          hashOut.close();
        } catch (IOException e) {
          // ignore
        }
      },
      directExecutor());
}
 

private static void writeStrategiesIndex(MarshallerWriteContext context,
                                          ProtobufMessages.Header.Builder _header) throws IOException {
     for( Entry<ObjectMarshallingStrategy,Integer> entry : context.usedStrategies.entrySet() ) {
Builder _strat = ProtobufMessages.Header.StrategyIndex.newBuilder()
                                  .setId( entry.getValue().intValue() )
                                  .setName( entry.getKey().getName()  );

         Context ctx = context.strategyContext.get( entry.getKey() );
         if( ctx != null ) {
             Output os = ByteString.newOutput();
             ctx.write( new DroolsObjectOutputStream( os ) );
             _strat.setData( os.toByteString() );
             os.close();
         }
         _header.addStrategy( _strat.build() );
     }
 }
 

public static ByteString getKey(ByteString scriptHash, Sha256Hash tx_id, int idx)
{
  try
  {
    ByteString.Output key_out = ByteString.newOutput(32+32+4);

    key_out.write(scriptHash.toByteArray());
    key_out.write(tx_id.getBytes());

    ByteBuffer bb = ByteBuffer.allocate(4);
    bb.order(java.nio.ByteOrder.LITTLE_ENDIAN);
    bb.putInt(idx);
    key_out.write(bb.array());
    return key_out.toByteString();
  }
  catch(java.io.IOException e)
  {
    throw new RuntimeException(e);
  }

}
 

private ByteString getDBKey(ByteString key)
{
  try
  {
    ByteString.Output out = ByteString.newOutput(100);
    out.write(name_bytes);
    out.write(key.toByteArray());
    out.write(sep);
    ByteString w = out.toByteString();
    return w;
  }
  catch(java.io.IOException e)
  {
    throw new RuntimeException(e);
  }
}
 

private void maybeStartCompression(int additionalBytes) throws IOException {
  if (!compressionEnabled) {
    return;
  }

  if (compressed) {
    return;
  }

  if (bytesWritten + additionalBytes < COMPRESSION_THRESHOLD) {
    return;
  }

  ByteString.Output compressedBytesOut = ByteString.newOutput();
  GZIPOutputStream gzipOut = new GZIPOutputStream(compressedBytesOut);
  bytesOut.writeTo(gzipOut);
  bytesOut = compressedBytesOut;
  out = gzipOut;
  compressed = true;
}
 

@Private
public static ByteString compressByteArrayToByteString(byte[] inBytes) throws IOException {
  ByteString.Output os = ByteString.newOutput();
  DeflaterOutputStream compressOs = new DeflaterOutputStream(os, new Deflater(
      Deflater.BEST_COMPRESSION));
  compressOs.write(inBytes);
  compressOs.finish();
  ByteString byteString = os.toByteString();
  return byteString;
}
 

public static ByteString toProtobuf(BytesOutput out) {
  ByteString.Output output = ByteString.newOutput();
  try {
    out.writeTo(output);
    return output.toByteString();
  } catch (IOException e) {
    throw new RuntimeException(e);
  }
}
 

public static <T> ByteString toBytes(SerializationContext serializationContext, T value)
    throws IOException, SerializationException {
  ByteString.Output output = ByteString.newOutput();
  CodedOutputStream codedOut = CodedOutputStream.newInstance(output);
  serializationContext.serialize(value, codedOut);
  codedOut.flush();
  return output.toByteString();
}
 

@Override
public ByteString getBytes() throws IOException {
  ByteString.Output out = ByteString.newOutput(size);
  try (GZIPInputStream gzipIn = new GZIPInputStream(compressedData.newInput())) {
    ByteStreams.copy(gzipIn, out);
  }
  return out.toByteString();
}
 

/**
 * Convert a Configuration to compressed ByteString using Protocol buffer
 *
 * @param conf
 *          : Configuration to be converted
 * @return PB ByteString (compressed)
 * @throws java.io.IOException
 */
public static ByteString createByteStringFromConf(Configuration conf) throws IOException {
  Objects.requireNonNull(conf, "Configuration must be specified");
  ByteString.Output os = ByteString.newOutput();
  SnappyOutputStream compressOs = new SnappyOutputStream(os);
  try {
    writeConfInPB(compressOs, conf);
  } finally {
    if (compressOs != null) {
      compressOs.close();
    }
  }
  return os.toByteString();
}
 

@Override
public void serialize(
    SerializationContext context,
    OptionsDiffForReconstruction diff,
    CodedOutputStream codedOut)
    throws SerializationException, IOException {
  OptionsDiffCache cache = context.getDependency(OptionsDiffCache.class);
  ByteString bytes = cache.getBytesFromOptionsDiff(diff);
  if (bytes == null) {
    context = context.getNewNonMemoizingContext();
    ByteString.Output byteStringOut = ByteString.newOutput();
    CodedOutputStream bytesOut = CodedOutputStream.newInstance(byteStringOut);
    context.serialize(diff.differingOptions, bytesOut);
    context.serialize(diff.extraFirstFragmentClasses, bytesOut);
    context.serialize(diff.extraSecondFragments, bytesOut);
    bytesOut.writeByteArrayNoTag(diff.baseFingerprint);
    context.serialize(diff.checksum, bytesOut);
    context.serialize(diff.differingStarlarkOptions, bytesOut);
    context.serialize(diff.extraFirstStarlarkOptions, bytesOut);
    context.serialize(diff.extraSecondStarlarkOptions, bytesOut);
    bytesOut.flush();
    byteStringOut.flush();
    int optionsDiffSize = byteStringOut.size();
    bytes = byteStringOut.toByteString();
    cache.putBytesFromOptionsDiff(diff, bytes);
    logger.atFine().log(
        "Serialized OptionsDiffForReconstruction %s. Diff took %d bytes.",
        diff, optionsDiffSize);
  }
  codedOut.writeBytesNoTag(bytes);
}
 

public static ByteString toBytesMemoized(Object original, ObjectCodecRegistry registry)
    throws IOException, SerializationException {
  ByteString.Output output = ByteString.newOutput();
  CodedOutputStream codedOut = CodedOutputStream.newInstance(output);
  new ObjectCodecs(registry).serializeMemoized(original, codedOut);
  codedOut.flush();
  return output.toByteString();
}
 

private static ByteString serializeToByteString(Object subject, SerializeCall wrapped)
    throws SerializationException {
  ByteString.Output resultOut = ByteString.newOutput();
  CodedOutputStream codedOut = CodedOutputStream.newInstance(resultOut);
  wrapped.serialize(subject, codedOut);
  try {
    codedOut.flush();
    return resultOut.toByteString();
  } catch (IOException e) {
    throw new SerializationException("Failed to serialize " + subject, e);
  }
}
 

Builder() {
  out = ByteString.newOutput(64*1024);
  cos = CodedOutputStream.newInstance(out);
}
 

@Override
public ByteString getBytes() throws IOException {
  ByteString.Output out = ByteString.newOutput();
  writeTo(out);
  return out.toByteString();
}
 

public ByteStreamServiceWriter(
    String resourceName, SettableFuture<ByteString> content, int initialCapacity) {
  this.resourceName = resourceName;
  this.content = content;
  out = ByteString.newOutput(initialCapacity);
}