com.google.protobuf.CodedOutputStream Java Examples

@Test
public void parseUnknownField() throws IOException {
  // Create an aggregator state proto with an unknown field in the middle.
  ByteArrayOutputStream stream = new ByteArrayOutputStream();
  CodedOutputStream coded = CodedOutputStream.newInstance(stream);
  coded.writeInt32(AggregatorStateProto.NUM_VALUES_FIELD_NUMBER, 42);
  coded.writeString(999, "foobar");
  coded.writeInt32(AggregatorStateProto.ENCODING_VERSION_FIELD_NUMBER, 43);
  coded.flush();

  // Check that we can parse the proto, despite the unknown field.
  State state = new State();
  state.parse(CodedInputStream.newInstance(stream.toByteArray()));

  // Check that the fields before and after the unknown fields were correctly read.
  assertEquals(42, state.numValues);
  assertEquals(43, state.encodingVersion);
}
 

@Override
public void serialize(
    SerializationContext context, Multimap<K, V> obj, CodedOutputStream codedOut)
    throws SerializationException, IOException {
  if (obj instanceof ListMultimap) {
    codedOut.writeBoolNoTag(true);
  } else if (obj instanceof SetMultimap) {
    codedOut.writeBoolNoTag(false);
  } else {
    throw new SerializationException("Unexpected multimap type: " + obj.getClass());
  }
  codedOut.writeInt32NoTag(obj.asMap().size());
  for (Map.Entry<K, Collection<V>> entry : obj.asMap().entrySet()) {
    context.serialize(entry.getKey(), codedOut);
    context.serialize(entry.getValue(), codedOut);
  }
}
 

@Override
public ByteBuf compressInput(Object proto, RpcMethodInfo rpcMethodInfo) throws IOException {
    int protoSize = rpcMethodInfo.getInputSerializedSize(proto);
    ByteBuf resBuf = Unpooled.buffer(protoSize);
    OutputStream outputStream = new ByteBufOutputStream(resBuf);
    GZIPOutputStream gzipOutputStream = new GZIPOutputStream(outputStream);

    if (protoSize > CodedOutputStream.DEFAULT_BUFFER_SIZE) {
        protoSize = CodedOutputStream.DEFAULT_BUFFER_SIZE;
    }
    final CodedOutputStream codedOutputStream =
            CodedOutputStream.newInstance(gzipOutputStream, protoSize);
    rpcMethodInfo.inputWriteToStream(proto, codedOutputStream);
    gzipOutputStream.close();
    return resBuf;
}
 

/**
 * Compute object size no tag.
 *
 * @param o the o
 * @return the int
 */
public static int computeObjectSizeNoTag(Object o) {
    int size = 0;
    if (o == null) {
        return size;
    }

    Class cls = o.getClass();
    Codec target = ProtobufProxy.create(cls);
    try {
        size = target.size(o);
        size = size + CodedOutputStream.computeRawVarint32Size(size);
        return size;
    } catch (IOException e) {
        throw new RuntimeException(e.getMessage(), e);
    }
}
 

@Test
public void serializeToBinary() {
  ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
  CodedOutputStream codedOutputStream = CodedOutputStream.newInstance(outputStream);
  try {

    codedOutputStream.writeStringNoTag("jc");
    codedOutputStream.writeStringNoTag("dan");
    codedOutputStream.writeInt32NoTag(1);
    codedOutputStream.writeStringNoTag("abc");
    codedOutputStream.writeStringNoTag("");
    codedOutputStream.writeByteArrayNoTag(NumericUtil.hexToBytes("0f0f0f"));
    codedOutputStream.flush();
    ByteBuffer byteBuffer = ByteBuffer.allocate(100);

  } catch (IOException e) {
    e.printStackTrace();
  }
}
 

private HistoryEvent testProtoConversion(HistoryEvent event) throws IOException, TezException {
  ByteArrayOutputStream os = new ByteArrayOutputStream();
  HistoryEvent deserializedEvent = null;
  CodedOutputStream codedOutputStream = CodedOutputStream.newInstance(os);
  event.toProtoStream(codedOutputStream);
  codedOutputStream.flush();
  os.flush();
  os.close();
  deserializedEvent = ReflectionUtils.createClazzInstance(
      event.getClass().getName());
  LOG.info("Serialized event to byte array"
      + ", eventType=" + event.getEventType()
      + ", bufLen=" + os.toByteArray().length);
  deserializedEvent.fromProtoStream(
      CodedInputStream.newInstance(os.toByteArray()));
  return deserializedEvent;
}
 

@Nullable
private ObjectCodecRegistry.CodecDescriptor recordAndGetDescriptorIfNotConstantMemoizedOrNull(
    @Nullable Object object, CodedOutputStream codedOut) throws IOException, NoCodecException {
  if (writeNullOrConstant(object, codedOut)) {
    return null;
  }
  if (serializer != null) {
    Integer memoizedIndex = serializer.getMemoizedIndex(object);
    if (memoizedIndex != null) {
      // Subtract 1 so it will be negative and not collide with null.
      codedOut.writeSInt32NoTag(-memoizedIndex - 1);
      return null;
    }
  }
  ObjectCodecRegistry.CodecDescriptor descriptor = registry.getCodecDescriptorForObject(object);
  codedOut.writeSInt32NoTag(descriptor.getTag());
  return descriptor;
}
 

/**
 * Saves track properties by modifying only file tail.
 */
public void saveProperties(FileDataSource source) throws Exception {
    Track track = source.tracks.get(0);
    // Prepare new properties tail
    ByteBuffer buffer = ByteBuffer.allocate(getSerializedPropertiesSize(track));
    CodedOutputStream output = CodedOutputStream.newInstance(buffer);
    output.writeBytes(FIELD_NAME, ByteString.copyFromUtf8(track.name));
    output.writeUInt32(FIELD_COLOR, track.style.color);
    output.writeFloat(FIELD_WIDTH, track.style.width);
    output.flush();
    // Modify tail of file
    File file = new File(source.path);
    long createTime = file.lastModified();
    RandomAccessFile access = new RandomAccessFile(file, "rw");
    access.setLength(source.propertiesOffset + 1);
    access.seek(source.propertiesOffset);
    access.write(buffer.array());
    access.close();
    //noinspection ResultOfMethodCallIgnored
    file.setLastModified(createTime);
}
 

public PhysicalWriterImpl(FSDataOutputStream out, OrcFile.WriterOptions opts) throws IOException {
	if (opts.isEnforceBufferSize()) {
		this.bufferSize = opts.getBufferSize();
	} else {
		this.bufferSize = getEstimatedBufferSize(
			opts.getStripeSize(), opts.getSchema().getMaximumId() + 1, opts.getBufferSize());
	}

	this.out = out;
	this.blockOffset = 0;
	this.blockSize = opts.getBlockSize();
	this.maxPadding = (int) (opts.getPaddingTolerance() * (double) opts.getBufferSize());
	this.compress = opts.getCompress();
	this.codec = OrcCodecPool.getCodec(this.compress);
	this.streams  = new TreeMap<>();
	this.writer = new OutStream("metadata", this.bufferSize, this.codec, new DirectStream(this.out));
	this.shims = opts.getHadoopShims();
	this.addBlockPadding = opts.getBlockPadding();
	this.protobufWriter = CodedOutputStream.newInstance(this.writer);
	this.writeVariableLengthBlocks = opts.getWriteVariableLengthBlocks();
}
 

private void sendSaslMessage(ChannelHandlerContext ctx, byte[] payload,
    List<CipherOption> options) throws IOException {
  DataTransferEncryptorMessageProto.Builder builder =
      DataTransferEncryptorMessageProto.newBuilder();
  builder.setStatus(DataTransferEncryptorStatus.SUCCESS);
  if (payload != null) {
    BuilderPayloadSetter.wrapAndSetPayload(builder, payload);
  }
  if (options != null) {
    builder.addAllCipherOption(PBHelperClient.convertCipherOptions(options));
  }
  DataTransferEncryptorMessageProto proto = builder.build();
  int size = proto.getSerializedSize();
  size += CodedOutputStream.computeRawVarint32Size(size);
  ByteBuf buf = ctx.alloc().buffer(size);
  proto.writeDelimitedTo(new ByteBufOutputStream(buf));
  ctx.write(buf);
}
 

public int getSerializedSize() {
  int size = 0;

  size += CodedOutputStream.computeUInt32SizeNoTag(TYPE_TAG);
  size += CodedOutputStream.computeEnumSizeNoTag(type.getNumber());

  size += CodedOutputStream.computeUInt32SizeNoTag(NUM_VALUES_TAG);
  size += CodedOutputStream.computeInt64SizeNoTag(numValues);

  if (encodingVersion != DEFAULT_ENCODING_VERSION) {
    size += CodedOutputStream.computeUInt32SizeNoTag(ENCODING_VERSION_TAG);
    size += CodedOutputStream.computeInt32SizeNoTag(encodingVersion);
  }

  if (!valueType.equals(DEFAULT_VALUE_TYPE)) {
    size += CodedOutputStream.computeUInt32SizeNoTag(VALUE_TYPE_TAG);
    size += CodedOutputStream.computeEnumSizeNoTag(valueType.getNumber());
  }

  int hllSize = getSerializedHllSize();
  size += CodedOutputStream.computeUInt32SizeNoTag(HYPERLOGLOGPLUS_UNIQUE_STATE_TAG);
  size += CodedOutputStream.computeUInt32SizeNoTag(hllSize);
  size += hllSize;

  return size;
}
 

public void writeTo(CodedOutputStream stream) throws IOException {
  // We use the NoTag write methods for consistency with the parsing functions and for
  // consistency with the variable-length writes where we can't use any convenience function.
  stream.writeUInt32NoTag(TYPE_TAG);
  stream.writeEnumNoTag(type.getNumber());

  stream.writeUInt32NoTag(NUM_VALUES_TAG);
  stream.writeInt64NoTag(numValues);

  if (encodingVersion != DEFAULT_ENCODING_VERSION) {
    stream.writeUInt32NoTag(ENCODING_VERSION_TAG);
    stream.writeInt32NoTag(encodingVersion);
  }

  if (!valueType.equals(DEFAULT_VALUE_TYPE)) {
    stream.writeUInt32NoTag(VALUE_TYPE_TAG);
    stream.writeEnumNoTag(valueType.getNumber());
  }

  stream.writeUInt32NoTag(HYPERLOGLOGPLUS_UNIQUE_STATE_TAG);
  stream.writeUInt32NoTag(getSerializedHllSize());
  writeHllTo(stream);
}
 

static <K, V> void serializeEntries(
    SerializationContext context,
    Iterable<? extends Map.Entry<K, V>> entrySet,
    CodedOutputStream codedOut)
    throws IOException, SerializationException {
  for (Map.Entry<?, ?> entry : entrySet) {
    context.serialize(entry.getKey(), codedOut);
    try {
      context.serialize(entry.getValue(), codedOut);
    } catch (SerializationException | IOException e) {
      throw SerializationException.propagate(
          String.format(
              "Exception while serializing value of type %s for key '%s'",
              entry.getValue().getClass().getName(), entry.getKey()),
          e);
    }
  }
}
 

@Override
public void serialize(SerializationContext context, NestedSet<?> obj, CodedOutputStream codedOut)
    throws SerializationException, IOException {
  context.serialize(obj.getOrder(), codedOut);
  if (obj.isEmpty()) {
    // If the NestedSet is empty, it needs to be assigned to the EMPTY_CHILDREN constant on
    // deserialization.
    codedOut.writeEnumNoTag(NestedSetSize.EMPTY.ordinal());
  } else if (obj.isSingleton()) {
    // If the NestedSet is a singleton, we serialize directly as an optimization.
    codedOut.writeEnumNoTag(NestedSetSize.LEAF.ordinal());
    context.serialize(obj.getChildren(), codedOut);
  } else {
    codedOut.writeEnumNoTag(NestedSetSize.NONLEAF.ordinal());
    context.serialize(obj.getApproxDepth(), codedOut);
    FingerprintComputationResult fingerprintComputationResult =
        nestedSetStore.computeFingerprintAndStore((Object[]) obj.getChildren(), context);
    context.addFutureToBlockWritingOn(fingerprintComputationResult.writeStatus());
    codedOut.writeByteArrayNoTag(fingerprintComputationResult.fingerprint().toByteArray());
  }
  interner.put(new EqualsWrapper(obj), obj);
}
 

@Override
public void serialize(SerializationContext context, StringLiteral lit, CodedOutputStream out)
    throws SerializationException, IOException {
  // Enable de-duplication of strings during encoding.
  // The encoder does not intern and de-duplicate Strings by default,
  // though it does for all other objects;
  // see skyframe.serialization.strings.StringCodec.getStrategy.
  // If that were to change, we could delete StringLiteralCodec.
  // (One wonders why Identifier.name strings are not similarly de-duped,
  // as they are as numerous and more repetitive than string literals.)
  context.serializeWithAdHocMemoizationStrategy(
      lit.getValue(), MemoizationStrategy.MEMOIZE_AFTER, out);
  out.writeInt32NoTag(lit.startOffset);
  out.writeInt32NoTag(lit.endOffset);
  context.serialize(lit.locs, out);
}
 

private void serializeCounterRollup(BluefloodCounterRollup rollup, byte[] buf) throws IOException {
    CodedOutputStream out = CodedOutputStream.newInstance(buf);
    counterRollupSize.update(buf.length);
    out.writeRawByte(Constants.VERSION_1_COUNTER_ROLLUP);
    putUnversionedDoubleOrLong(rollup.getCount(), out);
    out.writeDoubleNoTag(rollup.getRate());
    out.writeRawVarint32(rollup.getSampleCount());
}
 

@Override
public void write(DataOutput out) throws IOException {
  if (dos == null) {
    dos = new DataOutputStream();
    cos = CodedOutputStream.newInstance(dos);
  }
  dos.out = out;
  cos.writeMessageNoTag(message);
  cos.flush();
}
 

public static <T> ByteString toBytes(SerializationContext serializationContext, T value)
    throws IOException, SerializationException {
  ByteString.Output output = ByteString.newOutput();
  CodedOutputStream codedOut = CodedOutputStream.newInstance(output);
  serializationContext.serialize(value, codedOut);
  codedOut.flush();
  return output.toByteString();
}
 

private int sizeOf(BluefloodTimerRollup bluefloodTimerRollup, byte timerVersion) throws SerializationException {

        int sz = sizeOfSize();

        if (timerVersion == VERSION_1_TIMER) {
            sz += CodedOutputStream.computeRawVarint64Size((long) bluefloodTimerRollup.getSum());
        } else if (timerVersion == VERSION_2_TIMER) {
            sz += CodedOutputStream.computeDoubleSizeNoTag(bluefloodTimerRollup.getSum());
        } else {
            throw new SerializationException(String.format("Unexpected timer serialization version: %d", (int)timerVersion));
        }
        sz += CodedOutputStream.computeRawVarint64Size(bluefloodTimerRollup.getCount());
        sz += CodedOutputStream.computeDoubleSizeNoTag(bluefloodTimerRollup.getRate());
        sz += CodedOutputStream.computeRawVarint32Size(bluefloodTimerRollup.getSampleCount());
        sz += averageStatDeSer.sizeOf(bluefloodTimerRollup.getAverage());
        sz += maxStatDeSer.sizeOf(bluefloodTimerRollup.getMaxValue());
        sz += minStatDeSer.sizeOf(bluefloodTimerRollup.getMinValue());
        sz += varianceStatDeSer.sizeOf(bluefloodTimerRollup.getVariance());

        Map<String, BluefloodTimerRollup.Percentile> percentiles = bluefloodTimerRollup.getPercentiles();
        sz += CodedOutputStream.computeRawVarint32Size(bluefloodTimerRollup.getPercentiles().size());
        for (Map.Entry<String, BluefloodTimerRollup.Percentile> entry : percentiles.entrySet()) {
            sz += CodedOutputStream.computeStringSizeNoTag(entry.getKey());
            Number[] pctComponents = new Number[] {
                    entry.getValue().getMean(),
            };
            for (Number num : pctComponents) {
                sz += sizeOfType();
                if (num instanceof Long || num instanceof Integer) {
                    sz += CodedOutputStream.computeRawVarint64Size(num.longValue());
                } else if (num instanceof Double || num instanceof Float) {
                    sz += CodedOutputStream.computeDoubleSizeNoTag(num.doubleValue());
                }
            }
        }
        return sz;
    }
 

public static byte[] save(ChangeSet changeSet) {
    ArgumentValidation.notNull("changeSet", changeSet);
    checkFields(changeSet);
    int size = changeSet.getSerializedSize();
    byte[] buffer = new byte[size];
    CodedOutputStream stream = CodedOutputStream.newInstance(buffer);
    try {
        changeSet.writeTo(stream);
    } catch(IOException e) {
        // Only throws OutOfSpace, which shouldn't happen
        throw new IllegalStateException(e);
    }
    return buffer;
}
 

/**
 * Compute the number of bytes that would be needed to encode a single tag/value pair of arbitrary type.
 *
 * @param type The field's type.
 * @param number The field's number.
 * @param value Object representing the field's value. Must be of the exact type which would be returned by
 *            {@link Message#getField(Descriptors.FieldDescriptor)} for this field.
 * @return the int
 */
public static int computeElementSize(final WireFormat.FieldType type, final int number, final Object value) {
    int tagSize = CodedOutputStream.computeTagSize(number);
    if (type == WireFormat.FieldType.GROUP) {
        // Only count the end group tag for proto2 messages as for proto1 the end
        // group tag will be counted as a part of getSerializedSize().
        tagSize *= 2;
    }
    return tagSize + computeElementSizeNoTag(type, value);
}
 

public ByteBuffer serialize(String str) {
    try {
        byte[] buf = new byte[computeBufLength(str)];
        CodedOutputStream out = CodedOutputStream.newInstance(buf);
        writeToOutputStream(str, out);
        return ByteBuffer.wrap(buf);
    } catch (IOException e) {
        throw new RuntimeException("Error serializing string metadata", e);
    }
}
 

public int getSerializedPropertiesSize(Track track) {
    int size = 0;
    size += CodedOutputStream.computeStringSize(FIELD_NAME, track.name);
    size += CodedOutputStream.computeUInt32Size(FIELD_COLOR, track.style.color);
    size += CodedOutputStream.computeFloatSize(FIELD_WIDTH, track.style.width);
    return size;
}
 

@Override
public void serialize(SerializationContext context, Optional<?> obj, CodedOutputStream codedOut)
    throws SerializationException, IOException {
  codedOut.writeBoolNoTag(obj.isPresent());
  if (obj.isPresent()) {
    context.serialize(obj.get(), codedOut);
  }
}
 

private int sizeOf(BluefloodSetRollup setRollup) {
    int sz = sizeOfSize();
    sz += CodedOutputStream.computeRawVarint32Size(setRollup.getCount());
    for (Integer i : setRollup.getHashes()) {
        sz += CodedOutputStream.computeRawVarint32Size(i);
    }
    return sz;
}
 

private static byte[] encodeCommitTimestamp(long startTimestamp, long commitTimestamp) throws IOException {
    assert (startTimestamp < commitTimestamp);
    long diff = commitTimestamp - startTimestamp;
    byte[] bytes = new byte[CodedOutputStream.computeInt64SizeNoTag(diff)];
    CodedOutputStream cos = CodedOutputStream.newInstance(bytes);
    cos.writeInt64NoTag(diff);
    cos.flush();
    return bytes;

}
 

@Override
public byte[] startTimestampToKey(long startTimestamp) throws IOException {
    long reversedStartTimestamp = Long.reverse(startTimestamp);
    byte[] bytes = new byte[CodedOutputStream.computeSFixed64SizeNoTag(reversedStartTimestamp)];
    CodedOutputStream cos = CodedOutputStream.newInstance(bytes);
    cos.writeSFixed64NoTag(reversedStartTimestamp);
    cos.flush();
    return bytes;
}
 

private void doVarIntTest(int value) throws IOException {
  ByteArrayOutputStream baos = new ByteArrayOutputStream();
  CodedOutputStream cout = CodedOutputStream.newInstance(baos);
  cout.writeRawVarint32(value);
  cout.flush();

  DataInputStream dis = new DataInputStream(
      new ByteArrayInputStream(baos.toByteArray()));
  assertEquals(value, ProtoUtil.readRawVarint32(dis));
}
 

@Override
public int getLength() {
  int headerLen = requestHeader.getSerializedSize();
  int reqLen;
  if (theRequest != null) {
    reqLen = theRequest.getSerializedSize();
  } else if (theRequestRead != null ) {
    reqLen = theRequestRead.length;
  } else {
    throw new IllegalArgumentException(
        "getLength on uninitialized RpcWrapper");      
  }
  return CodedOutputStream.computeRawVarint32Size(headerLen) +  headerLen
      + CodedOutputStream.computeRawVarint32Size(reqLen) + reqLen;
}
 

private void serializeTimer(BluefloodTimerRollup rollup, byte[] buf, byte timerVersion) throws IOException {
    CodedOutputStream out = CodedOutputStream.newInstance(buf);
    timerRollupSize.update(buf.length);
    out.writeRawByte(timerVersion);

    // sum, count, countps, avg, max, min, var
    if (timerVersion == VERSION_1_TIMER) {
        out.writeRawVarint64((long)rollup.getSum());
    } else if (timerVersion == VERSION_2_TIMER) {
        out.writeDoubleNoTag(rollup.getSum());
    } else {
        throw new SerializationException(String.format("Unexpected timer serialization version: %d", (int)timerVersion));
    }

    out.writeRawVarint64(rollup.getCount());
    out.writeDoubleNoTag(rollup.getRate());
    out.writeRawVarint32(rollup.getSampleCount());
    putRollupStat(rollup.getAverage(), out);
    putRollupStat(rollup.getMaxValue(), out);
    putRollupStat(rollup.getMinValue(), out);
    putRollupStat(rollup.getVariance(), out);

    // percentiles.
    Map<String, BluefloodTimerRollup.Percentile> percentiles = rollup.getPercentiles();
    out.writeRawVarint32(percentiles.size());
    for (Map.Entry<String, BluefloodTimerRollup.Percentile> entry : percentiles.entrySet()) {
        out.writeStringNoTag(entry.getKey());
        putUnversionedDoubleOrLong(entry.getValue().getMean(), out);
    }
}