Java Code Examples for com.google.protobuf.CodedInputStream#newInstance()

/** Verify that the data is aliased when possible. */
@Test
@SuppressWarnings("boxing")
public void parseData_AliasesArray() throws IOException {
  byte[] bytes = build(HyperLogLogPlusUniqueStateProto.newBuilder()
      .setData(ByteString.copyFrom(new byte[] {1, 2, 3})));
  CodedInputStream stream = CodedInputStream.newInstance(bytes);
  stream.enableAliasing(true);
  State state = new State();
  state.parse(stream);

  // Preconditions.
  int idx = Bytes.indexOf(bytes, new byte[] {1, 2, 3});
  assertThat(idx).isAtLeast(0);
  assertThat(state.data.toByteArray()).isEqualTo(new byte[] {1, 2, 3});

  // Modify the bytes, make sure that it is reflected in builder.
  bytes[idx + 1] = 4;
  assertThat(state.data.toByteArray()).isEqualTo(new byte[] {1, 4, 3});
}
 

private void loadFromBuffer(ByteBuffer buffer) {
    final String MESSAGE_NAME = AISProtobuf.AkibanInformationSchema.getDescriptor().getFullName();
    checkBuffer(buffer);
    final int serializedSize = buffer.getInt();
    final int initialPos = buffer.position();
    final int bufferSize = buffer.limit() - initialPos;
    if(bufferSize < serializedSize) {
        throw new ProtobufReadException(MESSAGE_NAME, "Buffer corrupt, serialized size greater than remaining");
    }
    CodedInputStream codedInput = CodedInputStream.newInstance(buffer.array(), buffer.position(), Math.min(serializedSize, bufferSize));
    try {
        pbAISBuilder.mergeFrom(codedInput, storageFormatRegistry.getExtensionRegistry());
        // Successfully consumed, update byte buffer
        buffer.position(initialPos + serializedSize);
    } catch(IOException e) {
        // CodedInputStream really only throws InvalidProtocolBufferException, but declares IOE
        throw new ProtobufReadException(MESSAGE_NAME, e.getMessage());
    }
}
 

@Override
public <T> T decodeTime(byte[] bytes, int offset, int length, ValueFactory<T> vf) {
    try {
        CodedInputStream inputStream = CodedInputStream.newInstance(bytes, offset, length);
        boolean negative = inputStream.readRawByte() > 0;
        int hours = 0;
        int minutes = 0;
        int seconds = 0;

        int nanos = 0;

        if (!inputStream.isAtEnd()) {
            hours = (int) inputStream.readInt64();
            if (!inputStream.isAtEnd()) {
                minutes = (int) inputStream.readInt64();
                if (!inputStream.isAtEnd()) {
                    seconds = (int) inputStream.readInt64();
                    if (!inputStream.isAtEnd()) {
                        nanos = 1000 * (int) inputStream.readInt64();
                    }
                }
            }
        }

        return vf.createFromTime(negative ? -1 * hours : hours, minutes, seconds, nanos);
    } catch (IOException e) {
        throw new DataReadException(e);
    }
}
 

public BluefloodGaugeRollup deserialize(ByteBuffer byteBuffer) {
    CodedInputStream in = CodedInputStream.newInstance(byteBuffer.array());
    try {
        byte version = in.readRawByte();
        if (version != VERSION_1_ROLLUP)
            throw new SerializationException(String.format("Unexpected gauge deserialization version: %d", (int)version));
        return deserializeGauge( in, version );
    } catch (Exception e) {
        throw new RuntimeException("Gauge deserialization Failure", e);
    }
}
 

@Override
public void readFields(DataInput in) throws IOException {
  if (din == null) {
    din = new DataInputStream();
    cin = CodedInputStream.newInstance(din);
    cin.setSizeLimit(Integer.MAX_VALUE);
  }
  din.in = in;
  message = cin.readMessage(parser, null);
}
 

public long extractTimestampMillis(String topic, final byte[] bytes) throws IOException {
    if (timestampFieldPath != null) {
        com.google.protobuf.Message decodedMessage = protobufUtil.decodeProtobufOrJsonMessage(topic,
                bytes);
        int i = 0;
        for (; i < timestampFieldPath.length - 1; ++i) {
            decodedMessage = (com.google.protobuf.Message) decodedMessage
                    .getField(decodedMessage.getDescriptorForType().findFieldByName(timestampFieldPath[i]));
        }
        Object timestampObject = decodedMessage
                .getField(decodedMessage.getDescriptorForType().findFieldByName(timestampFieldPath[i]));
        if (timestampObject instanceof com.google.protobuf.Timestamp){
            return Timestamps.toMillis((com.google.protobuf.Timestamp) timestampObject);
        }else {
            return toMillis((Long) timestampObject);
        }
    } else {
        // Assume that the timestamp field is the first field, is required,
        // and is a uint64.

        CodedInputStream input = CodedInputStream.newInstance(bytes);
        // Don't really care about the tag, but need to read it to get, to
        // the payload.
        input.readTag();
        return toMillis(input.readUInt64());
    }
}
 

private HistoryEvent decodeHistoryEvent(String eventClass, String base64)
    throws IOException {
  CodedInputStream in = CodedInputStream.newInstance(Base64.decodeBase64(base64));
  try {
    HistoryEvent event = ReflectionUtils.createClazzInstance(eventClass);
    event.fromProtoStream(in);
    return event;
  } catch (TezReflectionException e) {
    throw new IOException(e);
  }
}
 

@Override
public <T> T decodeTimestamp(byte[] bytes, int offset, int length, ValueFactory<T> vf) {
    try {
        CodedInputStream inputStream = CodedInputStream.newInstance(bytes, offset, length);
        int year = (int) inputStream.readUInt64();
        int month = (int) inputStream.readUInt64();
        int day = (int) inputStream.readUInt64();

        // do we have a time too?
        if (inputStream.getBytesUntilLimit() > 0) {
            int hours = 0;
            int minutes = 0;
            int seconds = 0;

            int nanos = 0;

            if (!inputStream.isAtEnd()) {
                hours = (int) inputStream.readInt64();
                if (!inputStream.isAtEnd()) {
                    minutes = (int) inputStream.readInt64();
                    if (!inputStream.isAtEnd()) {
                        seconds = (int) inputStream.readInt64();
                        if (!inputStream.isAtEnd()) {
                            nanos = 1000 * (int) inputStream.readInt64();
                        }
                    }
                }
            }

            return vf.createFromTimestamp(year, month, day, hours, minutes, seconds, nanos);
        }
        return vf.createFromDate(year, month, day);
    } catch (IOException e) {
        throw new DataReadException(e);
    }
}
 

@Override
public <T> T decodeTime(byte[] bytes, int offset, int length, ValueFactory<T> vf) {
    try {
        CodedInputStream inputStream = CodedInputStream.newInstance(bytes, offset, length);
        boolean negative = inputStream.readRawByte() > 0;
        int hours = 0;
        int minutes = 0;
        int seconds = 0;

        int nanos = 0;

        if (!inputStream.isAtEnd()) {
            hours = (int) inputStream.readInt64();
            if (!inputStream.isAtEnd()) {
                minutes = (int) inputStream.readInt64();
                if (!inputStream.isAtEnd()) {
                    seconds = (int) inputStream.readInt64();
                    if (!inputStream.isAtEnd()) {
                        nanos = 1000 * (int) inputStream.readInt64();
                    }
                }
            }
        }

        return vf.createFromTime(negative ? -1 * hours : hours, minutes, seconds, nanos);
    } catch (IOException e) {
        throw new DataReadException(e);
    }
}
 

/**
 * Cheap test to see if input stream is a wallet. This checks for a magic value at the beginning of the stream.
 *
 * @param is
 *            input stream to test
 * @return true if input stream is a wallet
 */
public static boolean isWallet(InputStream is) {
    try {
        final CodedInputStream cis = CodedInputStream.newInstance(is);
        final int tag = cis.readTag();
        final int field = WireFormat.getTagFieldNumber(tag);
        if (field != 1) // network_identifier
            return false;
        final String network = cis.readString();
        return NetworkParameters.fromID(network) != null;
    } catch (IOException x) {
        return false;
    }
}
 

@Override
public <T> T decodeByteArray(byte[] bytes, int offset, int length, ValueFactory<T> vf) {
    try {
        CodedInputStream inputStream = CodedInputStream.newInstance(bytes, offset, length);
        // c.f. Streaming_command_delegate::get_string()
        int size = inputStream.getBytesUntilLimit();
        size--; // for null terminator
        return vf.createFromBytes(inputStream.readRawBytes(size), 0, size);
    } catch (IOException e) {
        throw new DataReadException(e);
    }
}
 

public String deserialize(ByteBuffer byteBuffer) {
    CodedInputStream is = CodedInputStream.newInstance(byteBuffer.array());
    try {
        byte type = is.readRawByte();
        if (type == STRING) {
            return is.readString();
        } else {
            throw new IOException("Unexpected first byte. Expected '4' (string). Got '" + type + "'.");
        }
    } catch (IOException e) {
        throw new RuntimeException("IOException during deserialization", e);
    }
}
 

public static ExtraActionInfo getExtraActionInfo(String extraActionFile) {
  ExtensionRegistry registry = ExtensionRegistry.newInstance();
  ExtraActionsBase.registerAllExtensions(registry);

  try (InputStream stream = Files.newInputStream(Paths.get(extraActionFile))) {
    CodedInputStream coded = CodedInputStream.newInstance(stream);
    return ExtraActionInfo.parseFrom(coded, registry);
  } catch (IOException e) {
    throw new RuntimeException("ERROR: failed to deserialize extra action file "
      + extraActionFile + ": " + e.getMessage(), e);
  }
}
 

@Override
public long keyToStartTimestamp(byte[] key) throws IOException {
    CodedInputStream cis = CodedInputStream.newInstance(key);
    return Long.reverse(cis.readSFixed64());
}
 

@Override
public <T> T decodeDecimal(byte[] bytes, int offset, int length, ValueFactory<T> vf) {
    try {
        CodedInputStream inputStream = CodedInputStream.newInstance(bytes, offset, length);
        // packed BCD format (c.f. wikipedia)
        // TODO: optimization possibilities include using int/long if the digits is < X and scale = 0
        byte scale = inputStream.readRawByte();
        // we allocate an extra char for the sign
        CharBuffer unscaledString = CharBuffer.allocate(2 * inputStream.getBytesUntilLimit());
        unscaledString.position(1);
        byte sign = 0;
        // read until we encounter the sign bit
        while (true) {
            int b = 0xFF & inputStream.readRawByte();
            if ((b >> 4) > 9) {
                sign = (byte) (b >> 4);
                break;
            }
            unscaledString.append((char) ((b >> 4) + '0'));
            if ((b & 0x0f) > 9) {
                sign = (byte) (b & 0x0f);
                break;
            }
            unscaledString.append((char) ((b & 0x0f) + '0'));
        }
        if (inputStream.getBytesUntilLimit() > 0) {
            throw AssertionFailedException
                    .shouldNotHappen("Did not read all bytes while decoding decimal. Bytes left: " + inputStream.getBytesUntilLimit());
        }
        switch (sign) {
            case 0xa:
            case 0xc:
            case 0xe:
            case 0xf:
                unscaledString.put(0, '+');
                break;
            case 0xb:
            case 0xd:
                unscaledString.put(0, '-');
                break;
        }
        // may have filled the CharBuffer or one remaining. need to remove it before toString()
        int characters = unscaledString.position();
        unscaledString.clear(); // reset position
        BigInteger unscaled = new BigInteger(unscaledString.subSequence(0, characters).toString());
        return vf.createFromBigDecimal(new BigDecimal(unscaled, scale));
    } catch (IOException e) {
        throw new DataReadException(e);
    }
}
 

public static ArrayList<ByteString> extractWireFormatTxOut(Transaction tx)
  throws ValidationException
{
  try
  {
    CodedInputStream code_in = CodedInputStream.newInstance(tx.getInnerData().toByteArray());

    ArrayList<ByteString> lst = new ArrayList<>();

    while(true)
    {
      int tag = code_in.readTag();
      if (tag == 0) break;
      
      // The least signficiate 3 bits are the proto field type
      // so shift to get out field number, which is 5 for TranasctionOutput
      if (tag >> 3 == 5)
      {
        ByteArrayOutputStream b_out = new ByteArrayOutputStream();
        CodedOutputStream c_out = CodedOutputStream.newInstance(b_out);
        code_in.skipField(tag, c_out);
        c_out.flush();

        ByteString bs = ByteString.copyFrom(b_out.toByteArray());

        // So funny story...when you get an inner message like this as opposed to just serializing
        // the object, protobuf puts a tag and size on the coded input stream.  So we need to figure
        // out how many bytes that is an trim it off.
        CodedInputStream read_again = CodedInputStream.newInstance(bs.toByteArray());
        // Expected tag
        int tag2 = read_again.readTag();
        // Size of element
        int size = read_again.readInt32();

        // All we really care is how many bytes those two take.  For shorter messages
        // it will be 2, but could be higher if protobuf needs more bytes to encode the size
        int offset = read_again.getTotalBytesRead();

        bs = bs.substring(offset);
        lst.add(bs);
      }
      else
      {
        code_in.skipField(tag);
      }

    }
    return lst;
  }
  catch(java.io.IOException e)
  {
    throw new ValidationException(e);
  }
}
 

@NonNull
@Override
public FileDataSource loadData(InputStream inputStream, String filePath) throws Exception {
    long propertiesOffset = 0L;
    Track track = new Track();
    CodedInputStream input = CodedInputStream.newInstance(inputStream);
    boolean done = false;
    while (!done) {
        long offset = input.getTotalBytesRead();
        int tag = input.readTag();
        int field = WireFormat.getTagFieldNumber(tag);
        switch (field) {
            case 0:
                done = true;
                break;
            default: {
                throw new com.google.protobuf.InvalidProtocolBufferException("Unsupported proto field: " + tag);
            }
            case FIELD_VERSION: {
                // skip version
                input.skipField(tag);
                break;
            }
            case FIELD_POINT: {
                int length = input.readRawVarint32();
                int oldLimit = input.pushLimit(length);
                readPoint(track, input);
                input.popLimit(oldLimit);
                input.checkLastTagWas(0);
                break;
            }
            case FIELD_NAME: {
                propertiesOffset = offset;
                track.name = input.readBytes().toStringUtf8();
                break;
            }
            case FIELD_COLOR: {
                track.style.color = input.readUInt32();
                break;
            }
            case FIELD_WIDTH: {
                track.style.width = input.readFloat();
                break;
            }
        }
    }
    inputStream.close();
    track.id = 31 * filePath.hashCode() + 1;
    FileDataSource dataSource = new FileDataSource();
    dataSource.name = track.name;
    dataSource.tracks.add(track);
    track.source = dataSource;
    dataSource.propertiesOffset = propertiesOffset;
    return dataSource;
}
 

public static void varintFirst(File inFile, File outFile) throws Exception {
    FileInputStream fis = new FileInputStream(inFile);
    CodedInputStream cis = CodedInputStream.newInstance(fis);
    FileUtils.writeByteArrayToFile(outFile, cis.readRawBytes(cis.readRawVarint32()));
}
 

@Override
public boolean preload(SourceLoader sourceLoader) {
  SourceLibrary library = sourceLoader.getLibrary();
  ModulePath modulePath = getModulePath();
  ChildGroup group = null;
  try (InputStream inputStream = getInputStream()) {
    if (inputStream == null) {
      return false;
    }

    CodedInputStream codedInputStream = CodedInputStream.newInstance(inputStream);
    codedInputStream.setRecursionLimit(Integer.MAX_VALUE);
    ModuleProtos.Module moduleProto = ModuleProtos.Module.parseFrom(codedInputStream);
    boolean isComplete = moduleProto.getComplete();
    if (!isComplete && !library.hasRawSources()) {
      sourceLoader.getLibraryErrorReporter().report(new PartialModuleError(modulePath));
      return false;
    }

    for (ModuleProtos.ModuleCallTargets moduleCallTargets : moduleProto.getModuleCallTargetsList()) {
      ModulePath module = new ModulePath(moduleCallTargets.getNameList());
      if (library.containsModule(module) && !sourceLoader.preloadBinary(module, myKeyRegistry, myDefinitionListener)) {
        return false;
      }
    }

    ReferableConverter referableConverter = sourceLoader.getReferableConverter();
    myModuleDeserialization = new ModuleDeserialization(moduleProto, library.getTypecheckerState(), referableConverter, myKeyRegistry, myDefinitionListener);

    if (referableConverter == null) {
      group = myModuleDeserialization.readGroup(new ModuleLocation(library.getName(), ModuleLocation.LocationKind.SOURCE, modulePath));
      library.groupLoaded(modulePath, group, false, false);
    } else {
      group = library.getModuleGroup(modulePath, false);
      if (group == null) {
        sourceLoader.getLibraryErrorReporter().report(LibraryError.moduleNotFound(modulePath, library.getName()));
        library.groupLoaded(modulePath, null, false, false);
        return false;
      }
      myModuleDeserialization.readDefinitions(group);
    }

    return true;
  } catch (IOException | DeserializationException e) {
    loadingFailed(sourceLoader, modulePath, group, e);
    return false;
  }
}
 

private void deserializeEvent(DataInput in) throws IOException {
  if (!in.readBoolean()) {
    event = null;
    return;
  }
  eventType = EventType.values()[in.readInt()];
  eventReceivedTime = in.readLong();
  if (eventType.equals(EventType.TASK_STATUS_UPDATE_EVENT)) {
    // TODO NEWTEZ convert to PB
    event = new TaskStatusUpdateEvent();
    ((TaskStatusUpdateEvent)event).readFields(in);
  } else {
    int eventBytesLen = in.readInt();
    byte[] eventBytes;
    CodedInputStream input;
    int startOffset = 0;
    if (in instanceof DataInputBuffer) {
      eventBytes = ((DataInputBuffer)in).getData();
      startOffset = ((DataInputBuffer) in).getPosition();
    } else {
      eventBytes = new byte[eventBytesLen];
      in.readFully(eventBytes);
    }
    input = CodedInputStream.newInstance(eventBytes, startOffset, eventBytesLen);
    switch (eventType) {
    case CUSTOM_PROCESSOR_EVENT:
      CustomProcessorEventProto cpProto =
          CustomProcessorEventProto.parseFrom(input);
      event = ProtoConverters.convertCustomProcessorEventFromProto(cpProto);
      break;
    case DATA_MOVEMENT_EVENT:
      DataMovementEventProto dmProto =
          DataMovementEventProto.parseFrom(input);
      event = ProtoConverters.convertDataMovementEventFromProto(dmProto);
      break;
    case COMPOSITE_ROUTED_DATA_MOVEMENT_EVENT:
      CompositeRoutedDataMovementEventProto edmProto =
          CompositeRoutedDataMovementEventProto.parseFrom(eventBytes);
    event = ProtoConverters.convertCompositeRoutedDataMovementEventFromProto(edmProto);
    break;
    case COMPOSITE_DATA_MOVEMENT_EVENT:
      CompositeEventProto cProto = CompositeEventProto.parseFrom(input);
      event = ProtoConverters.convertCompositeDataMovementEventFromProto(cProto);
      break;
    case VERTEX_MANAGER_EVENT:
      VertexManagerEventProto vmProto = VertexManagerEventProto.parseFrom(input);
      event = ProtoConverters.convertVertexManagerEventFromProto(vmProto);
      break;
    case INPUT_READ_ERROR_EVENT:
      InputReadErrorEventProto ideProto =
          InputReadErrorEventProto.parseFrom(input);
      event = InputReadErrorEvent.create(ideProto.getDiagnostics(),
          ideProto.getIndex(), ideProto.getVersion());
      break;
    case TASK_ATTEMPT_FAILED_EVENT:
      TaskAttemptFailedEventProto tfProto =
          TaskAttemptFailedEventProto.parseFrom(input);
      event = new TaskAttemptFailedEvent(tfProto.getDiagnostics(),
          TezConverterUtils.failureTypeFromProto(tfProto.getTaskFailureType()));
      break;
    case TASK_ATTEMPT_KILLED_EVENT:
      TaskAttemptKilledEventProto tkProto = TaskAttemptKilledEventProto.parseFrom(input);
      event = new TaskAttemptKilledEvent(tkProto.getDiagnostics());
      break;
    case TASK_ATTEMPT_COMPLETED_EVENT:
      event = new TaskAttemptCompletedEvent();
      break;
    case INPUT_FAILED_EVENT:
      InputFailedEventProto ifProto =
          InputFailedEventProto.parseFrom(input);
      event = InputFailedEvent.create(ifProto.getTargetIndex(), ifProto.getVersion());
      break;
    case ROOT_INPUT_DATA_INFORMATION_EVENT:
      RootInputDataInformationEventProto difProto = RootInputDataInformationEventProto
          .parseFrom(input);
      event = ProtoConverters.convertRootInputDataInformationEventFromProto(difProto);
      break;
    case ROOT_INPUT_INITIALIZER_EVENT:
      EventProtos.RootInputInitializerEventProto riiProto = EventProtos.RootInputInitializerEventProto.parseFrom(input);
      event = ProtoConverters.convertRootInputInitializerEventFromProto(riiProto);
      break;
    default:
      // RootInputUpdatePayload event not wrapped in a TezEvent.
      throw new TezUncheckedException("Unexpected TezEvent"
         + ", type=" + eventType);
    }
    if (in instanceof DataInputBuffer) {
      // Skip so that position is updated
      int skipped = in.skipBytes(eventBytesLen);
      if (skipped != eventBytesLen) {
        throw new TezUncheckedException("Expected to skip " + eventBytesLen + " bytes. Actually skipped = " + skipped);
      }
    }
  }
}