Java Code Examples for com.google.protobuf.CodedInputStream#setSizeLimit()

public static List<HistoryEvent> parseDAGRecoveryFile(FSDataInputStream inputStream)
    throws IOException {
  List<HistoryEvent> historyEvents = new ArrayList<HistoryEvent>();
  CodedInputStream codedInputStream = CodedInputStream.newInstance(inputStream);
  codedInputStream.setSizeLimit(Integer.MAX_VALUE);
  while (true) {
    HistoryEvent historyEvent = getNextEvent(codedInputStream);
    if (historyEvent == null) {
      LOG.info("Reached end of stream");
      break;
    }
    LOG.debug("Read HistoryEvent, eventType={}, event={}", historyEvent.getEventType(), historyEvent);
    historyEvents.add(historyEvent);
  }
  return historyEvents;
}
 

/**
 * Actually processes the GTFS-realtime file and calls handleAvlReport()
 * for each AvlReport.
 */
public static Collection<AvlReport> process(InputStream inputStream) {
	IntervalTimer timer = new IntervalTimer();
	
	// Create a CodedInputStream instead of just a regular InputStream
	// so that can change the size limit. Otherwise if file is greater
	// than 64MB get an exception.
	CodedInputStream codedStream = 
			CodedInputStream.newInstance(inputStream);
	// What to use instead of default 64MB limit
	final int GTFS_SIZE_LIMIT = 200000000;
	codedStream.setSizeLimit(GTFS_SIZE_LIMIT);	
	
	// Actual read in the data into a protobuffer FeedMessage object.
	// Would prefer to do this one VehiclePosition at a time using
	// something like VehiclePosition.parseFrom(codedStream) so that
	// wouldn't have to load entire protobuffer file into memory. But
	// it never seemed to complete, even for just a single call to
	// parseFrom(). Therefore loading in entire file at once.
	FeedMessage feedMessage;
	try {
		feedMessage = FeedMessage.parseFrom(codedStream);
		logger.info("Parsing GTFS-realtime file into a FeedMessage took " +
				"{} msec", timer.elapsedMsec());
		return processMessage(feedMessage);
	} catch (IOException e) {
		logger.error("Exception when reading GTFS-realtime data from " +
				"input stream.", e);
		return new ArrayList<AvlReport>();
	}
	
}
 

/**
 * Convert a byte string to a Configuration object
 *
 * @param byteString byteString representation of the conf created using {@link
 *                   #createByteStringFromConf(org.apache.hadoop.conf.Configuration)}
 * @return Configuration
 * @throws java.io.IOException
 */
public static Configuration createConfFromByteString(ByteString byteString) throws IOException {
  Objects.requireNonNull(byteString, "ByteString must be specified");
  try(SnappyInputStream uncompressIs = new SnappyInputStream(byteString.newInput());) {
    CodedInputStream in = CodedInputStream.newInstance(uncompressIs);
    in.setSizeLimit(Integer.MAX_VALUE);
    DAGProtos.ConfigurationProto confProto = DAGProtos.ConfigurationProto.parseFrom(in);
    Configuration conf = new Configuration(false);
    readConfFromPB(confProto, conf);
    return conf;
  }
}
 

/**
 * Returns the loaded protocol buffer from the given byte stream. You normally want
 * {@link Wallet#loadFromFile(File, WalletExtension...)} instead - this method is designed for low level
 * work involving the wallet file format itself.
 */
public static Protos.Wallet parseToProto(InputStream input) throws IOException {
    CodedInputStream codedInput = CodedInputStream.newInstance(input);
    codedInput.setSizeLimit(WALLET_SIZE_LIMIT);
    return Protos.Wallet.parseFrom(codedInput);
}
 

/**
 * Returns the loaded protocol buffer from the given byte stream. You normally want
 * {@link Wallet#loadFromFile(java.io.File, WalletExtension...)} instead - this method is designed for low level
 * work involving the wallet file format itself.
 */
public static Protos.Wallet parseToProto(InputStream input) throws IOException {
    CodedInputStream codedInput = CodedInputStream.newInstance(input);
    codedInput.setSizeLimit(WALLET_SIZE_LIMIT);
    return Protos.Wallet.parseFrom(codedInput);
}
 

/**
 * Returns the loaded protocol buffer from the given byte stream. You normally want
 * {@link Wallet#loadFromFile(java.io.File, WalletExtension...)} instead - this method is designed for low level
 * work involving the wallet file format itself.
 */
public static Protos.Wallet parseToProto(InputStream input) throws IOException {
    CodedInputStream codedInput = CodedInputStream.newInstance(input);
    codedInput.setSizeLimit(WALLET_SIZE_LIMIT);
    return Protos.Wallet.parseFrom(codedInput);
}
 

@Override
public T parsePartialFrom(CodedInputStream input, ExtensionRegistryLite extensionRegistry)
        throws InvalidProtocolBufferException {
    input.setSizeLimit(Integer.MAX_VALUE);
    return parser.parsePartialFrom(input, extensionRegistry);
}