Java Code Examples for com.amazonaws.services.kinesis.model.GetRecordsResult#getNextShardIterator()

The following examples show how to use com.amazonaws.services.kinesis.model.GetRecordsResult#getNextShardIterator() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: KinesisTestConsumer.java    From attic-apex-malhar with Apache License 2.0 6 votes vote down vote up 
public String processNextIterator(String iterator)
{
  GetRecordsRequest getRequest = new GetRecordsRequest();
  getRequest.setLimit(1000);

  getRequest.setShardIterator(iterator);
  // call "get" operation and get everything in this shard range
  GetRecordsResult getResponse = client.getRecords(getRequest);

  iterator = getResponse.getNextShardIterator();

  List<Record> records = getResponse.getRecords();
  processResponseRecords(records);

  return iterator;
}
Example 2
Source File: ShardConsumer.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up 
protected String getShardIteratorForAggregatedSequenceNumber(SequenceNumber sequenceNumber)
		throws Exception {

	String itrForLastAggregatedRecord =
			kinesis.getShardIterator(
					subscribedShard,
					ShardIteratorType.AT_SEQUENCE_NUMBER.toString(),
					sequenceNumber.getSequenceNumber());

	// get only the last aggregated record
	GetRecordsResult getRecordsResult = getRecords(itrForLastAggregatedRecord, 1);

	List<UserRecord> fetchedRecords = deaggregateRecords(
			getRecordsResult.getRecords(),
			subscribedShard.getShard().getHashKeyRange().getStartingHashKey(),
			subscribedShard.getShard().getHashKeyRange().getEndingHashKey());

	long lastSubSequenceNum = sequenceNumber.getSubSequenceNumber();
	for (UserRecord record : fetchedRecords) {
		// we have found a dangling sub-record if it has a larger subsequence number
		// than our last sequence number; if so, collect the record and update state
		if (record.getSubSequenceNumber() > lastSubSequenceNum) {
			deserializeRecordForCollectionAndUpdateState(record);
		}
	}

	return getRecordsResult.getNextShardIterator();
}
Example 3
Source File: ShardConsumer.java    From flink with Apache License 2.0 5 votes vote down vote up 
protected String getShardIteratorForAggregatedSequenceNumber(SequenceNumber sequenceNumber)
		throws Exception {

	String itrForLastAggregatedRecord =
			kinesis.getShardIterator(
					subscribedShard,
					ShardIteratorType.AT_SEQUENCE_NUMBER.toString(),
					sequenceNumber.getSequenceNumber());

	// get only the last aggregated record
	GetRecordsResult getRecordsResult = getRecords(itrForLastAggregatedRecord, 1);

	List<UserRecord> fetchedRecords = deaggregateRecords(
			getRecordsResult.getRecords(),
			subscribedShard.getShard().getHashKeyRange().getStartingHashKey(),
			subscribedShard.getShard().getHashKeyRange().getEndingHashKey());

	long lastSubSequenceNum = sequenceNumber.getSubSequenceNumber();
	for (UserRecord record : fetchedRecords) {
		// we have found a dangling sub-record if it has a larger subsequence number
		// than our last sequence number; if so, collect the record and update state
		if (record.getSubSequenceNumber() > lastSubSequenceNum) {
			deserializeRecordForCollectionAndUpdateState(record);
		}
	}

	return getRecordsResult.getNextShardIterator();
}
Example 4
Source File: ShardConsumer.java    From flink with Apache License 2.0 5 votes vote down vote up 
protected String getShardIteratorForAggregatedSequenceNumber(SequenceNumber sequenceNumber)
		throws Exception {

	String itrForLastAggregatedRecord =
			kinesis.getShardIterator(
					subscribedShard,
					ShardIteratorType.AT_SEQUENCE_NUMBER.toString(),
					sequenceNumber.getSequenceNumber());

	// get only the last aggregated record
	GetRecordsResult getRecordsResult = getRecords(itrForLastAggregatedRecord, 1);

	List<UserRecord> fetchedRecords = deaggregateRecords(
			getRecordsResult.getRecords(),
			subscribedShard.getShard().getHashKeyRange().getStartingHashKey(),
			subscribedShard.getShard().getHashKeyRange().getEndingHashKey());

	long lastSubSequenceNum = sequenceNumber.getSubSequenceNumber();
	for (UserRecord record : fetchedRecords) {
		// we have found a dangling sub-record if it has a larger subsequence number
		// than our last sequence number; if so, collect the record and update state
		if (record.getSubSequenceNumber() > lastSubSequenceNum) {
			deserializeRecordForCollectionAndUpdateState(record);
		}
	}

	return getRecordsResult.getNextShardIterator();
}
Example 5
Source File: ShardConsumer.java    From Flink-CEPplus with Apache License 2.0 4 votes vote down vote up 
@SuppressWarnings("unchecked")
@Override
public void run() {
	try {
		String nextShardItr = getShardIterator(lastSequenceNum);

		long processingStartTimeNanos = System.nanoTime();

		while (isRunning()) {
			if (nextShardItr == null) {
				fetcherRef.updateState(subscribedShardStateIndex, SentinelSequenceNumber.SENTINEL_SHARD_ENDING_SEQUENCE_NUM.get());

				// we can close this consumer thread once we've reached the end of the subscribed shard
				break;
			} else {
				shardMetricsReporter.setMaxNumberOfRecordsPerFetch(maxNumberOfRecordsPerFetch);
				GetRecordsResult getRecordsResult = getRecords(nextShardItr, maxNumberOfRecordsPerFetch);

				List<Record> aggregatedRecords = getRecordsResult.getRecords();
				int numberOfAggregatedRecords = aggregatedRecords.size();
				shardMetricsReporter.setNumberOfAggregatedRecords(numberOfAggregatedRecords);

				// each of the Kinesis records may be aggregated, so we must deaggregate them before proceeding
				List<UserRecord> fetchedRecords = deaggregateRecords(
					aggregatedRecords,
					subscribedShard.getShard().getHashKeyRange().getStartingHashKey(),
					subscribedShard.getShard().getHashKeyRange().getEndingHashKey());

				long recordBatchSizeBytes = 0L;
				for (UserRecord record : fetchedRecords) {
					recordBatchSizeBytes += record.getData().remaining();
					deserializeRecordForCollectionAndUpdateState(record);
				}

				int numberOfDeaggregatedRecords = fetchedRecords.size();
				shardMetricsReporter.setNumberOfDeaggregatedRecords(numberOfDeaggregatedRecords);

				nextShardItr = getRecordsResult.getNextShardIterator();

				long adjustmentEndTimeNanos = adjustRunLoopFrequency(processingStartTimeNanos, System.nanoTime());
				long runLoopTimeNanos = adjustmentEndTimeNanos - processingStartTimeNanos;
				maxNumberOfRecordsPerFetch = adaptRecordsToRead(runLoopTimeNanos, fetchedRecords.size(), recordBatchSizeBytes, maxNumberOfRecordsPerFetch);
				shardMetricsReporter.setRunLoopTimeNanos(runLoopTimeNanos);
				processingStartTimeNanos = adjustmentEndTimeNanos; // for next time through the loop
			}
		}
	} catch (Throwable t) {
		fetcherRef.stopWithError(t);
	}
}
Example 6
Source File: ShardConsumer.java    From flink with Apache License 2.0 4 votes vote down vote up 
@SuppressWarnings("unchecked")
@Override
public void run() {
	try {
		String nextShardItr = getShardIterator(lastSequenceNum);

		long processingStartTimeNanos = System.nanoTime();

		while (isRunning()) {
			if (nextShardItr == null) {
				fetcherRef.updateState(subscribedShardStateIndex, SentinelSequenceNumber.SENTINEL_SHARD_ENDING_SEQUENCE_NUM.get());

				// we can close this consumer thread once we've reached the end of the subscribed shard
				break;
			} else {
				shardMetricsReporter.setMaxNumberOfRecordsPerFetch(maxNumberOfRecordsPerFetch);
				GetRecordsResult getRecordsResult = getRecords(nextShardItr, maxNumberOfRecordsPerFetch);

				List<Record> aggregatedRecords = getRecordsResult.getRecords();
				int numberOfAggregatedRecords = aggregatedRecords.size();
				shardMetricsReporter.setNumberOfAggregatedRecords(numberOfAggregatedRecords);

				// each of the Kinesis records may be aggregated, so we must deaggregate them before proceeding
				List<UserRecord> fetchedRecords = deaggregateRecords(
					aggregatedRecords,
					subscribedShard.getShard().getHashKeyRange().getStartingHashKey(),
					subscribedShard.getShard().getHashKeyRange().getEndingHashKey());

				long recordBatchSizeBytes = 0L;
				for (UserRecord record : fetchedRecords) {
					recordBatchSizeBytes += record.getData().remaining();
					deserializeRecordForCollectionAndUpdateState(record);
				}

				int numberOfDeaggregatedRecords = fetchedRecords.size();
				shardMetricsReporter.setNumberOfDeaggregatedRecords(numberOfDeaggregatedRecords);

				nextShardItr = getRecordsResult.getNextShardIterator();

				long adjustmentEndTimeNanos = adjustRunLoopFrequency(processingStartTimeNanos, System.nanoTime());
				long runLoopTimeNanos = adjustmentEndTimeNanos - processingStartTimeNanos;
				maxNumberOfRecordsPerFetch = adaptRecordsToRead(runLoopTimeNanos, fetchedRecords.size(), recordBatchSizeBytes, maxNumberOfRecordsPerFetch);
				shardMetricsReporter.setRunLoopTimeNanos(runLoopTimeNanos);
				processingStartTimeNanos = adjustmentEndTimeNanos; // for next time through the loop
			}
		}
	} catch (Throwable t) {
		fetcherRef.stopWithError(t);
	}
}
Example 7
Source File: ShardConsumer.java    From flink with Apache License 2.0 4 votes vote down vote up 
@SuppressWarnings("unchecked")
@Override
public void run() {
	try {
		String nextShardItr = getShardIterator(lastSequenceNum);

		long processingStartTimeNanos = System.nanoTime();

		while (isRunning()) {
			if (nextShardItr == null) {
				fetcherRef.updateState(subscribedShardStateIndex, SentinelSequenceNumber.SENTINEL_SHARD_ENDING_SEQUENCE_NUM.get());

				// we can close this consumer thread once we've reached the end of the subscribed shard
				break;
			} else {
				shardMetricsReporter.setMaxNumberOfRecordsPerFetch(maxNumberOfRecordsPerFetch);
				GetRecordsResult getRecordsResult = getRecords(nextShardItr, maxNumberOfRecordsPerFetch);

				List<Record> aggregatedRecords = getRecordsResult.getRecords();
				int numberOfAggregatedRecords = aggregatedRecords.size();
				shardMetricsReporter.setNumberOfAggregatedRecords(numberOfAggregatedRecords);

				// each of the Kinesis records may be aggregated, so we must deaggregate them before proceeding
				List<UserRecord> fetchedRecords = deaggregateRecords(
					aggregatedRecords,
					subscribedShard.getShard().getHashKeyRange().getStartingHashKey(),
					subscribedShard.getShard().getHashKeyRange().getEndingHashKey());

				long recordBatchSizeBytes = 0L;
				for (UserRecord record : fetchedRecords) {
					recordBatchSizeBytes += record.getData().remaining();
					deserializeRecordForCollectionAndUpdateState(record);
				}

				int numberOfDeaggregatedRecords = fetchedRecords.size();
				shardMetricsReporter.setNumberOfDeaggregatedRecords(numberOfDeaggregatedRecords);

				nextShardItr = getRecordsResult.getNextShardIterator();

				long adjustmentEndTimeNanos = adjustRunLoopFrequency(processingStartTimeNanos, System.nanoTime());
				long runLoopTimeNanos = adjustmentEndTimeNanos - processingStartTimeNanos;
				maxNumberOfRecordsPerFetch = adaptRecordsToRead(runLoopTimeNanos, fetchedRecords.size(), recordBatchSizeBytes, maxNumberOfRecordsPerFetch);
				shardMetricsReporter.setRunLoopTimeNanos(runLoopTimeNanos);
				processingStartTimeNanos = adjustmentEndTimeNanos; // for next time through the loop
			}
		}
	} catch (Throwable t) {
		fetcherRef.stopWithError(t);
	}
}