com.amazonaws.services.kinesis.model.GetRecordsResult Java Examples

@Override
public GetRecordsResult getRecords(String shardIterator, int maxRecordsToGet) {
	BlockingQueue<String> queue = Preconditions.checkNotNull(this.shardIteratorToQueueMap.get(shardIterator),
	"no queue for iterator %s", shardIterator);
	List<Record> records = Collections.emptyList();
	try {
		String data = queue.take();
		Record record = new Record()
			.withData(
				ByteBuffer.wrap(String.valueOf(data).getBytes(ConfigConstants.DEFAULT_CHARSET)))
			.withPartitionKey(UUID.randomUUID().toString())
			.withApproximateArrivalTimestamp(new Date(System.currentTimeMillis()))
			.withSequenceNumber(String.valueOf(0));
		records = Collections.singletonList(record);
	} catch (InterruptedException e) {
		shardIterator = null;
	}
	return new GetRecordsResult()
		.withRecords(records)
		.withMillisBehindLatest(0L)
		.withNextShardIterator(shardIterator);
}
 

@Override
public GetRecordsResult getRecords(String shardIterator, int maxRecordsToGet) {
	BlockingQueue<String> queue = Preconditions.checkNotNull(this.shardIteratorToQueueMap.get(shardIterator),
	"no queue for iterator %s", shardIterator);
	List<Record> records = Collections.emptyList();
	try {
		String data = queue.take();
		Record record = new Record()
			.withData(
				ByteBuffer.wrap(data.getBytes(ConfigConstants.DEFAULT_CHARSET)))
			.withPartitionKey(UUID.randomUUID().toString())
			.withApproximateArrivalTimestamp(new Date(System.currentTimeMillis()))
			.withSequenceNumber(String.valueOf(0));
		records = Collections.singletonList(record);
	} catch (InterruptedException e) {
		shardIterator = null;
	}
	return new GetRecordsResult()
		.withRecords(records)
		.withMillisBehindLatest(0L)
		.withNextShardIterator(shardIterator);
}
 

/**
 * Gets records from Kinesis and deaggregates them if needed.
 *
 * @return list of deaggregated records
 * @throws TransientKinesisException - in case of recoverable situation
 */
public GetKinesisRecordsResult getRecords(
    final String shardIterator,
    final String streamName,
    final String shardId,
    final Integer limit)
    throws TransientKinesisException {
  return wrapExceptions(
      () -> {
        GetRecordsResult response =
            kinesis.getRecords(
                new GetRecordsRequest().withShardIterator(shardIterator).withLimit(limit));
        return new GetKinesisRecordsResult(
            UserRecord.deaggregate(response.getRecords()),
            response.getNextShardIterator(),
            response.getMillisBehindLatest(),
            streamName,
            shardId);
      });
}
 

@Test
public void shouldReturnLimitedNumberOfRecords() throws Exception {
  final Integer limit = 100;

  doAnswer(
          (Answer<GetRecordsResult>)
              invocation -> {
                GetRecordsRequest request = (GetRecordsRequest) invocation.getArguments()[0];
                List<Record> records = generateRecords(request.getLimit());
                return new GetRecordsResult().withRecords(records).withMillisBehindLatest(1000L);
              })
      .when(kinesis)
      .getRecords(any(GetRecordsRequest.class));

  GetKinesisRecordsResult result = underTest.getRecords(SHARD_ITERATOR, STREAM, SHARD_1, limit);
  assertThat(result.getRecords().size()).isEqualTo(limit);
}
 

@Override
public GetRecordsResult getRecords(String shardIterator, int maxRecordsToGet) {
	if ((Integer.valueOf(shardIterator) == orderOfCallToExpire - 1) && !expiredOnceAlready) {
		// we fake only once the expired iterator exception at the specified get records attempt order
		expiredOnceAlready = true;
		throw new ExpiredIteratorException("Artificial expired shard iterator");
	} else if (expiredOnceAlready && !expiredIteratorRefreshed) {
		// if we've thrown the expired iterator exception already, but the iterator was not refreshed,
		// throw a hard exception to the test that is testing this Kinesis behaviour
		throw new RuntimeException("expired shard iterator was not refreshed on the next getRecords() call");
	} else {
		// assuming that the maxRecordsToGet is always large enough
		return new GetRecordsResult()
			.withRecords(shardItrToRecordBatch.get(shardIterator))
			.withMillisBehindLatest(millisBehindLatest)
			.withNextShardIterator(
				(Integer.valueOf(shardIterator) == totalNumOfGetRecordsCalls - 1)
					? null : String.valueOf(Integer.valueOf(shardIterator) + 1)); // last next shard iterator is null
	}
}
 

public String processNextIterator(String iterator)
{
  GetRecordsRequest getRequest = new GetRecordsRequest();
  getRequest.setLimit(1000);

  getRequest.setShardIterator(iterator);
  // call "get" operation and get everything in this shard range
  GetRecordsResult getResponse = client.getRecords(getRequest);

  iterator = getResponse.getNextShardIterator();

  List<Record> records = getResponse.getRecords();
  processResponseRecords(records);

  return iterator;
}
 

public static List<com.amazonaws.services.kinesis.model.Record> getPreviewRecords(
    ClientConfiguration awsClientConfig,
    KinesisConfigBean conf,
    int maxBatchSize,
    GetShardIteratorRequest getShardIteratorRequest
) throws StageException {
  AmazonKinesis kinesisClient = getKinesisClient(awsClientConfig, conf);

  GetShardIteratorResult getShardIteratorResult = kinesisClient.getShardIterator(getShardIteratorRequest);
  String shardIterator = getShardIteratorResult.getShardIterator();

  GetRecordsRequest getRecordsRequest = new GetRecordsRequest();
  getRecordsRequest.setShardIterator(shardIterator);
  getRecordsRequest.setLimit(maxBatchSize);

  GetRecordsResult getRecordsResult = kinesisClient.getRecords(getRecordsRequest);
  return getRecordsResult.getRecords();
}
 

/**
 * Calls {@link KinesisProxyInterface#getRecords(String, int)}, while also handling unexpected
 * AWS {@link ExpiredIteratorException}s to assure that we get results and don't just fail on
 * such occasions. The returned shard iterator within the successful {@link GetRecordsResult} should
 * be used for the next call to this method.
 *
 * <p>Note: it is important that this method is not called again before all the records from the last result have been
 * fully collected with {@link ShardConsumer#deserializeRecordForCollectionAndUpdateState(UserRecord)}, otherwise
 * {@link ShardConsumer#lastSequenceNum} may refer to a sub-record in the middle of an aggregated record, leading to
 * incorrect shard iteration if the iterator had to be refreshed.
 *
 * @param shardItr shard iterator to use
 * @param maxNumberOfRecords the maximum number of records to fetch for this getRecords attempt
 * @return get records result
 * @throws InterruptedException
 */
private GetRecordsResult getRecords(String shardItr, int maxNumberOfRecords) throws Exception {
	GetRecordsResult getRecordsResult = null;
	while (getRecordsResult == null) {
		try {
			getRecordsResult = kinesis.getRecords(shardItr, maxNumberOfRecords);

			// Update millis behind latest so it gets reported by the millisBehindLatest gauge
			Long millisBehindLatest = getRecordsResult.getMillisBehindLatest();
			if (millisBehindLatest != null) {
				shardMetricsReporter.setMillisBehindLatest(millisBehindLatest);
			}
		} catch (ExpiredIteratorException eiEx) {
			LOG.warn("Encountered an unexpected expired iterator {} for shard {};" +
				" refreshing the iterator ...", shardItr, subscribedShard);

			shardItr = getShardIterator(lastSequenceNum);

			// sleep for the fetch interval before the next getRecords attempt with the refreshed iterator
			if (fetchIntervalMillis != 0) {
				Thread.sleep(fetchIntervalMillis);
			}
		}
	}
	return getRecordsResult;
}
 

/**
 * Calls {@link KinesisProxyInterface#getRecords(String, int)}, while also handling unexpected
 * AWS {@link ExpiredIteratorException}s to assure that we get results and don't just fail on
 * such occasions. The returned shard iterator within the successful {@link GetRecordsResult} should
 * be used for the next call to this method.
 *
 * <p>Note: it is important that this method is not called again before all the records from the last result have been
 * fully collected with {@link ShardConsumer#deserializeRecordForCollectionAndUpdateState(UserRecord)}, otherwise
 * {@link ShardConsumer#lastSequenceNum} may refer to a sub-record in the middle of an aggregated record, leading to
 * incorrect shard iteration if the iterator had to be refreshed.
 *
 * @param shardItr shard iterator to use
 * @param maxNumberOfRecords the maximum number of records to fetch for this getRecords attempt
 * @return get records result
 * @throws InterruptedException
 */
private GetRecordsResult getRecords(String shardItr, int maxNumberOfRecords) throws Exception {
	GetRecordsResult getRecordsResult = null;
	while (getRecordsResult == null) {
		try {
			getRecordsResult = kinesis.getRecords(shardItr, maxNumberOfRecords);

			// Update millis behind latest so it gets reported by the millisBehindLatest gauge
			Long millisBehindLatest = getRecordsResult.getMillisBehindLatest();
			if (millisBehindLatest != null) {
				shardMetricsReporter.setMillisBehindLatest(millisBehindLatest);
			}
		} catch (ExpiredIteratorException eiEx) {
			LOG.warn("Encountered an unexpected expired iterator {} for shard {};" +
				" refreshing the iterator ...", shardItr, subscribedShard);

			shardItr = getShardIterator(lastSequenceNum);

			// sleep for the fetch interval before the next getRecords attempt with the refreshed iterator
			if (fetchIntervalMillis != 0) {
				Thread.sleep(fetchIntervalMillis);
			}
		}
	}
	return getRecordsResult;
}
 

public static KinesisProxyInterface noShardsFoundForRequestedStreamsBehaviour() {

		return new KinesisProxyInterface() {
			@Override
			public GetShardListResult getShardList(Map<String, String> streamNamesWithLastSeenShardIds) {
				return new GetShardListResult(); // not setting any retrieved shards for result
			}

			@Override
			public String getShardIterator(StreamShardHandle shard, String shardIteratorType, Object startingMarker) {
				return null;
			}

			@Override
			public GetRecordsResult getRecords(String shardIterator, int maxRecordsToGet) {
				return null;
			}
		};

	}
 

@Override
public GetRecordsResult getRecords(String shardIterator, int maxRecordsToGet) {
	if ((Integer.valueOf(shardIterator) == orderOfCallToExpire - 1) && !expiredOnceAlready) {
		// we fake only once the expired iterator exception at the specified get records attempt order
		expiredOnceAlready = true;
		throw new ExpiredIteratorException("Artificial expired shard iterator");
	} else if (expiredOnceAlready && !expiredIteratorRefreshed) {
		// if we've thrown the expired iterator exception already, but the iterator was not refreshed,
		// throw a hard exception to the test that is testing this Kinesis behaviour
		throw new RuntimeException("expired shard iterator was not refreshed on the next getRecords() call");
	} else {
		// assuming that the maxRecordsToGet is always large enough
		return new GetRecordsResult()
			.withRecords(shardItrToRecordBatch.get(shardIterator))
			.withMillisBehindLatest(millisBehindLatest)
			.withNextShardIterator(
				(Integer.valueOf(shardIterator) == totalNumOfGetRecordsCalls - 1)
					? null : String.valueOf(Integer.valueOf(shardIterator) + 1)); // last next shard iterator is null
	}
}
 

@Override
public GetRecordsResult getRecords(String shardIterator, int maxRecordsToGet) {
	if ((Integer.valueOf(shardIterator) == orderOfCallToExpire - 1) && !expiredOnceAlready) {
		// we fake only once the expired iterator exception at the specified get records attempt order
		expiredOnceAlready = true;
		throw new ExpiredIteratorException("Artificial expired shard iterator");
	} else if (expiredOnceAlready && !expiredIteratorRefreshed) {
		// if we've thrown the expired iterator exception already, but the iterator was not refreshed,
		// throw a hard exception to the test that is testing this Kinesis behaviour
		throw new RuntimeException("expired shard iterator was not refreshed on the next getRecords() call");
	} else {
		// assuming that the maxRecordsToGet is always large enough
		return new GetRecordsResult()
			.withRecords(shardItrToRecordBatch.get(shardIterator))
			.withMillisBehindLatest(millisBehindLatest)
			.withNextShardIterator(
				(Integer.valueOf(shardIterator) == totalNumOfGetRecordsCalls - 1)
					? null : String.valueOf(Integer.valueOf(shardIterator) + 1)); // last next shard iterator is null
	}
}
 

public static KinesisProxyInterface noShardsFoundForRequestedStreamsBehaviour() {

		return new KinesisProxyInterface() {
			@Override
			public GetShardListResult getShardList(Map<String, String> streamNamesWithLastSeenShardIds) {
				return new GetShardListResult(); // not setting any retrieved shards for result
			}

			@Override
			public String getShardIterator(StreamShardHandle shard, String shardIteratorType, Object startingMarker) {
				return null;
			}

			@Override
			public GetRecordsResult getRecords(String shardIterator, int maxRecordsToGet) {
				return null;
			}
		};

	}
 

@Override
public GetRecordsResult getRecords(String shardIterator, int maxRecordsToGet) {
	BlockingQueue<String> queue = Preconditions.checkNotNull(this.shardIteratorToQueueMap.get(shardIterator),
	"no queue for iterator %s", shardIterator);
	List<Record> records = Collections.emptyList();
	try {
		String data = queue.take();
		Record record = new Record()
			.withData(
				ByteBuffer.wrap(data.getBytes(ConfigConstants.DEFAULT_CHARSET)))
			.withPartitionKey(UUID.randomUUID().toString())
			.withApproximateArrivalTimestamp(new Date(System.currentTimeMillis()))
			.withSequenceNumber(String.valueOf(0));
		records = Collections.singletonList(record);
	} catch (InterruptedException e) {
		shardIterator = null;
	}
	return new GetRecordsResult()
		.withRecords(records)
		.withMillisBehindLatest(0L)
		.withNextShardIterator(shardIterator);
}
 

/**
 * Calls {@link KinesisProxyInterface#getRecords(String, int)}, while also handling unexpected
 * AWS {@link ExpiredIteratorException}s to assure that we get results and don't just fail on
 * such occasions. The returned shard iterator within the successful {@link GetRecordsResult} should
 * be used for the next call to this method.
 *
 * <p>Note: it is important that this method is not called again before all the records from the last result have been
 * fully collected with {@link ShardConsumer#deserializeRecordForCollectionAndUpdateState(UserRecord)}, otherwise
 * {@link ShardConsumer#lastSequenceNum} may refer to a sub-record in the middle of an aggregated record, leading to
 * incorrect shard iteration if the iterator had to be refreshed.
 *
 * @param shardItr shard iterator to use
 * @param maxNumberOfRecords the maximum number of records to fetch for this getRecords attempt
 * @return get records result
 * @throws InterruptedException
 */
private GetRecordsResult getRecords(String shardItr, int maxNumberOfRecords) throws Exception {
	GetRecordsResult getRecordsResult = null;
	while (getRecordsResult == null) {
		try {
			getRecordsResult = kinesis.getRecords(shardItr, maxNumberOfRecords);

			// Update millis behind latest so it gets reported by the millisBehindLatest gauge
			Long millisBehindLatest = getRecordsResult.getMillisBehindLatest();
			if (millisBehindLatest != null) {
				shardMetricsReporter.setMillisBehindLatest(millisBehindLatest);
			}
		} catch (ExpiredIteratorException eiEx) {
			LOG.warn("Encountered an unexpected expired iterator {} for shard {};" +
				" refreshing the iterator ...", shardItr, subscribedShard);

			shardItr = getShardIterator(lastSequenceNum);

			// sleep for the fetch interval before the next getRecords attempt with the refreshed iterator
			if (fetchIntervalMillis != 0) {
				Thread.sleep(fetchIntervalMillis);
			}
		}
	}
	return getRecordsResult;
}
 

public List<String> readAllMessages(String streamName) throws Exception {
	KinesisProxyInterface kinesisProxy = KinesisProxy.create(properties);
	Map<String, String> streamNamesWithLastSeenShardIds = new HashMap<>();
	streamNamesWithLastSeenShardIds.put(streamName, null);

	GetShardListResult shardListResult = kinesisProxy.getShardList(streamNamesWithLastSeenShardIds);
	int maxRecordsToFetch = 10;

	List<String> messages = new ArrayList<>();
	// retrieve records from all shards
	for (StreamShardHandle ssh : shardListResult.getRetrievedShardListOfStream(streamName)) {
		String shardIterator = kinesisProxy.getShardIterator(ssh, "TRIM_HORIZON", null);
		GetRecordsResult getRecordsResult = kinesisProxy.getRecords(shardIterator, maxRecordsToFetch);
		List<Record> aggregatedRecords = getRecordsResult.getRecords();
		for (Record record : aggregatedRecords) {
			messages.add(new String(record.getData().array()));
		}
	}
	return messages;
}
 

@Test
public void testGetRecordsRetry() throws Exception {
	Properties kinesisConsumerConfig = new Properties();
	kinesisConsumerConfig.setProperty(ConsumerConfigConstants.AWS_REGION, "us-east-1");

	final GetRecordsResult expectedResult = new GetRecordsResult();
	MutableInt retries = new MutableInt();
	final Throwable[] retriableExceptions = new Throwable[] {
		new AmazonKinesisException("mock"),
	};

	AmazonKinesisClient mockClient = mock(AmazonKinesisClient.class);
	Mockito.when(mockClient.getRecords(any())).thenAnswer(new Answer<GetRecordsResult>() {
		@Override
		public GetRecordsResult answer(InvocationOnMock invocation) throws Throwable{
			if (retries.intValue() < retriableExceptions.length) {
				retries.increment();
				throw retriableExceptions[retries.intValue() - 1];
			}
			return expectedResult;
		}
	});

	KinesisProxy kinesisProxy = new KinesisProxy(kinesisConsumerConfig);
	Whitebox.getField(KinesisProxy.class, "kinesisClient").set(kinesisProxy, mockClient);

	GetRecordsResult result = kinesisProxy.getRecords("fakeShardIterator", 1);
	assertEquals(retriableExceptions.length, retries.intValue());
	assertEquals(expectedResult, result);
}
 

@Override
public GetRecordsResult getRecords(String shardIterator, int maxRecordsToGet) {
	// assuming that the maxRecordsToGet is always large enough
	return new GetRecordsResult()
			.withRecords(shardItrToRecordBatch.get(shardIterator))
			.withMillisBehindLatest(millisBehindLatest)
			.withNextShardIterator(
					(Integer.valueOf(shardIterator) == totalNumOfGetRecordsCalls - 1)
							? null : String
							.valueOf(Integer.valueOf(shardIterator) + 1)); // last next shard iterator is null
}
 

/**
 * {@inheritDoc}
 */
@Override
public GetRecordsResult getRecords(String shardIterator, int maxRecordsToGet) throws InterruptedException {
	final GetRecordsRequest getRecordsRequest = new GetRecordsRequest();
	getRecordsRequest.setShardIterator(shardIterator);
	getRecordsRequest.setLimit(maxRecordsToGet);

	GetRecordsResult getRecordsResult = null;

	int retryCount = 0;
	while (retryCount <= getRecordsMaxRetries && getRecordsResult == null) {
		try {
			getRecordsResult = kinesisClient.getRecords(getRecordsRequest);
		} catch (SdkClientException ex) {
			if (isRecoverableSdkClientException(ex)) {
				long backoffMillis = fullJitterBackoff(
					getRecordsBaseBackoffMillis, getRecordsMaxBackoffMillis, getRecordsExpConstant, retryCount++);
				LOG.warn("Got recoverable SdkClientException. Backing off for "
					+ backoffMillis + " millis (" + ex.getClass().getName() + ": " + ex.getMessage() + ")");
				Thread.sleep(backoffMillis);
			} else {
				throw ex;
			}
		}
	}

	if (getRecordsResult == null) {
		throw new RuntimeException("Retries exceeded for getRecords operation - all " + getRecordsMaxRetries +
			" retry attempts failed.");
	}

	return getRecordsResult;
}
 

protected String getShardIteratorForAggregatedSequenceNumber(SequenceNumber sequenceNumber)
		throws Exception {

	String itrForLastAggregatedRecord =
			kinesis.getShardIterator(
					subscribedShard,
					ShardIteratorType.AT_SEQUENCE_NUMBER.toString(),
					sequenceNumber.getSequenceNumber());

	// get only the last aggregated record
	GetRecordsResult getRecordsResult = getRecords(itrForLastAggregatedRecord, 1);

	List<UserRecord> fetchedRecords = deaggregateRecords(
			getRecordsResult.getRecords(),
			subscribedShard.getShard().getHashKeyRange().getStartingHashKey(),
			subscribedShard.getShard().getHashKeyRange().getEndingHashKey());

	long lastSubSequenceNum = sequenceNumber.getSubSequenceNumber();
	for (UserRecord record : fetchedRecords) {
		// we have found a dangling sub-record if it has a larger subsequence number
		// than our last sequence number; if so, collect the record and update state
		if (record.getSubSequenceNumber() > lastSubSequenceNum) {
			deserializeRecordForCollectionAndUpdateState(record);
		}
	}

	return getRecordsResult.getNextShardIterator();
}
 

@Override
public GetRecordsResult getRecords(String shardIterator, int maxRecordsToGet) {
	// assuming that the maxRecordsToGet is always large enough
	return new GetRecordsResult()
		.withRecords(shardItrToRecordBatch.get(shardIterator))
		.withMillisBehindLatest(millisBehindLatest)
		.withNextShardIterator(
			(Integer.valueOf(shardIterator) == totalNumOfGetRecordsCalls - 1)
				? null : String.valueOf(Integer.valueOf(shardIterator) + 1)); // last next shard iterator is null
}
 

@Override
public GetRecordsResult getRecords(GetRecordsRequest getRecordsRequest) {
  List<String> shardIteratorParts =
      Splitter.on(':').splitToList(getRecordsRequest.getShardIterator());
  int shardId = parseInt(shardIteratorParts.get(0));
  int startingRecord = parseInt(shardIteratorParts.get(1));
  List<Record> shardData = shardedData.get(shardId);

  int toIndex = min(startingRecord + numberOfRecordsPerGet, shardData.size());
  int fromIndex = min(startingRecord, toIndex);
  return new GetRecordsResult()
      .withRecords(shardData.subList(fromIndex, toIndex))
      .withNextShardIterator(String.format("%s:%s", shardId, toIndex))
      .withMillisBehindLatest(0L);
}
 

@Override
public GetRecordsResult getRecords(String shardIterator, int maxRecordsToGet) {
	// assuming that the maxRecordsToGet is always large enough
	return new GetRecordsResult()
			.withRecords(shardItrToRecordBatch.get(shardIterator))
			.withMillisBehindLatest(millisBehindLatest)
			.withNextShardIterator(
					(Integer.valueOf(shardIterator) == totalNumOfGetRecordsCalls - 1)
							? null : String
							.valueOf(Integer.valueOf(shardIterator) + 1)); // last next shard iterator is null
}
 

protected String getShardIteratorForAggregatedSequenceNumber(SequenceNumber sequenceNumber)
		throws Exception {

	String itrForLastAggregatedRecord =
			kinesis.getShardIterator(
					subscribedShard,
					ShardIteratorType.AT_SEQUENCE_NUMBER.toString(),
					sequenceNumber.getSequenceNumber());

	// get only the last aggregated record
	GetRecordsResult getRecordsResult = getRecords(itrForLastAggregatedRecord, 1);

	List<UserRecord> fetchedRecords = deaggregateRecords(
			getRecordsResult.getRecords(),
			subscribedShard.getShard().getHashKeyRange().getStartingHashKey(),
			subscribedShard.getShard().getHashKeyRange().getEndingHashKey());

	long lastSubSequenceNum = sequenceNumber.getSubSequenceNumber();
	for (UserRecord record : fetchedRecords) {
		// we have found a dangling sub-record if it has a larger subsequence number
		// than our last sequence number; if so, collect the record and update state
		if (record.getSubSequenceNumber() > lastSubSequenceNum) {
			deserializeRecordForCollectionAndUpdateState(record);
		}
	}

	return getRecordsResult.getNextShardIterator();
}
 

/**
 * {@inheritDoc}
 */
@Override
public GetRecordsResult getRecords(String shardIterator, int maxRecordsToGet) throws InterruptedException {
	final GetRecordsRequest getRecordsRequest = new GetRecordsRequest();
	getRecordsRequest.setShardIterator(shardIterator);
	getRecordsRequest.setLimit(maxRecordsToGet);

	GetRecordsResult getRecordsResult = null;

	int retryCount = 0;
	while (retryCount <= getRecordsMaxRetries && getRecordsResult == null) {
		try {
			getRecordsResult = kinesisClient.getRecords(getRecordsRequest);
		} catch (SdkClientException ex) {
			if (isRecoverableSdkClientException(ex)) {
				long backoffMillis = fullJitterBackoff(
					getRecordsBaseBackoffMillis, getRecordsMaxBackoffMillis, getRecordsExpConstant, retryCount++);
				LOG.warn("Got recoverable SdkClientException. Backing off for "
					+ backoffMillis + " millis (" + ex.getClass().getName() + ": " + ex.getMessage() + ")");
				Thread.sleep(backoffMillis);
			} else {
				throw ex;
			}
		}
	}

	if (getRecordsResult == null) {
		throw new RuntimeException("Retries exceeded for getRecords operation - all " + getRecordsMaxRetries +
			" retry attempts failed.");
	}

	return getRecordsResult;
}
 

@Test
public void testGetRecordsRetry() throws Exception {
	Properties kinesisConsumerConfig = new Properties();
	kinesisConsumerConfig.setProperty(ConsumerConfigConstants.AWS_REGION, "us-east-1");

	final GetRecordsResult expectedResult = new GetRecordsResult();
	MutableInt retries = new MutableInt();
	final Throwable[] retriableExceptions = new Throwable[] {
		new AmazonKinesisException("mock"),
	};

	AmazonKinesisClient mockClient = mock(AmazonKinesisClient.class);
	Mockito.when(mockClient.getRecords(any())).thenAnswer(new Answer<GetRecordsResult>() {
		@Override
		public GetRecordsResult answer(InvocationOnMock invocation) throws Throwable{
			if (retries.intValue() < retriableExceptions.length) {
				retries.increment();
				throw retriableExceptions[retries.intValue() - 1];
			}
			return expectedResult;
		}
	});

	KinesisProxy kinesisProxy = new KinesisProxy(kinesisConsumerConfig);
	Whitebox.getField(KinesisProxy.class, "kinesisClient").set(kinesisProxy, mockClient);

	GetRecordsResult result = kinesisProxy.getRecords("fakeShardIterator", 1);
	assertEquals(retriableExceptions.length, retries.intValue());
	assertEquals(expectedResult, result);
}
 

@Override
public GetRecordsResult getRecords(String shardIterator, int maxRecordsToGet) {
	// assuming that the maxRecordsToGet is always large enough
	return new GetRecordsResult()
		.withRecords(shardItrToRecordBatch.get(shardIterator))
		.withMillisBehindLatest(millisBehindLatest)
		.withNextShardIterator(
			(Integer.valueOf(shardIterator) == totalNumOfGetRecordsCalls - 1)
				? null : String.valueOf(Integer.valueOf(shardIterator) + 1)); // last next shard iterator is null
}
 

@Override
public GetRecordsResult getRecords(String shardIterator, int maxRecordsToGet) {
	// assuming that the maxRecordsToGet is always large enough
	return new GetRecordsResult()
			.withRecords(shardItrToRecordBatch.get(shardIterator))
			.withMillisBehindLatest(millisBehindLatest)
			.withNextShardIterator(
					(Integer.valueOf(shardIterator) == totalNumOfGetRecordsCalls - 1)
							? null : String
							.valueOf(Integer.valueOf(shardIterator) + 1)); // last next shard iterator is null
}
 

protected String getShardIteratorForAggregatedSequenceNumber(SequenceNumber sequenceNumber)
		throws Exception {

	String itrForLastAggregatedRecord =
			kinesis.getShardIterator(
					subscribedShard,
					ShardIteratorType.AT_SEQUENCE_NUMBER.toString(),
					sequenceNumber.getSequenceNumber());

	// get only the last aggregated record
	GetRecordsResult getRecordsResult = getRecords(itrForLastAggregatedRecord, 1);

	List<UserRecord> fetchedRecords = deaggregateRecords(
			getRecordsResult.getRecords(),
			subscribedShard.getShard().getHashKeyRange().getStartingHashKey(),
			subscribedShard.getShard().getHashKeyRange().getEndingHashKey());

	long lastSubSequenceNum = sequenceNumber.getSubSequenceNumber();
	for (UserRecord record : fetchedRecords) {
		// we have found a dangling sub-record if it has a larger subsequence number
		// than our last sequence number; if so, collect the record and update state
		if (record.getSubSequenceNumber() > lastSubSequenceNum) {
			deserializeRecordForCollectionAndUpdateState(record);
		}
	}

	return getRecordsResult.getNextShardIterator();
}
 

/**
 * {@inheritDoc}
 */
@Override
public GetRecordsResult getRecords(String shardIterator, int maxRecordsToGet) throws InterruptedException {
	final GetRecordsRequest getRecordsRequest = new GetRecordsRequest();
	getRecordsRequest.setShardIterator(shardIterator);
	getRecordsRequest.setLimit(maxRecordsToGet);

	GetRecordsResult getRecordsResult = null;

	int retryCount = 0;
	while (retryCount <= getRecordsMaxRetries && getRecordsResult == null) {
		try {
			getRecordsResult = kinesisClient.getRecords(getRecordsRequest);
		} catch (SdkClientException ex) {
			if (isRecoverableSdkClientException(ex)) {
				long backoffMillis = fullJitterBackoff(
					getRecordsBaseBackoffMillis, getRecordsMaxBackoffMillis, getRecordsExpConstant, retryCount++);
				LOG.warn("Got recoverable SdkClientException. Backing off for "
					+ backoffMillis + " millis (" + ex.getClass().getName() + ": " + ex.getMessage() + ")");
				Thread.sleep(backoffMillis);
			} else {
				throw ex;
			}
		}
	}

	if (getRecordsResult == null) {
		throw new RuntimeException("Retries exceeded for getRecords operation - all " + getRecordsMaxRetries +
			" retry attempts failed.");
	}

	return getRecordsResult;
}