com.amazonaws.services.kinesis.model.GetRecordsRequest Java Examples

/**
 * Gets records from Kinesis and deaggregates them if needed.
 *
 * @return list of deaggregated records
 * @throws TransientKinesisException - in case of recoverable situation
 */
public GetKinesisRecordsResult getRecords(
    final String shardIterator,
    final String streamName,
    final String shardId,
    final Integer limit)
    throws TransientKinesisException {
  return wrapExceptions(
      () -> {
        GetRecordsResult response =
            kinesis.getRecords(
                new GetRecordsRequest().withShardIterator(shardIterator).withLimit(limit));
        return new GetKinesisRecordsResult(
            UserRecord.deaggregate(response.getRecords()),
            response.getNextShardIterator(),
            response.getMillisBehindLatest(),
            streamName,
            shardId);
      });
}
 

@Test
public void shouldReturnLimitedNumberOfRecords() throws Exception {
  final Integer limit = 100;

  doAnswer(
          (Answer<GetRecordsResult>)
              invocation -> {
                GetRecordsRequest request = (GetRecordsRequest) invocation.getArguments()[0];
                List<Record> records = generateRecords(request.getLimit());
                return new GetRecordsResult().withRecords(records).withMillisBehindLatest(1000L);
              })
      .when(kinesis)
      .getRecords(any(GetRecordsRequest.class));

  GetKinesisRecordsResult result = underTest.getRecords(SHARD_ITERATOR, STREAM, SHARD_1, limit);
  assertThat(result.getRecords().size()).isEqualTo(limit);
}
 

public String processNextIterator(String iterator)
{
  GetRecordsRequest getRequest = new GetRecordsRequest();
  getRequest.setLimit(1000);

  getRequest.setShardIterator(iterator);
  // call "get" operation and get everything in this shard range
  GetRecordsResult getResponse = client.getRecords(getRequest);

  iterator = getResponse.getNextShardIterator();

  List<Record> records = getResponse.getRecords();
  processResponseRecords(records);

  return iterator;
}
 

public static List<com.amazonaws.services.kinesis.model.Record> getPreviewRecords(
    ClientConfiguration awsClientConfig,
    KinesisConfigBean conf,
    int maxBatchSize,
    GetShardIteratorRequest getShardIteratorRequest
) throws StageException {
  AmazonKinesis kinesisClient = getKinesisClient(awsClientConfig, conf);

  GetShardIteratorResult getShardIteratorResult = kinesisClient.getShardIterator(getShardIteratorRequest);
  String shardIterator = getShardIteratorResult.getShardIterator();

  GetRecordsRequest getRecordsRequest = new GetRecordsRequest();
  getRecordsRequest.setShardIterator(shardIterator);
  getRecordsRequest.setLimit(maxBatchSize);

  GetRecordsResult getRecordsResult = kinesisClient.getRecords(getRecordsRequest);
  return getRecordsResult.getRecords();
}
 

/**
 * {@inheritDoc}
 */
@Override
public GetRecordsResult getRecords(String shardIterator, int maxRecordsToGet) throws InterruptedException {
	final GetRecordsRequest getRecordsRequest = new GetRecordsRequest();
	getRecordsRequest.setShardIterator(shardIterator);
	getRecordsRequest.setLimit(maxRecordsToGet);

	GetRecordsResult getRecordsResult = null;

	int retryCount = 0;
	while (retryCount <= getRecordsMaxRetries && getRecordsResult == null) {
		try {
			getRecordsResult = kinesisClient.getRecords(getRecordsRequest);
		} catch (SdkClientException ex) {
			if (isRecoverableSdkClientException(ex)) {
				long backoffMillis = fullJitterBackoff(
					getRecordsBaseBackoffMillis, getRecordsMaxBackoffMillis, getRecordsExpConstant, retryCount++);
				LOG.warn("Got recoverable SdkClientException. Backing off for "
					+ backoffMillis + " millis (" + ex.getClass().getName() + ": " + ex.getMessage() + ")");
				Thread.sleep(backoffMillis);
			} else {
				throw ex;
			}
		}
	}

	if (getRecordsResult == null) {
		throw new RuntimeException("Retries exceeded for getRecords operation - all " + getRecordsMaxRetries +
			" retry attempts failed.");
	}

	return getRecordsResult;
}
 

/**
 * {@inheritDoc}
 */
@Override
public GetRecordsResult getRecords(String shardIterator, int maxRecordsToGet) throws InterruptedException {
	final GetRecordsRequest getRecordsRequest = new GetRecordsRequest();
	getRecordsRequest.setShardIterator(shardIterator);
	getRecordsRequest.setLimit(maxRecordsToGet);

	GetRecordsResult getRecordsResult = null;

	int retryCount = 0;
	while (retryCount <= getRecordsMaxRetries && getRecordsResult == null) {
		try {
			getRecordsResult = kinesisClient.getRecords(getRecordsRequest);
		} catch (SdkClientException ex) {
			if (isRecoverableSdkClientException(ex)) {
				long backoffMillis = fullJitterBackoff(
					getRecordsBaseBackoffMillis, getRecordsMaxBackoffMillis, getRecordsExpConstant, retryCount++);
				LOG.warn("Got recoverable SdkClientException. Backing off for "
					+ backoffMillis + " millis (" + ex.getClass().getName() + ": " + ex.getMessage() + ")");
				Thread.sleep(backoffMillis);
			} else {
				throw ex;
			}
		}
	}

	if (getRecordsResult == null) {
		throw new RuntimeException("Retries exceeded for getRecords operation - all " + getRecordsMaxRetries +
			" retry attempts failed.");
	}

	return getRecordsResult;
}
 

@Override
public GetRecordsResult getRecords(GetRecordsRequest getRecordsRequest) {
  List<String> shardIteratorParts =
      Splitter.on(':').splitToList(getRecordsRequest.getShardIterator());
  int shardId = parseInt(shardIteratorParts.get(0));
  int startingRecord = parseInt(shardIteratorParts.get(1));
  List<Record> shardData = shardedData.get(shardId);

  int toIndex = min(startingRecord + numberOfRecordsPerGet, shardData.size());
  int fromIndex = min(startingRecord, toIndex);
  return new GetRecordsResult()
      .withRecords(shardData.subList(fromIndex, toIndex))
      .withNextShardIterator(String.format("%s:%s", shardId, toIndex))
      .withMillisBehindLatest(0L);
}
 

/**
 * {@inheritDoc}
 */
@Override
public GetRecordsResult getRecords(String shardIterator, int maxRecordsToGet) throws InterruptedException {
	final GetRecordsRequest getRecordsRequest = new GetRecordsRequest();
	getRecordsRequest.setShardIterator(shardIterator);
	getRecordsRequest.setLimit(maxRecordsToGet);

	GetRecordsResult getRecordsResult = null;

	int retryCount = 0;
	while (retryCount <= getRecordsMaxRetries && getRecordsResult == null) {
		try {
			getRecordsResult = kinesisClient.getRecords(getRecordsRequest);
		} catch (SdkClientException ex) {
			if (isRecoverableSdkClientException(ex)) {
				long backoffMillis = fullJitterBackoff(
					getRecordsBaseBackoffMillis, getRecordsMaxBackoffMillis, getRecordsExpConstant, retryCount++);
				LOG.warn("Got recoverable SdkClientException. Backing off for "
					+ backoffMillis + " millis (" + ex.getClass().getName() + ": " + ex.getMessage() + ")");
				Thread.sleep(backoffMillis);
			} else {
				throw ex;
			}
		}
	}

	if (getRecordsResult == null) {
		throw new RuntimeException("Retries exceeded for getRecords operation - all " + getRecordsMaxRetries +
			" retry attempts failed.");
	}

	return getRecordsResult;
}
 

/**
 * Retrieves the next batch of records from Kinesis using the shard iterator.
 * <p>
 * Most of the time this results in one getRecords call.  However we allow for
 * a call to return an empty list, and we'll try again if we are far enough
 * away from the latest record.
 */
private void getKinesisRecords()
        throws ResourceNotFoundException
{
    // Normally this loop will execute once, but we have to allow for the odd Kinesis
    // behavior, per the docs:
    // A single call to getRecords might return an empty record list, even when the shard contains
    // more records at later sequence numbers
    boolean fetchedRecords = false;
    int attempts = 0;
    while (!fetchedRecords && attempts < fetchAttempts) {
        Duration duration = nanosSince(lastReadTime);
        if (duration.toMillis() <= sleepTime) {
            try {
                Thread.sleep(duration.toMillis());
            }
            catch (InterruptedException e) {
                Thread.currentThread().interrupt();
                throw new RuntimeException("thread interrupted");
            }
        }
        getRecordsRequest = new GetRecordsRequest();
        getRecordsRequest.setShardIterator(shardIterator);
        getRecordsRequest.setLimit(batchSize);

        getRecordsResult = clientManager.getClient().getRecords(getRecordsRequest);
        lastReadTime = System.nanoTime();

        shardIterator = getRecordsResult.getNextShardIterator();
        kinesisRecords = getRecordsResult.getRecords();
        if (isLogBatches) {
            log.info("Fetched %d records from Kinesis.  MillisBehindLatest=%d", kinesisRecords.size(), getRecordsResult.getMillisBehindLatest());
        }

        fetchedRecords = (kinesisRecords.size() > 0 || getMillisBehindLatest() <= MILLIS_BEHIND_LIMIT);
        attempts++;
    }

    listIterator = kinesisRecords.iterator();
    batchesRead++;
    messagesRead += kinesisRecords.size();
}
 

/**
 * Retrieves the next batch of records from Kinesis using the shard iterator.
 *
 * Most of the time this results in one getRecords call.  However we allow for
 * a call to return an empty list, and we'll try again if we are far enough
 * away from the latest record.
 */
private void getKinesisRecords()
        throws ResourceNotFoundException
{
    // Normally this loop will execute once, but we have to allow for the odd Kinesis
    // behavior, per the docs:
    // A single call to getRecords might return an empty record list, even when the shard contains
    // more records at later sequence numbers
    boolean fetchedRecords = false;
    int attempts = 0;
    while (!fetchedRecords && attempts < fetchAttempts) {
        long now = System.currentTimeMillis();
        if (now - lastReadTime <= sleepTime) {
            try {
                Thread.sleep(now - lastReadTime);
            }
            catch (InterruptedException e) {
                log.error("Sleep interrupted.", e);
            }
        }
        getRecordsRequest = new GetRecordsRequest();
        getRecordsRequest.setShardIterator(shardIterator);
        getRecordsRequest.setLimit(batchSize);

        getRecordsResult = clientManager.getClient().getRecords(getRecordsRequest);
        lastReadTime = System.currentTimeMillis();

        shardIterator = getRecordsResult.getNextShardIterator();
        kinesisRecords = getRecordsResult.getRecords();
        if (kinesisConnectorConfig.isLogBatches()) {
            log.info("Fetched %d records from Kinesis.  MillisBehindLatest=%d", kinesisRecords.size(), getRecordsResult.getMillisBehindLatest());
        }

        fetchedRecords = (kinesisRecords.size() > 0 || getMillisBehindLatest() <= MILLIS_BEHIND_LIMIT);
        attempts++;
    }

    listIterator = kinesisRecords.iterator();
    batchesRead++;
    messagesRead += kinesisRecords.size();
}