com.amazonaws.services.kinesis.model.ResourceNotFoundException Java Examples

@Test
void testProvisionConsumerSuccessfulWithNewStream() {
	AmazonKinesis amazonKinesisMock = mock(AmazonKinesis.class);
	KinesisBinderConfigurationProperties binderProperties = new KinesisBinderConfigurationProperties();
	KinesisStreamProvisioner provisioner = new KinesisStreamProvisioner(
			amazonKinesisMock, binderProperties);
	int instanceCount = 1;
	int concurrency = 1;

	ExtendedConsumerProperties<KinesisConsumerProperties> extendedConsumerProperties =
			new ExtendedConsumerProperties<>(
			new KinesisConsumerProperties());
	extendedConsumerProperties.setInstanceCount(instanceCount);
	extendedConsumerProperties.setConcurrency(concurrency);

	String name = "test-stream";
	String group = "test-group";

	DescribeStreamResult describeStreamResult = describeStreamResultWithShards(
			Collections.singletonList(new Shard()));

	when(amazonKinesisMock.describeStream(any(DescribeStreamRequest.class)))
			.thenThrow(new ResourceNotFoundException("I got nothing"))
			.thenReturn(describeStreamResult);

	when(amazonKinesisMock.createStream(name, instanceCount * concurrency))
			.thenReturn(new CreateStreamResult());

	ConsumerDestination destination = provisioner.provisionConsumerDestination(name,
			group, extendedConsumerProperties);

	verify(amazonKinesisMock, times(2))
			.describeStream(any(DescribeStreamRequest.class));

	verify(amazonKinesisMock).createStream(name, instanceCount * concurrency);

	assertThat(destination.getName()).isEqualTo(name);
}
 

/**
 * Helper method to determine if an Amazon Kinesis stream exists.
 * 
 * @param kinesisClient
 *        The {@link AmazonKinesisClient} with Amazon Kinesis read privileges
 * @param streamName
 *        The Amazon Kinesis stream to check for
 * @return true if the Amazon Kinesis stream exists, otherwise return false
 */
private static boolean streamExists(AmazonKinesisClient kinesisClient, String streamName) {
    DescribeStreamRequest describeStreamRequest = new DescribeStreamRequest();
    describeStreamRequest.setStreamName(streamName);
    try {
        kinesisClient.describeStream(describeStreamRequest);
        return true;
    } catch (ResourceNotFoundException e) {
        return false;
    }
}
 

/**
 * Get metainfo for a Kinesis stream, which contains information about which shards this
 * Kinesis stream possess.
 *
 * <p>This method is using a "full jitter" approach described in AWS's article,
 * <a href="https://www.awsarchitectureblog.com/2015/03/backoff.html">
 *   "Exponential Backoff and Jitter"</a>.
 * This is necessary because concurrent calls will be made by all parallel subtask's fetcher.
 * This jitter backoff approach will help distribute calls across the fetchers over time.
 *
 * @param streamName the stream to describe
 * @param startShardId which shard to start with for this describe operation
 *
 * @return the result of the describe stream operation
 */
protected DescribeStreamResult describeStream(String streamName, @Nullable String startShardId)
		throws InterruptedException {
	final DescribeStreamRequest describeStreamRequest = new DescribeStreamRequest();
	describeStreamRequest.setStreamName(streamName);
	describeStreamRequest.setExclusiveStartShardId(startShardId);

	DescribeStreamResult describeStreamResult = null;

	// Call DescribeStream, with full-jitter backoff (if we get LimitExceededException).
	int attemptCount = 0;
	while (describeStreamResult == null) { // retry until we get a result
		try {
			describeStreamResult = kinesisClient.describeStream(describeStreamRequest);
		} catch (LimitExceededException le) {
			long backoffMillis = fullJitterBackoff(
					describeStreamBaseBackoffMillis,
					describeStreamMaxBackoffMillis,
					describeStreamExpConstant,
					attemptCount++);
			LOG.warn(String.format("Got LimitExceededException when describing stream %s. "
					+ "Backing off for %d millis.", streamName, backoffMillis));
			Thread.sleep(backoffMillis);
		} catch (ResourceNotFoundException re) {
			throw new RuntimeException("Error while getting stream details", re);
		}
	}

	String streamStatus = describeStreamResult.getStreamDescription().getStreamStatus();
	if (!(streamStatus.equals(StreamStatus.ACTIVE.toString())
			|| streamStatus.equals(StreamStatus.UPDATING.toString()))) {
		if (LOG.isWarnEnabled()) {
			LOG.warn(String.format("The status of stream %s is %s ; result of the current "
							+ "describeStream operation will not contain any shard information.",
					streamName, streamStatus));
		}
	}

	return describeStreamResult;
}
 

/**
 * Helper method to determine if an Amazon Kinesis stream exists.
 * 
 * @param kinesisClient
 *        The {@link AmazonKinesisClient} with Amazon Kinesis read privileges
 * @param streamName
 *        The Amazon Kinesis stream to check for
 * @return true if the Amazon Kinesis stream exists, otherwise return false
 */
private static boolean streamExists(AmazonKinesisClient kinesisClient, String streamName) {
    DescribeStreamRequest describeStreamRequest = new DescribeStreamRequest();
    describeStreamRequest.setStreamName(streamName);
    try {
        kinesisClient.describeStream(describeStreamRequest);
        return true;
    } catch (ResourceNotFoundException e) {
        return false;
    }
}
 

private void getIterator()
        throws ResourceNotFoundException
{
    GetShardIteratorRequest getShardIteratorRequest = new GetShardIteratorRequest();
    getShardIteratorRequest.setStreamName(split.getStreamName());
    getShardIteratorRequest.setShardId(split.getShardId());

    // Explanation: when we have a sequence number from a prior read or checkpoint, always use it.
    // Otherwise, decide if starting at a timestamp or the trim horizon based on configuration.
    // If starting at a timestamp, sue the session variable ITER_START_TIMESTAMP when given, otherwise
    // fallback on starting at ITER_OFFSET_SECONDS from timestamp.
    if (lastReadSeqNo == null) {
        // Important: shard iterator type AT_TIMESTAMP requires 1.11.x or above of the AWS SDK.
        if (SessionVariables.getIterFromTimestamp(session)) {
            getShardIteratorRequest.setShardIteratorType("AT_TIMESTAMP");
            long iterStartTs = SessionVariables.getIterStartTimestamp(session);
            if (iterStartTs == 0) {
                long startTs = System.currentTimeMillis() - (SessionVariables.getIterOffsetSeconds(session) * 1000);
                getShardIteratorRequest.setTimestamp(new Date(startTs));
            }
            else {
                getShardIteratorRequest.setTimestamp(new Date(iterStartTs));
            }
        }
        else {
            getShardIteratorRequest.setShardIteratorType("TRIM_HORIZON");
        }
    }
    else {
        getShardIteratorRequest.setShardIteratorType("AFTER_SEQUENCE_NUMBER");
        getShardIteratorRequest.setStartingSequenceNumber(lastReadSeqNo);
    }

    GetShardIteratorResult getShardIteratorResult = clientManager.getClient().getShardIterator(getShardIteratorRequest);
    shardIterator = getShardIteratorResult.getShardIterator();
}
 

/**
 * Internal method to retrieve the stream description and get the shards from AWS.
 *
 * Gets from the internal cache unless not yet created or too old.
 *
 * @param streamName Kinesis stream name
 * @return Returns kinesis stream description
 */
protected InternalStreamDescription getStreamDescription(String streamName)
{
    InternalStreamDescription desc = this.streamMap.get(streamName);
    if (desc == null || System.currentTimeMillis() - desc.getCreateTimeStamp() >= MAX_CACHE_AGE_MILLIS) {
        desc = new InternalStreamDescription(streamName);

        DescribeStreamRequest describeStreamRequest = clientManager.getDescribeStreamRequest();
        describeStreamRequest.setStreamName(streamName);

        // Collect shards from Kinesis
        String exclusiveStartShardId = null;
        List<Shard> shards = new ArrayList<>();
        do {
            describeStreamRequest.setExclusiveStartShardId(exclusiveStartShardId);
            DescribeStreamResult describeStreamResult = clientManager.getClient().describeStream(describeStreamRequest);

            String streamStatus = describeStreamResult.getStreamDescription().getStreamStatus();
            if (!streamStatus.equals("ACTIVE") && !streamStatus.equals("UPDATING")) {
                throw new ResourceNotFoundException("Stream not Active");
            }

            desc.addAllShards(describeStreamResult.getStreamDescription().getShards());

            if (describeStreamResult.getStreamDescription().getHasMoreShards() && (shards.size() > 0)) {
                exclusiveStartShardId = shards.get(shards.size() - 1).getShardId();
            }
            else {
                exclusiveStartShardId = null;
            }
        } while (exclusiveStartShardId != null);

        this.streamMap.put(streamName, desc);
    }

    return desc;
}
 

@Test
void testProvisionConsumerResourceNotFoundException() {
	AmazonKinesis amazonKinesisMock = mock(AmazonKinesis.class);
	KinesisBinderConfigurationProperties binderProperties = new KinesisBinderConfigurationProperties();
	binderProperties.setAutoCreateStream(false);
	KinesisStreamProvisioner provisioner = new KinesisStreamProvisioner(
			amazonKinesisMock, binderProperties);
	int instanceCount = 1;
	int concurrency = 1;

	ExtendedConsumerProperties<KinesisConsumerProperties> extendedConsumerProperties =
			new ExtendedConsumerProperties<>(
			new KinesisConsumerProperties());
	extendedConsumerProperties.setInstanceCount(instanceCount);
	extendedConsumerProperties.setConcurrency(concurrency);

	String name = "test-stream";
	String group = "test-group";

	when(amazonKinesisMock.describeStream(any(DescribeStreamRequest.class)))
			.thenThrow(new ResourceNotFoundException("Stream not found"));

	assertThatThrownBy(() -> provisioner.provisionConsumerDestination(name, group,
			extendedConsumerProperties))
			.isInstanceOf(ProvisioningException.class)
			.hasMessageContaining(
					"The stream [test-stream] was not found and auto creation is disabled.")
			.hasCauseInstanceOf(ResourceNotFoundException.class);

	verify(amazonKinesisMock, times(1))
			.describeStream(any(DescribeStreamRequest.class));

	verify(amazonKinesisMock, never()).createStream(name,
			instanceCount * concurrency);
}
 

/**
 * Internal method to retrieve the stream description and get the shards from AWS.
 * <p>
 * Gets from the internal cache unless not yet created or too old.
 */
protected InternalStreamDescription getStreamDescription(String streamName)
{
    InternalStreamDescription internalStreamDescription = this.streamMap.get(streamName);
    if (internalStreamDescription == null || System.currentTimeMillis() - internalStreamDescription.getCreateTimeStamp() >= MAX_CACHE_AGE_MILLIS) {
        internalStreamDescription = new InternalStreamDescription(streamName);

        DescribeStreamRequest describeStreamRequest = new DescribeStreamRequest();
        describeStreamRequest.setStreamName(streamName);

        // Collect shards from Kinesis
        String exclusiveStartShardId = null;
        List<Shard> shards = new ArrayList<>();
        do {
            describeStreamRequest.setExclusiveStartShardId(exclusiveStartShardId);
            DescribeStreamResult describeStreamResult = clientManager.getClient().describeStream(describeStreamRequest);

            String streamStatus = describeStreamResult.getStreamDescription().getStreamStatus();
            if (!streamStatus.equals("ACTIVE") && !streamStatus.equals("UPDATING")) {
                throw new ResourceNotFoundException("Stream not Active");
            }

            internalStreamDescription.addAllShards(describeStreamResult.getStreamDescription().getShards());

            if (describeStreamResult.getStreamDescription().getHasMoreShards() && (shards.size() > 0)) {
                exclusiveStartShardId = shards.get(shards.size() - 1).getShardId();
            }
            else {
                exclusiveStartShardId = null;
            }
        }
        while (exclusiveStartShardId != null);

        this.streamMap.put(streamName, internalStreamDescription);
    }

    return internalStreamDescription;
}
 

@Test
void testProvisionProducerSuccessfulWithNewStream() {
	AmazonKinesis amazonKinesisMock = mock(AmazonKinesis.class);
	KinesisBinderConfigurationProperties binderProperties = new KinesisBinderConfigurationProperties();
	KinesisStreamProvisioner provisioner = new KinesisStreamProvisioner(
			amazonKinesisMock, binderProperties);
	ExtendedProducerProperties<KinesisProducerProperties> extendedProducerProperties =
			new ExtendedProducerProperties<>(
			new KinesisProducerProperties());

	String name = "test-stream";
	Integer shards = 1;

	DescribeStreamResult describeStreamResult = describeStreamResultWithShards(
			Collections.singletonList(new Shard()));

	when(amazonKinesisMock.describeStream(any(DescribeStreamRequest.class)))
			.thenThrow(new ResourceNotFoundException("I got nothing"))
			.thenReturn(describeStreamResult);

	when(amazonKinesisMock.createStream(name, shards))
			.thenReturn(new CreateStreamResult());

	ProducerDestination destination = provisioner.provisionProducerDestination(name,
			extendedProducerProperties);

	verify(amazonKinesisMock, times(2))
			.describeStream(any(DescribeStreamRequest.class));

	verify(amazonKinesisMock).createStream(name, shards);

	assertThat(destination.getName()).isEqualTo(name);
}
 

private List<Shard> createOrUpdate(String stream, int shards) {
	List<Shard> shardList = new ArrayList<>();
	try {
		shardList = getShardList(stream);
	}
	catch (ResourceNotFoundException ex) {
		if (!this.configurationProperties.isAutoCreateStream()) {
			throw new ProvisioningException(
					"The stream [" + stream + "] was not found and auto creation is disabled.", ex);
		}
		if (logger.isInfoEnabled()) {
			logger.info("Stream '" + stream + "' not found. Create one...");
		}

		this.amazonKinesis.createStream(stream, Math.max(this.configurationProperties.getMinShardCount(), shards));

	}

	int effectiveShardCount = Math.max(this.configurationProperties.getMinShardCount(), shards);

	if ((shardList.size() < effectiveShardCount)
			&& this.configurationProperties.isAutoAddShards()) {
		return updateShardCount(stream, shardList.size(), effectiveShardCount);
	}

	return shardList;
}
 

private List<Shard> getShardList(String stream, int retryCount) {
	List<Shard> shardList = new ArrayList<>();

	if (retryCount > configurationProperties.getDescribeStreamRetries()) {
		ResourceNotFoundException resourceNotFoundException = new ResourceNotFoundException(
				"The stream [" + stream + "] isn't ACTIVE or doesn't exist.");
		resourceNotFoundException.setServiceName("Kinesis");

		throw new ProvisioningException(
				"Kinesis org.springframework.cloud.stream.binder.kinesis.provisioning error",
				resourceNotFoundException);
	}

	ListShardsRequest listShardsRequest = new ListShardsRequest().withStreamName(stream);

	try {
		ListShardsResult listShardsResult = amazonKinesis.listShards(listShardsRequest);

		shardList.addAll(listShardsResult.getShards());

	}
	catch (LimitExceededException limitExceededException) {
		logger.info("Got LimitExceededException when describing stream [" + stream + "]. " + "Backing off for ["
				+ this.configurationProperties.getDescribeStreamBackoff() + "] millis.");

		try {
			Thread.sleep(this.configurationProperties.getDescribeStreamBackoff());
			getShardList(stream, retryCount++);
		}
		catch (InterruptedException ex) {
			Thread.currentThread().interrupt();
			throw new ProvisioningException(
					"The [describeStream] thread for the stream [" + stream + "] has been interrupted.", ex);
		}
	}

	return shardList;
}
 

/**
 * Get metainfo for a Kinesis stream, which contains information about which shards this
 * Kinesis stream possess.
 *
 * <p>This method is using a "full jitter" approach described in AWS's article,
 * <a href="https://www.awsarchitectureblog.com/2015/03/backoff.html">
 *   "Exponential Backoff and Jitter"</a>.
 * This is necessary because concurrent calls will be made by all parallel subtask's fetcher.
 * This jitter backoff approach will help distribute calls across the fetchers over time.
 *
 * @param streamName the stream to describe
 * @param startShardId which shard to start with for this describe operation
 *
 * @return the result of the describe stream operation
 */
protected DescribeStreamResult describeStream(String streamName, @Nullable String startShardId)
		throws InterruptedException {
	final DescribeStreamRequest describeStreamRequest = new DescribeStreamRequest();
	describeStreamRequest.setStreamName(streamName);
	describeStreamRequest.setExclusiveStartShardId(startShardId);

	DescribeStreamResult describeStreamResult = null;

	// Call DescribeStream, with full-jitter backoff (if we get LimitExceededException).
	int attemptCount = 0;
	while (describeStreamResult == null) { // retry until we get a result
		try {
			describeStreamResult = kinesisClient.describeStream(describeStreamRequest);
		} catch (LimitExceededException le) {
			long backoffMillis = fullJitterBackoff(
					describeStreamBaseBackoffMillis,
					describeStreamMaxBackoffMillis,
					describeStreamExpConstant,
					attemptCount++);
			LOG.warn(String.format("Got LimitExceededException when describing stream %s. "
					+ "Backing off for %d millis.", streamName, backoffMillis));
			Thread.sleep(backoffMillis);
		} catch (ResourceNotFoundException re) {
			throw new RuntimeException("Error while getting stream details", re);
		}
	}

	String streamStatus = describeStreamResult.getStreamDescription().getStreamStatus();
	if (!(streamStatus.equals(StreamStatus.ACTIVE.toString())
			|| streamStatus.equals(StreamStatus.UPDATING.toString()))) {
		if (LOG.isWarnEnabled()) {
			LOG.warn(String.format("The status of stream %s is %s ; result of the current "
							+ "describeStream operation will not contain any shard information.",
					streamName, streamStatus));
		}
	}

	return describeStreamResult;
}
 

/**
 * Get metainfo for a Kinesis stream, which contains information about which shards this
 * Kinesis stream possess.
 *
 * <p>This method is using a "full jitter" approach described in AWS's article,
 * <a href="https://www.awsarchitectureblog.com/2015/03/backoff.html">
 *   "Exponential Backoff and Jitter"</a>.
 * This is necessary because concurrent calls will be made by all parallel subtask's fetcher.
 * This jitter backoff approach will help distribute calls across the fetchers over time.
 *
 * @param streamName the stream to describe
 * @param startShardId which shard to start with for this describe operation
 *
 * @return the result of the describe stream operation
 */
protected DescribeStreamResult describeStream(String streamName, @Nullable String startShardId)
		throws InterruptedException {
	final DescribeStreamRequest describeStreamRequest = new DescribeStreamRequest();
	describeStreamRequest.setStreamName(streamName);
	describeStreamRequest.setExclusiveStartShardId(startShardId);

	DescribeStreamResult describeStreamResult = null;

	// Call DescribeStream, with full-jitter backoff (if we get LimitExceededException).
	int attemptCount = 0;
	while (describeStreamResult == null) { // retry until we get a result
		try {
			describeStreamResult = kinesisClient.describeStream(describeStreamRequest);
		} catch (LimitExceededException le) {
			long backoffMillis = fullJitterBackoff(
					describeStreamBaseBackoffMillis,
					describeStreamMaxBackoffMillis,
					describeStreamExpConstant,
					attemptCount++);
			LOG.warn(String.format("Got LimitExceededException when describing stream %s. "
					+ "Backing off for %d millis.", streamName, backoffMillis));
			Thread.sleep(backoffMillis);
		} catch (ResourceNotFoundException re) {
			throw new RuntimeException("Error while getting stream details", re);
		}
	}

	String streamStatus = describeStreamResult.getStreamDescription().getStreamStatus();
	if (!(streamStatus.equals(StreamStatus.ACTIVE.toString())
			|| streamStatus.equals(StreamStatus.UPDATING.toString()))) {
		if (LOG.isWarnEnabled()) {
			LOG.warn(String.format("The status of stream %s is %s ; result of the current "
							+ "describeStream operation will not contain any shard information.",
					streamName, streamStatus));
		}
	}

	return describeStreamResult;
}
 

private void getIterator()
        throws ResourceNotFoundException
{
    GetShardIteratorRequest getShardIteratorRequest = new GetShardIteratorRequest();
    getShardIteratorRequest.setStreamName(split.getStreamName());
    getShardIteratorRequest.setShardId(split.getShardId());

    // Explanation: when we have a sequence number from a prior read or checkpoint, always use it.
    // Otherwise, decide if starting at a timestamp or the trim horizon based on configuration.
    // If starting at a timestamp, use the session variable STARTING_TIMESTAMP when given, otherwise
    // fallback on starting at STARTING_OFFSET_SECONDS from timestamp.
    if (lastReadSequenceNumber == null) {
        if (isIteratorFromTimestamp(session)) {
            getShardIteratorRequest.setShardIteratorType("AT_TIMESTAMP");
            long iteratorStartTimestamp = getIteratorStartTimestamp(session);
            if (iteratorStartTimestamp == 0) {
                long startTimestamp = System.currentTimeMillis() - (getIteratorOffsetSeconds(session) * 1000);
                getShardIteratorRequest.setTimestamp(new Date(startTimestamp));
            }
            else {
                getShardIteratorRequest.setTimestamp(new Date(iteratorStartTimestamp));
            }
        }
        else {
            getShardIteratorRequest.setShardIteratorType("TRIM_HORIZON");
        }
    }
    else {
        getShardIteratorRequest.setShardIteratorType("AFTER_SEQUENCE_NUMBER");
        getShardIteratorRequest.setStartingSequenceNumber(lastReadSequenceNumber);
    }

    GetShardIteratorResult getShardIteratorResult = clientManager.getClient().getShardIterator(getShardIteratorRequest);
    shardIterator = getShardIteratorResult.getShardIterator();
}
 

/**
 * Retrieves the next batch of records from Kinesis using the shard iterator.
 *
 * Most of the time this results in one getRecords call.  However we allow for
 * a call to return an empty list, and we'll try again if we are far enough
 * away from the latest record.
 */
private void getKinesisRecords()
        throws ResourceNotFoundException
{
    // Normally this loop will execute once, but we have to allow for the odd Kinesis
    // behavior, per the docs:
    // A single call to getRecords might return an empty record list, even when the shard contains
    // more records at later sequence numbers
    boolean fetchedRecords = false;
    int attempts = 0;
    while (!fetchedRecords && attempts < fetchAttempts) {
        long now = System.currentTimeMillis();
        if (now - lastReadTime <= sleepTime) {
            try {
                Thread.sleep(now - lastReadTime);
            }
            catch (InterruptedException e) {
                log.error("Sleep interrupted.", e);
            }
        }
        getRecordsRequest = new GetRecordsRequest();
        getRecordsRequest.setShardIterator(shardIterator);
        getRecordsRequest.setLimit(batchSize);

        getRecordsResult = clientManager.getClient().getRecords(getRecordsRequest);
        lastReadTime = System.currentTimeMillis();

        shardIterator = getRecordsResult.getNextShardIterator();
        kinesisRecords = getRecordsResult.getRecords();
        if (kinesisConnectorConfig.isLogBatches()) {
            log.info("Fetched %d records from Kinesis.  MillisBehindLatest=%d", kinesisRecords.size(), getRecordsResult.getMillisBehindLatest());
        }

        fetchedRecords = (kinesisRecords.size() > 0 || getMillisBehindLatest() <= MILLIS_BEHIND_LIMIT);
        attempts++;
    }

    listIterator = kinesisRecords.iterator();
    batchesRead++;
    messagesRead += kinesisRecords.size();
}
 

/**
  * Configures this appender instance and makes it ready for use by the
  * consumers. It validates mandatory parameters and confirms if the configured
  * stream is ready for publishing data yet.
  * 
  * Error details are made available through the fallback handler for this
  * appender
  * 
  * @throws IllegalStateException
  *           if we encounter issues configuring this appender instance
  */
 @Override
 public void activateOptions() {
   if (streamName == null) {
     initializationFailed = true;
     error("Invalid configuration - streamName cannot be null for appender: " + name);
   }

   if (layout == null) {
     initializationFailed = true;
     error("Invalid configuration - No layout for appender: " + name);
   }

   ClientConfiguration clientConfiguration = new ClientConfiguration();
   clientConfiguration = setProxySettingsFromSystemProperties(clientConfiguration);

   clientConfiguration.setMaxErrorRetry(maxRetries);
   clientConfiguration.setRetryPolicy(new RetryPolicy(PredefinedRetryPolicies.DEFAULT_RETRY_CONDITION,
       PredefinedRetryPolicies.DEFAULT_BACKOFF_STRATEGY, maxRetries, true));
   clientConfiguration.setUserAgent(AppenderConstants.USER_AGENT_STRING);

   BlockingQueue<Runnable> taskBuffer = new LinkedBlockingDeque<Runnable>(bufferSize);
   ThreadPoolExecutor threadPoolExecutor = new ThreadPoolExecutor(threadCount, threadCount,
       AppenderConstants.DEFAULT_THREAD_KEEP_ALIVE_SEC, TimeUnit.SECONDS, taskBuffer, new BlockFastProducerPolicy());
   threadPoolExecutor.prestartAllCoreThreads();
   kinesisClient = new AmazonKinesisAsyncClient(new CustomCredentialsProviderChain(), clientConfiguration,
       threadPoolExecutor);

   boolean regionProvided = !Validator.isBlank(region);
   if (!regionProvided) {
     region = AppenderConstants.DEFAULT_REGION;
   }
   if (!Validator.isBlank(endpoint)) {
     if (regionProvided) {
LOGGER
    .warn("Received configuration for both region as well as Amazon Kinesis endpoint. ("
	+ endpoint
	+ ") will be used as endpoint instead of default endpoint for region ("
	+ region + ")");
     }
     kinesisClient.setEndpoint(endpoint,
  AppenderConstants.DEFAULT_SERVICE_NAME, region);
   } else {
     kinesisClient.setRegion(Region.getRegion(Regions.fromName(region)));
   }

   DescribeStreamResult describeResult = null;
   try {
     describeResult = kinesisClient.describeStream(streamName);
     String streamStatus = describeResult.getStreamDescription().getStreamStatus();
     if (!StreamStatus.ACTIVE.name().equals(streamStatus) && !StreamStatus.UPDATING.name().equals(streamStatus)) {
       initializationFailed = true;
       error("Stream " + streamName + " is not ready (in active/updating status) for appender: " + name);
     }
   } catch (ResourceNotFoundException rnfe) {
     initializationFailed = true;
     error("Stream " + streamName + " doesn't exist for appender: " + name, rnfe);
   }

   asyncCallHander = new AsyncPutCallStatsReporter(name);
 }
 

/**
 * Retrieves the next batch of records from Kinesis using the shard iterator.
 * <p>
 * Most of the time this results in one getRecords call.  However we allow for
 * a call to return an empty list, and we'll try again if we are far enough
 * away from the latest record.
 */
private void getKinesisRecords()
        throws ResourceNotFoundException
{
    // Normally this loop will execute once, but we have to allow for the odd Kinesis
    // behavior, per the docs:
    // A single call to getRecords might return an empty record list, even when the shard contains
    // more records at later sequence numbers
    boolean fetchedRecords = false;
    int attempts = 0;
    while (!fetchedRecords && attempts < fetchAttempts) {
        Duration duration = nanosSince(lastReadTime);
        if (duration.toMillis() <= sleepTime) {
            try {
                Thread.sleep(duration.toMillis());
            }
            catch (InterruptedException e) {
                Thread.currentThread().interrupt();
                throw new RuntimeException("thread interrupted");
            }
        }
        getRecordsRequest = new GetRecordsRequest();
        getRecordsRequest.setShardIterator(shardIterator);
        getRecordsRequest.setLimit(batchSize);

        getRecordsResult = clientManager.getClient().getRecords(getRecordsRequest);
        lastReadTime = System.nanoTime();

        shardIterator = getRecordsResult.getNextShardIterator();
        kinesisRecords = getRecordsResult.getRecords();
        if (isLogBatches) {
            log.info("Fetched %d records from Kinesis.  MillisBehindLatest=%d", kinesisRecords.size(), getRecordsResult.getMillisBehindLatest());
        }

        fetchedRecords = (kinesisRecords.size() > 0 || getMillisBehindLatest() <= MILLIS_BEHIND_LIMIT);
        attempts++;
    }

    listIterator = kinesisRecords.iterator();
    batchesRead++;
    messagesRead += kinesisRecords.size();
}