Java Code Examples for com.amazonaws.services.kinesis.model.DescribeStreamRequest#setStreamName()

/**
 * Internal method to retrieve the stream description and get the shards from AWS.
 * <p>
 * Gets from the internal cache unless not yet created or too old.
 */
protected InternalStreamDescription getStreamDescription(String streamName)
{
    InternalStreamDescription internalStreamDescription = this.streamMap.get(streamName);
    if (internalStreamDescription == null || System.currentTimeMillis() - internalStreamDescription.getCreateTimeStamp() >= MAX_CACHE_AGE_MILLIS) {
        internalStreamDescription = new InternalStreamDescription(streamName);

        DescribeStreamRequest describeStreamRequest = new DescribeStreamRequest();
        describeStreamRequest.setStreamName(streamName);

        // Collect shards from Kinesis
        String exclusiveStartShardId = null;
        List<Shard> shards = new ArrayList<>();
        do {
            describeStreamRequest.setExclusiveStartShardId(exclusiveStartShardId);
            DescribeStreamResult describeStreamResult = clientManager.getClient().describeStream(describeStreamRequest);

            String streamStatus = describeStreamResult.getStreamDescription().getStreamStatus();
            if (!streamStatus.equals("ACTIVE") && !streamStatus.equals("UPDATING")) {
                throw new ResourceNotFoundException("Stream not Active");
            }

            internalStreamDescription.addAllShards(describeStreamResult.getStreamDescription().getShards());

            if (describeStreamResult.getStreamDescription().getHasMoreShards() && (shards.size() > 0)) {
                exclusiveStartShardId = shards.get(shards.size() - 1).getShardId();
            }
            else {
                exclusiveStartShardId = null;
            }
        }
        while (exclusiveStartShardId != null);

        this.streamMap.put(streamName, internalStreamDescription);
    }

    return internalStreamDescription;
}
 

private String checkStreamStatus(String streamName)
{
    DescribeStreamRequest describeStreamRequest = new DescribeStreamRequest();
    describeStreamRequest.setStreamName(streamName);

    StreamDescription streamDescription = amazonKinesisClient.describeStream(describeStreamRequest).getStreamDescription();
    return streamDescription.getStreamStatus();
}
 

/**
 * Get metainfo for a Kinesis stream, which contains information about which shards this
 * Kinesis stream possess.
 *
 * <p>This method is using a "full jitter" approach described in AWS's article,
 * <a href="https://www.awsarchitectureblog.com/2015/03/backoff.html">
 *   "Exponential Backoff and Jitter"</a>.
 * This is necessary because concurrent calls will be made by all parallel subtask's fetcher.
 * This jitter backoff approach will help distribute calls across the fetchers over time.
 *
 * @param streamName the stream to describe
 * @param startShardId which shard to start with for this describe operation
 *
 * @return the result of the describe stream operation
 */
protected DescribeStreamResult describeStream(String streamName, @Nullable String startShardId)
		throws InterruptedException {
	final DescribeStreamRequest describeStreamRequest = new DescribeStreamRequest();
	describeStreamRequest.setStreamName(streamName);
	describeStreamRequest.setExclusiveStartShardId(startShardId);

	DescribeStreamResult describeStreamResult = null;

	// Call DescribeStream, with full-jitter backoff (if we get LimitExceededException).
	int attemptCount = 0;
	while (describeStreamResult == null) { // retry until we get a result
		try {
			describeStreamResult = kinesisClient.describeStream(describeStreamRequest);
		} catch (LimitExceededException le) {
			long backoffMillis = fullJitterBackoff(
					describeStreamBaseBackoffMillis,
					describeStreamMaxBackoffMillis,
					describeStreamExpConstant,
					attemptCount++);
			LOG.warn(String.format("Got LimitExceededException when describing stream %s. "
					+ "Backing off for %d millis.", streamName, backoffMillis));
			Thread.sleep(backoffMillis);
		} catch (ResourceNotFoundException re) {
			throw new RuntimeException("Error while getting stream details", re);
		}
	}

	String streamStatus = describeStreamResult.getStreamDescription().getStreamStatus();
	if (!(streamStatus.equals(StreamStatus.ACTIVE.toString())
			|| streamStatus.equals(StreamStatus.UPDATING.toString()))) {
		if (LOG.isWarnEnabled()) {
			LOG.warn(String.format("The status of stream %s is %s ; result of the current "
							+ "describeStream operation will not contain any shard information.",
					streamName, streamStatus));
		}
	}

	return describeStreamResult;
}
 

/**
 * Get metainfo for a Kinesis stream, which contains information about which shards this
 * Kinesis stream possess.
 *
 * <p>This method is using a "full jitter" approach described in AWS's article,
 * <a href="https://www.awsarchitectureblog.com/2015/03/backoff.html">
 *   "Exponential Backoff and Jitter"</a>.
 * This is necessary because concurrent calls will be made by all parallel subtask's fetcher.
 * This jitter backoff approach will help distribute calls across the fetchers over time.
 *
 * @param streamName the stream to describe
 * @param startShardId which shard to start with for this describe operation
 *
 * @return the result of the describe stream operation
 */
protected DescribeStreamResult describeStream(String streamName, @Nullable String startShardId)
		throws InterruptedException {
	final DescribeStreamRequest describeStreamRequest = new DescribeStreamRequest();
	describeStreamRequest.setStreamName(streamName);
	describeStreamRequest.setExclusiveStartShardId(startShardId);

	DescribeStreamResult describeStreamResult = null;

	// Call DescribeStream, with full-jitter backoff (if we get LimitExceededException).
	int attemptCount = 0;
	while (describeStreamResult == null) { // retry until we get a result
		try {
			describeStreamResult = kinesisClient.describeStream(describeStreamRequest);
		} catch (LimitExceededException le) {
			long backoffMillis = fullJitterBackoff(
					describeStreamBaseBackoffMillis,
					describeStreamMaxBackoffMillis,
					describeStreamExpConstant,
					attemptCount++);
			LOG.warn(String.format("Got LimitExceededException when describing stream %s. "
					+ "Backing off for %d millis.", streamName, backoffMillis));
			Thread.sleep(backoffMillis);
		} catch (ResourceNotFoundException re) {
			throw new RuntimeException("Error while getting stream details", re);
		}
	}

	String streamStatus = describeStreamResult.getStreamDescription().getStreamStatus();
	if (!(streamStatus.equals(StreamStatus.ACTIVE.toString())
			|| streamStatus.equals(StreamStatus.UPDATING.toString()))) {
		if (LOG.isWarnEnabled()) {
			LOG.warn(String.format("The status of stream %s is %s ; result of the current "
							+ "describeStream operation will not contain any shard information.",
					streamName, streamStatus));
		}
	}

	return describeStreamResult;
}
 

private static boolean streamActive(AmazonKinesis client, String streamName) {
  try {
    DescribeStreamRequest describeStreamRequest = new DescribeStreamRequest();
    describeStreamRequest.setStreamName(streamName);
    DescribeStreamResult describeStreamResult = client.describeStream(describeStreamRequest);
    String streamStatus = describeStreamResult.getStreamDescription().getStreamStatus();
    if ("ACTIVE".equals(streamStatus)) {
      return true;
    }
  } catch (Exception e) {
    return false;
  }
  return false;
}
 

/**
 * Internal method to retrieve the stream description and get the shards from AWS.
 *
 * Gets from the internal cache unless not yet created or too old.
 *
 * @param streamName Kinesis stream name
 * @return Returns kinesis stream description
 */
protected InternalStreamDescription getStreamDescription(String streamName)
{
    InternalStreamDescription desc = this.streamMap.get(streamName);
    if (desc == null || System.currentTimeMillis() - desc.getCreateTimeStamp() >= MAX_CACHE_AGE_MILLIS) {
        desc = new InternalStreamDescription(streamName);

        DescribeStreamRequest describeStreamRequest = clientManager.getDescribeStreamRequest();
        describeStreamRequest.setStreamName(streamName);

        // Collect shards from Kinesis
        String exclusiveStartShardId = null;
        List<Shard> shards = new ArrayList<>();
        do {
            describeStreamRequest.setExclusiveStartShardId(exclusiveStartShardId);
            DescribeStreamResult describeStreamResult = clientManager.getClient().describeStream(describeStreamRequest);

            String streamStatus = describeStreamResult.getStreamDescription().getStreamStatus();
            if (!streamStatus.equals("ACTIVE") && !streamStatus.equals("UPDATING")) {
                throw new ResourceNotFoundException("Stream not Active");
            }

            desc.addAllShards(describeStreamResult.getStreamDescription().getShards());

            if (describeStreamResult.getStreamDescription().getHasMoreShards() && (shards.size() > 0)) {
                exclusiveStartShardId = shards.get(shards.size() - 1).getShardId();
            }
            else {
                exclusiveStartShardId = null;
            }
        } while (exclusiveStartShardId != null);

        this.streamMap.put(streamName, desc);
    }

    return desc;
}
 

private String checkStreamStatus(String streamName)
{
    DescribeStreamRequest describeStreamRequest = new DescribeStreamRequest();
    describeStreamRequest.setStreamName(streamName);

    StreamDescription streamDescription = amazonKinesisClient.describeStream(describeStreamRequest).getStreamDescription();
    return streamDescription.getStreamStatus();
}
 

/**
 * Helper method to determine if an Amazon Kinesis stream exists.
 * 
 * @param kinesisClient
 *        The {@link AmazonKinesisClient} with Amazon Kinesis read privileges
 * @param streamName
 *        The Amazon Kinesis stream to check for
 * @return true if the Amazon Kinesis stream exists, otherwise return false
 */
private static boolean streamExists(AmazonKinesisClient kinesisClient, String streamName) {
    DescribeStreamRequest describeStreamRequest = new DescribeStreamRequest();
    describeStreamRequest.setStreamName(streamName);
    try {
        kinesisClient.describeStream(describeStreamRequest);
        return true;
    } catch (ResourceNotFoundException e) {
        return false;
    }
}
 

/**
 * Return the state of a Amazon Kinesis stream.
 * 
 * @param kinesisClient
 *        The {@link AmazonKinesisClient} with Amazon Kinesis read privileges
 * @param streamName
 *        The Amazon Kinesis stream to get the state of
 * @return String representation of the Stream state
 */
private static String streamState(AmazonKinesisClient kinesisClient, String streamName) {
    DescribeStreamRequest describeStreamRequest = new DescribeStreamRequest();
    describeStreamRequest.setStreamName(streamName);
    try {
        return kinesisClient.describeStream(describeStreamRequest).getStreamDescription().getStreamStatus();
    } catch (AmazonServiceException e) {
        return null;
    }
}
 

/**
 * Get metainfo for a Kinesis stream, which contains information about which shards this
 * Kinesis stream possess.
 *
 * <p>This method is using a "full jitter" approach described in AWS's article,
 * <a href="https://www.awsarchitectureblog.com/2015/03/backoff.html">
 *   "Exponential Backoff and Jitter"</a>.
 * This is necessary because concurrent calls will be made by all parallel subtask's fetcher.
 * This jitter backoff approach will help distribute calls across the fetchers over time.
 *
 * @param streamName the stream to describe
 * @param startShardId which shard to start with for this describe operation
 *
 * @return the result of the describe stream operation
 */
protected DescribeStreamResult describeStream(String streamName, @Nullable String startShardId)
		throws InterruptedException {
	final DescribeStreamRequest describeStreamRequest = new DescribeStreamRequest();
	describeStreamRequest.setStreamName(streamName);
	describeStreamRequest.setExclusiveStartShardId(startShardId);

	DescribeStreamResult describeStreamResult = null;

	// Call DescribeStream, with full-jitter backoff (if we get LimitExceededException).
	int attemptCount = 0;
	while (describeStreamResult == null) { // retry until we get a result
		try {
			describeStreamResult = kinesisClient.describeStream(describeStreamRequest);
		} catch (LimitExceededException le) {
			long backoffMillis = fullJitterBackoff(
					describeStreamBaseBackoffMillis,
					describeStreamMaxBackoffMillis,
					describeStreamExpConstant,
					attemptCount++);
			LOG.warn(String.format("Got LimitExceededException when describing stream %s. "
					+ "Backing off for %d millis.", streamName, backoffMillis));
			Thread.sleep(backoffMillis);
		} catch (ResourceNotFoundException re) {
			throw new RuntimeException("Error while getting stream details", re);
		}
	}

	String streamStatus = describeStreamResult.getStreamDescription().getStreamStatus();
	if (!(streamStatus.equals(StreamStatus.ACTIVE.toString())
			|| streamStatus.equals(StreamStatus.UPDATING.toString()))) {
		if (LOG.isWarnEnabled()) {
			LOG.warn(String.format("The status of stream %s is %s ; result of the current "
							+ "describeStream operation will not contain any shard information.",
					streamName, streamStatus));
		}
	}

	return describeStreamResult;
}
 

/**
 * Helper method to determine if an Amazon Kinesis stream exists.
 * 
 * @param kinesisClient
 *        The {@link AmazonKinesisClient} with Amazon Kinesis read privileges
 * @param streamName
 *        The Amazon Kinesis stream to check for
 * @return true if the Amazon Kinesis stream exists, otherwise return false
 */
private static boolean streamExists(AmazonKinesisClient kinesisClient, String streamName) {
    DescribeStreamRequest describeStreamRequest = new DescribeStreamRequest();
    describeStreamRequest.setStreamName(streamName);
    try {
        kinesisClient.describeStream(describeStreamRequest);
        return true;
    } catch (ResourceNotFoundException e) {
        return false;
    }
}
 

/**
 * Return the state of a Amazon Kinesis stream.
 * 
 * @param kinesisClient
 *        The {@link AmazonKinesisClient} with Amazon Kinesis read privileges
 * @param streamName
 *        The Amazon Kinesis stream to get the state of
 * @return String representation of the Stream state
 */
private static String streamState(AmazonKinesisClient kinesisClient, String streamName) {
    DescribeStreamRequest describeStreamRequest = new DescribeStreamRequest();
    describeStreamRequest.setStreamName(streamName);
    try {
        return kinesisClient.describeStream(describeStreamRequest).getStreamDescription().getStreamStatus();
    } catch (AmazonServiceException e) {
        return null;
    }
}