com.amazonaws.services.kinesis.model.StreamDescription Java Examples

@Test
public void shouldListAllShards() throws Exception {
  Shard shard1 = new Shard().withShardId(SHARD_1);
  Shard shard2 = new Shard().withShardId(SHARD_2);
  Shard shard3 = new Shard().withShardId(SHARD_3);
  when(kinesis.describeStream(STREAM, null))
      .thenReturn(
          new DescribeStreamResult()
              .withStreamDescription(
                  new StreamDescription().withShards(shard1, shard2).withHasMoreShards(true)));
  when(kinesis.describeStream(STREAM, SHARD_2))
      .thenReturn(
          new DescribeStreamResult()
              .withStreamDescription(
                  new StreamDescription().withShards(shard3).withHasMoreShards(false)));

  List<Shard> shards = underTest.listShards(STREAM);

  assertThat(shards).containsOnly(shard1, shard2, shard3);
}
 

@Override
public DescribeStreamResult describeStream(DescribeStreamRequest describeStreamRequest)
        throws AmazonClientException
{
    InternalStream theStream = this.getStream(describeStreamRequest.getStreamName());
    if (theStream == null) {
        throw new AmazonClientException("This stream does not exist!");
    }

    StreamDescription desc = new StreamDescription();
    desc = desc.withStreamName(theStream.getStreamName()).withStreamStatus(theStream.getStreamStatus()).withStreamARN(theStream.getStreamAmazonResourceName());

    if (describeStreamRequest.getExclusiveStartShardId() == null || describeStreamRequest.getExclusiveStartShardId().isEmpty()) {
        desc.setShards(this.getShards(theStream));
        desc.setHasMoreShards(false);
    }
    else {
        // Filter from given shard Id, or may not have any more
        String startId = describeStreamRequest.getExclusiveStartShardId();
        desc.setShards(this.getShards(theStream, startId));
        desc.setHasMoreShards(false);
    }

    DescribeStreamResult result = new DescribeStreamResult();
    result = result.withStreamDescription(desc);
    return result;
}
 

private String checkStreamStatus(String streamName)
{
    DescribeStreamRequest describeStreamRequest = new DescribeStreamRequest();
    describeStreamRequest.setStreamName(streamName);

    StreamDescription streamDescription = amazonKinesisClient.describeStream(describeStreamRequest).getStreamDescription();
    return streamDescription.getStreamStatus();
}
 

private List<Shard> describeStream(String stream) {
	String exclusiveStartShardId = null;

	DescribeStreamRequest describeStreamRequest = new DescribeStreamRequest()
			.withStreamName(stream);

	List<Shard> shardList = new ArrayList<>();

	while (true) {
		DescribeStreamResult describeStreamResult;

		describeStreamRequest.withExclusiveStartShardId(exclusiveStartShardId);
		describeStreamResult = AMAZON_KINESIS.describeStream(describeStreamRequest);
		StreamDescription streamDescription = describeStreamResult
				.getStreamDescription();
		if (StreamStatus.ACTIVE.toString()
				.equals(streamDescription.getStreamStatus())) {
			shardList.addAll(streamDescription.getShards());

			if (streamDescription.getHasMoreShards()) {
				exclusiveStartShardId = shardList.get(shardList.size() - 1)
						.getShardId();
				continue;
			}
			else {
				return shardList;
			}
		}
		try {
			Thread.sleep(100);
		}
		catch (InterruptedException ex) {
			Thread.currentThread().interrupt();
			throw new IllegalStateException(ex);
		}
	}
}
 

public List<Shard> listShards(final String streamName) throws TransientKinesisException {
  return wrapExceptions(
      () -> {
        List<Shard> shards = Lists.newArrayList();
        String lastShardId = null;

        // DescribeStream has limits that can be hit fairly easily if we are attempting
        // to configure multiple KinesisIO inputs in the same account. Retry up to
        // LIST_SHARDS_DESCRIBE_STREAM_MAX_ATTEMPTS times if we end up hitting that limit.
        //
        // Only pass the wrapped exception up once that limit is reached. Use FluentBackoff
        // to implement the retry policy.
        FluentBackoff retryBackoff =
            FluentBackoff.DEFAULT
                .withMaxRetries(LIST_SHARDS_DESCRIBE_STREAM_MAX_ATTEMPTS)
                .withInitialBackoff(LIST_SHARDS_DESCRIBE_STREAM_INITIAL_BACKOFF);
        StreamDescription description = null;
        do {
          BackOff backoff = retryBackoff.backoff();
          Sleeper sleeper = Sleeper.DEFAULT;
          while (true) {
            try {
              description =
                  kinesis.describeStream(streamName, lastShardId).getStreamDescription();
              break;
            } catch (LimitExceededException exc) {
              if (!BackOffUtils.next(sleeper, backoff)) {
                throw exc;
              }
            }
          }

          shards.addAll(description.getShards());
          lastShardId = shards.get(shards.size() - 1).getShardId();
        } while (description.getHasMoreShards());

        return shards;
      });
}
 

@Override
public DescribeStreamResult describeStream(String streamName, String exclusiveStartShardId) {
  if (rateLimitDescribeStream-- > 0) {
    throw new LimitExceededException("DescribeStream rate limit exceeded");
  }
  int nextShardId = 0;
  if (exclusiveStartShardId != null) {
    nextShardId = parseInt(exclusiveStartShardId) + 1;
  }
  boolean hasMoreShards = nextShardId + 1 < shardedData.size();

  List<Shard> shards = new ArrayList<>();
  if (nextShardId < shardedData.size()) {
    shards.add(new Shard().withShardId(Integer.toString(nextShardId)));
  }

  HttpResponse response = new HttpResponse(null, null);
  response.setStatusCode(200);
  DescribeStreamResult result = new DescribeStreamResult();
  result.setSdkHttpMetadata(SdkHttpMetadata.from(response));
  result.withStreamDescription(
      new StreamDescription()
          .withHasMoreShards(hasMoreShards)
          .withShards(shards)
          .withStreamName(streamName));
  return result;
}
 

public static long getShardCount(
  ClientConfiguration awsClientConfig,
  KinesisConfigBean conf, String streamName
) throws StageException {
  AmazonKinesis kinesisClient = getKinesisClient(awsClientConfig, conf);

  try {
    long numShards = 0;
    String lastShardId = null;
    StreamDescription description;
    do {
      if (lastShardId == null) {
        description = kinesisClient.describeStream(streamName).getStreamDescription();
      } else {
        description = kinesisClient.describeStream(streamName, lastShardId).getStreamDescription();
      }

      for (Shard shard : description.getShards()) {
        if (shard.getSequenceNumberRange().getEndingSequenceNumber() == null) {
          // Then this shard is open, so we should count it. Shards with an ending sequence number
          // are closed and cannot be written to, so we skip counting them.
          ++numShards;
        }
      }

      int pageSize = description.getShards().size();
      lastShardId = description.getShards().get(pageSize - 1).getShardId();

    } while (description.getHasMoreShards());

    LOG.debug("Connected successfully to stream: '{}' with '{}' shards.", streamName, numShards);

    return numShards;
  } finally {
    kinesisClient.shutdown();
  }
}
 

/**
 * Get the last shard Id in the given stream
 * In preview mode, kinesis source uses the last Shard Id to get records from kinesis
 * @param awsClientConfig generic AWS client configuration
 * @param conf
 * @param streamName
 */
public static String getLastShardId(
    ClientConfiguration awsClientConfig,
    KinesisConfigBean conf,
    String streamName
) throws StageException {
  AmazonKinesis kinesisClient = getKinesisClient(awsClientConfig, conf);

  String lastShardId = null;
  try {
    StreamDescription description;
    do {
      if (lastShardId == null) {
        description = kinesisClient.describeStream(streamName).getStreamDescription();
      } else {
        description = kinesisClient.describeStream(streamName, lastShardId).getStreamDescription();
      }

      int pageSize = description.getShards().size();
      lastShardId = description.getShards().get(pageSize - 1).getShardId();

    } while (description.getHasMoreShards());

    return lastShardId;

  } finally {
    kinesisClient.shutdown();
  }
}
 

@Override
public DescribeStreamResult describeStream(DescribeStreamRequest describeStreamRequest) throws AmazonServiceException, AmazonClientException
{
    InternalStream theStream = this.getStream(describeStreamRequest.getStreamName());
    if (theStream != null) {
        StreamDescription desc = new StreamDescription();
        desc = desc.withStreamName(theStream.getStreamName()).withStreamStatus(theStream.getStreamStatus()).withStreamARN(theStream.getStreamARN());

        if (describeStreamRequest.getExclusiveStartShardId() == null || describeStreamRequest.getExclusiveStartShardId().isEmpty()) {
            desc.setShards(this.getShards(theStream));
            desc.setHasMoreShards(false);
        }
        else {
            // Filter from given shard Id, or may not have any more
            String startId = describeStreamRequest.getExclusiveStartShardId();
            desc.setShards(this.getShards(theStream, startId));
            desc.setHasMoreShards(false);
        }

        DescribeStreamResult result = new DescribeStreamResult();
        result = result.withStreamDescription(desc);
        return result;
    }
    else {
        throw new AmazonClientException("This stream does not exist!");
    }
}
 

private String checkStreamStatus(String streamName)
{
    DescribeStreamRequest describeStreamRequest = new DescribeStreamRequest();
    describeStreamRequest.setStreamName(streamName);

    StreamDescription streamDescription = amazonKinesisClient.describeStream(describeStreamRequest).getStreamDescription();
    return streamDescription.getStreamStatus();
}
 

public static Map<String,List<DataStreamVH>> fetchDataStreamInfo(BasicSessionCredentials temporaryCredentials, String skipRegions,String accountId,String accountName) {
    
    Map<String,List<DataStreamVH>> dataStream = new LinkedHashMap<>();
    AmazonKinesis amazonKinesis;
    String expPrefix = InventoryConstants.ERROR_PREFIX_CODE+accountId + "\",\"Message\": \"Exception in fetching info for resource\" ,\"type\": \"datastream\"" ;
    for(Region region : RegionUtils.getRegions()) { 
        try{
            if(!skipRegions.contains(region.getName())){
                amazonKinesis = AmazonKinesisClientBuilder.standard().withCredentials(new AWSStaticCredentialsProvider(temporaryCredentials)).withRegion(region.getName()).build();
                ListStreamsResult listStreamsResult = amazonKinesis.listStreams();
                List<String> streamNamesTemp = listStreamsResult.getStreamNames();
                List<String> streamNames = new ArrayList<>(streamNamesTemp);
                while (listStreamsResult.isHasMoreStreams() && !streamNamesTemp.isEmpty()) {
                    listStreamsResult = amazonKinesis.listStreams(streamNamesTemp.get(streamNamesTemp.size() - 1));
                    streamNamesTemp = listStreamsResult.getStreamNames();
                    streamNames.addAll(streamNamesTemp);
                }
                
                List<DataStreamVH> dataStreamList = new ArrayList<>();
                for(String streamName : streamNames) {
                    StreamDescription streamDescription = amazonKinesis.describeStream(new DescribeStreamRequest().withStreamName(streamName)).getStreamDescription();
                    ListTagsForStreamResult listTagsForStreamResult = amazonKinesis.listTagsForStream(new ListTagsForStreamRequest().withStreamName(streamName));
                    List<com.amazonaws.services.kinesis.model.Tag> tagsTemp = listTagsForStreamResult.getTags();
                    List<com.amazonaws.services.kinesis.model.Tag> tags = new ArrayList<>(tagsTemp);
                    while (listTagsForStreamResult.isHasMoreTags() && !tagsTemp.isEmpty()) {
                        listTagsForStreamResult = amazonKinesis.listTagsForStream(new ListTagsForStreamRequest().withExclusiveStartTagKey(tagsTemp.get(tagsTemp.size() - 1).getKey()));
                        tagsTemp = listTagsForStreamResult.getTags();
                        tags.addAll(tagsTemp);
                    }
                    dataStreamList.add(new DataStreamVH(streamDescription, tags));
                }
                if( !dataStreamList.isEmpty() ) {
                    log.debug(InventoryConstants.ACCOUNT + accountId +" Type : datastream "+region.getName() + " >> "+dataStreamList.size());
                    dataStream.put(accountId+delimiter+accountName+delimiter+region.getName(),dataStreamList);
                }
            }
        } catch(Exception e){
            log.warn(expPrefix+ region.getName()+InventoryConstants.ERROR_CAUSE +e.getMessage()+"\"}");
            ErrorManageUtil.uploadError(accountId, region.getName(),"datastream",e.getMessage());
        }
    }
    return dataStream;
}
 

public DataStreamVH(StreamDescription streamDescription, List<Tag> tags) {
    super();
    this.streamDescription = streamDescription;
    this.tags = tags;
}
 

public StreamDescription getStreamDescription() {
    return streamDescription;
}
 

public void setStreamDescription(StreamDescription streamDescription) {
    this.streamDescription = streamDescription;
}
 

private static DescribeStreamResult describeStreamResultWithShards(
		List<Shard> shards) {
	return new DescribeStreamResult().withStreamDescription(new StreamDescription()
			.withShards(shards).withStreamStatus(StreamStatus.ACTIVE)
			.withHasMoreShards(Boolean.FALSE));
}