com.amazonaws.services.kinesis.AmazonKinesisClient Java Examples

@Inject
public KinesisClientManager(KinesisConfig config)
{
    if (!isNullOrEmpty(config.getAccessKey()) && !isNullOrEmpty(config.getSecretKey())) {
        BasicAWSCredentials awsCredentials = new BasicAWSCredentials(config.getAccessKey(), config.getSecretKey());
        this.client = new AmazonKinesisClient(awsCredentials);
        this.amazonS3Client = new AmazonS3Client(awsCredentials);
        this.dynamoDbClient = new AmazonDynamoDBClient(awsCredentials);
    }
    else {
        DefaultAWSCredentialsProviderChain defaultChain = new DefaultAWSCredentialsProviderChain();
        this.client = new AmazonKinesisClient(defaultChain);
        this.amazonS3Client = new AmazonS3Client(defaultChain);
        this.dynamoDbClient = new AmazonDynamoDBClient(defaultChain);
    }

    this.client.setEndpoint("kinesis." + config.getAwsRegion() + ".amazonaws.com");
    this.dynamoDbClient.setEndpoint("dynamodb." + config.getAwsRegion() + ".amazonaws.com");
}
 

@Override
public void prepare(Object configurationObject) {
  // Connect to Kinesis
  synchronized (this) {
    // Create the credentials Object
    AWSCredentials credentials = new BasicAWSCredentials(config.getKey(), config.getSecretKey());

    ClientConfiguration clientConfig = new ClientConfiguration();
    clientConfig.setProtocol(Protocol.valueOf(config.getProtocol().toString()));

    this.client = new AmazonKinesisClient(credentials, clientConfig);
    if (StringUtils.isNotEmpty(config.getRegion()))
      this.client.setRegion(Region.getRegion(Regions.fromName(config.getRegion())));
  }
  streamNames = this.config.getStreams();
  executor = Executors.newFixedThreadPool(streamNames.size());
}
 

@Override
public void prepare(Object configurationObject) {
  // Connect to Kinesis
  synchronized (this) {
    // Create the credentials Object
    AWSCredentials credentials = new BasicAWSCredentials(config.getKey(), config.getSecretKey());

    ClientConfiguration clientConfig = new ClientConfiguration();
    clientConfig.setProtocol(Protocol.valueOf(config.getProtocol().toString()));

    this.client = new AmazonKinesisClient(credentials, clientConfig);
    if (StringUtils.isNotEmpty(config.getRegion())) {
      this.client.setRegion(Region.getRegion(Regions.fromName(config.getRegion())));
    }
  }
  executor = Executors.newSingleThreadExecutor();

}
 

@Inject
KinesisClientManager(KinesisConnectorConfig kinesisConnectorConfig)
{
    log.info("Creating new client for Consumer");
    if (nonEmpty(kinesisConnectorConfig.getAccessKey()) && nonEmpty(kinesisConnectorConfig.getSecretKey())) {
        this.kinesisAwsCredentials = new KinesisAwsCredentials(kinesisConnectorConfig.getAccessKey(), kinesisConnectorConfig.getSecretKey());
        this.client = new AmazonKinesisClient(this.kinesisAwsCredentials);
        this.amazonS3Client = new AmazonS3Client(this.kinesisAwsCredentials);
        this.dynamoDBClient = new AmazonDynamoDBClient(this.kinesisAwsCredentials);
    }
    else {
        this.kinesisAwsCredentials = null;
        DefaultAWSCredentialsProviderChain defaultChain = new DefaultAWSCredentialsProviderChain();
        this.client = new AmazonKinesisClient(defaultChain);
        this.amazonS3Client = new AmazonS3Client(defaultChain);
        this.dynamoDBClient = new AmazonDynamoDBClient(defaultChain);
    }

    this.client.setEndpoint("kinesis." + kinesisConnectorConfig.getAwsRegion() + ".amazonaws.com");
    this.dynamoDBClient.setEndpoint("dynamodb." + kinesisConnectorConfig.getAwsRegion() + ".amazonaws.com");
}
 

/**
 * Gets a list of all Amazon Kinesis streams
 * 
 * @param kinesisClient
 *        The {@link AmazonKinesisClient} with Amazon Kinesis read privileges
 * @return list of Amazon Kinesis streams
 */
public static List<String> listAllStreams(AmazonKinesisClient kinesisClient) {

    ListStreamsRequest listStreamsRequest = new ListStreamsRequest();
    listStreamsRequest.setLimit(10);
    ListStreamsResult listStreamsResult = kinesisClient.listStreams(listStreamsRequest);
    List<String> streamNames = listStreamsResult.getStreamNames();
    while (listStreamsResult.isHasMoreStreams()) {
        if (streamNames.size() > 0) {
            listStreamsRequest.setExclusiveStartStreamName(streamNames.get(streamNames.size() - 1));
        }

        listStreamsResult = kinesisClient.listStreams(listStreamsRequest);
        streamNames.addAll(listStreamsResult.getStreamNames());
    }
    return streamNames;
}
 

/**
 * Gets a list of all Amazon Kinesis streams
 * 
 * @param kinesisClient
 *        The {@link AmazonKinesisClient} with Amazon Kinesis read privileges
 * @return list of Amazon Kinesis streams
 */
public static List<String> listAllStreams(AmazonKinesisClient kinesisClient) {

    ListStreamsRequest listStreamsRequest = new ListStreamsRequest();
    listStreamsRequest.setLimit(10);
    ListStreamsResult listStreamsResult = kinesisClient.listStreams(listStreamsRequest);
    List<String> streamNames = listStreamsResult.getStreamNames();
    while (listStreamsResult.isHasMoreStreams()) {
        if (streamNames.size() > 0) {
            listStreamsRequest.setExclusiveStartStreamName(streamNames.get(streamNames.size() - 1));
        }

        listStreamsResult = kinesisClient.listStreams(listStreamsRequest);
        streamNames.addAll(listStreamsResult.getStreamNames());
    }
    return streamNames;
}
 

/**
 * Creates a new StreamSource.
 * 
 * @param config
 *        Configuration to determine which stream to put records to and get {@link AWSCredentialsProvider}
 * @param inputFile
 *        File containing record data to emit on each line
 * @param loopOverStreamSource
 *        Loop over the stream source to continually put records
 */
public StreamSource(KinesisConnectorConfiguration config, String inputFile, boolean loopOverStreamSource) {
    this.config = config;
    this.inputFile = inputFile;
    this.loopOverInputFile = loopOverStreamSource;
    this.objectMapper = new ObjectMapper();
    kinesisClient = new AmazonKinesisClient(config.AWS_CREDENTIALS_PROVIDER);
    kinesisClient.setRegion(RegionUtils.getRegion(config.REGION_NAME));
    if (config.KINESIS_ENDPOINT != null) {
        kinesisClient.setEndpoint(config.KINESIS_ENDPOINT);
    }
    KinesisUtils.createInputStream(config);
}
 

public S3ManifestEmitter(KinesisConnectorConfiguration configuration) {
    super(configuration);
    manifestStream = configuration.KINESIS_OUTPUT_STREAM;
    kinesisClient = new AmazonKinesisClient(configuration.AWS_CREDENTIALS_PROVIDER);
    if (configuration.KINESIS_ENDPOINT != null) {
        kinesisClient.setEndpoint(configuration.KINESIS_ENDPOINT);
    }
}
 

public static void createStream(AmazonKinesisClient client, String streamName) throws Exception {

        client.createStream(streamName, 1);

        Waiter<DescribeStreamRequest> waiter = client.waiters().streamExists();
        DescribeStreamRequest request = new DescribeStreamRequest().withStreamName(streamName);
        Assert.assertNotNull("Cannot obtain stream description", request);
        Future<Void> future = waiter.runAsync(new WaiterParameters<DescribeStreamRequest>(request), new NoOpWaiterHandler());
        future.get(1, TimeUnit.MINUTES);
    }
 

/**
 * Create client using aws credentials provider. This is the preferred way for creating clients
 */
@Override
protected AmazonKinesisClient createClient(final ProcessContext context, final AWSCredentialsProvider credentialsProvider, final ClientConfiguration config) {
    getLogger().info("Creating client using aws credentials provider");

    return new AmazonKinesisClient(credentialsProvider, config);
}
 

public static void assertNoStaleStreams(AmazonKinesisClient client, String when) {
    List<String> staleInstances = client.listStreams().getStreamNames().stream() //
    .filter(s -> !s.startsWith(KinesisIntegrationTest.class.getSimpleName()) ||
            System.currentTimeMillis() - AWSUtils.toEpochMillis(s) > AWSUtils.HOUR || !client.describeStream(s).getStreamDescription().getStreamStatus().equals("DELETING")) //
    .collect(Collectors.toList());
    Assert.assertEquals(String.format("Found stale Kinesis streams %s running the test: %s", when, staleInstances), 0, staleInstances.size());
}
 

/**
 * Create client using AWSCredentails
 *
 * @deprecated use {@link #createClient(ProcessContext, AWSCredentialsProvider, ClientConfiguration)} instead
 */
@Override
protected AmazonKinesisClient createClient(final ProcessContext context, final AWSCredentials credentials, final ClientConfiguration config) {
    getLogger().info("Creating client using aws credentials");

    return new AmazonKinesisClient(credentials, config);
}
 

public static AmazonKinesisClient createKinesisClient() {
    BasicCredentialsProvider credentials = BasicCredentialsProvider.standard();
    AmazonKinesisClient client = !credentials.isValid() ? null : (AmazonKinesisClient)
            AmazonKinesisClientBuilder.standard()
            .withCredentials(credentials)
            .withRegion("eu-west-1").build();
    return client;
}
 

/**
 * Creates the Amazon Kinesis stream specified by config.KINESIS_INPUT_STREAM
 * 
 * @param config
 *        The configuration with the specified input stream name and {@link AWSCredentialsProvider}
 * @param shardCount
 *        The shard count to create the stream with
 */
public static void createInputStream(KinesisConnectorConfiguration config) {
    AmazonKinesisClient kinesisClient = new AmazonKinesisClient(config.AWS_CREDENTIALS_PROVIDER);
    kinesisClient.setRegion(RegionUtils.getRegion(config.REGION_NAME));
    if (config.KINESIS_ENDPOINT != null) {
        kinesisClient.setEndpoint(config.KINESIS_ENDPOINT);
    }
    createAndWaitForStreamToBecomeAvailable(kinesisClient,
            config.KINESIS_INPUT_STREAM,
            config.KINESIS_INPUT_STREAM_SHARD_COUNT);
}
 

/**
 * Creates the Amazon Kinesis stream specified by config.KINESIS_OUTPUT_STREAM.
 * 
 * @param config
 *        The configuration with the specified output stream name and {@link AWSCredentialsProvider}
 * @param shardCount
 *        The shard count to create the stream with
 */
public static void createOutputStream(KinesisConnectorConfiguration config) {
    AmazonKinesisClient kinesisClient = new AmazonKinesisClient(config.AWS_CREDENTIALS_PROVIDER);
    kinesisClient.setRegion(RegionUtils.getRegion(config.REGION_NAME));
    if (config.KINESIS_ENDPOINT != null) {
        kinesisClient.setEndpoint(config.KINESIS_ENDPOINT);
    }
    createAndWaitForStreamToBecomeAvailable(kinesisClient,
            config.KINESIS_OUTPUT_STREAM,
            config.KINESIS_OUTPUT_STREAM_SHARD_COUNT);
}
 

/**
 * Helper method to determine if an Amazon Kinesis stream exists.
 * 
 * @param kinesisClient
 *        The {@link AmazonKinesisClient} with Amazon Kinesis read privileges
 * @param streamName
 *        The Amazon Kinesis stream to check for
 * @return true if the Amazon Kinesis stream exists, otherwise return false
 */
private static boolean streamExists(AmazonKinesisClient kinesisClient, String streamName) {
    DescribeStreamRequest describeStreamRequest = new DescribeStreamRequest();
    describeStreamRequest.setStreamName(streamName);
    try {
        kinesisClient.describeStream(describeStreamRequest);
        return true;
    } catch (ResourceNotFoundException e) {
        return false;
    }
}
 

/**
 * Return the state of a Amazon Kinesis stream.
 * 
 * @param kinesisClient
 *        The {@link AmazonKinesisClient} with Amazon Kinesis read privileges
 * @param streamName
 *        The Amazon Kinesis stream to get the state of
 * @return String representation of the Stream state
 */
private static String streamState(AmazonKinesisClient kinesisClient, String streamName) {
    DescribeStreamRequest describeStreamRequest = new DescribeStreamRequest();
    describeStreamRequest.setStreamName(streamName);
    try {
        return kinesisClient.describeStream(describeStreamRequest).getStreamDescription().getStreamStatus();
    } catch (AmazonServiceException e) {
        return null;
    }
}
 

/**
 * Deletes the input stream specified by config.KINESIS_INPUT_STREAM
 * 
 * @param config
 *        The configuration containing the stream name and {@link AWSCredentialsProvider}
 */
public static void deleteInputStream(KinesisConnectorConfiguration config) {
    AmazonKinesisClient kinesisClient = new AmazonKinesisClient(config.AWS_CREDENTIALS_PROVIDER);
    kinesisClient.setRegion(RegionUtils.getRegion(config.REGION_NAME));
    if (config.KINESIS_ENDPOINT != null) {
        kinesisClient.setEndpoint(config.KINESIS_ENDPOINT);
    }
    deleteStream(kinesisClient, config.KINESIS_INPUT_STREAM);
}
 

/**
 * Deletes the output stream specified by config.KINESIS_OUTPUT_STREAM
 * 
 * @param config
 *        The configuration containing the stream name and {@link AWSCredentialsProvider}
 */
public static void deleteOutputStream(KinesisConnectorConfiguration config) {
    AmazonKinesisClient kinesisClient = new AmazonKinesisClient(config.AWS_CREDENTIALS_PROVIDER);
    kinesisClient.setRegion(RegionUtils.getRegion(config.REGION_NAME));
    if (config.KINESIS_ENDPOINT != null) {
        kinesisClient.setEndpoint(config.KINESIS_ENDPOINT);
    }
    deleteStream(kinesisClient, config.KINESIS_OUTPUT_STREAM);
}
 

/**
 * Return the state of a Amazon Kinesis stream.
 * 
 * @param kinesisClient
 *        The {@link AmazonKinesisClient} with Amazon Kinesis read privileges
 * @param streamName
 *        The Amazon Kinesis stream to get the state of
 * @return String representation of the Stream state
 */
private static String streamState(AmazonKinesisClient kinesisClient, String streamName) {
    DescribeStreamRequest describeStreamRequest = new DescribeStreamRequest();
    describeStreamRequest.setStreamName(streamName);
    try {
        return kinesisClient.describeStream(describeStreamRequest).getStreamDescription().getStreamStatus();
    } catch (AmazonServiceException e) {
        return null;
    }
}
 

/**
 * Deletes an Amazon Kinesis stream if it exists.
 * 
 * @param kinesisClient
 *        The {@link AmazonKinesisClient} with Amazon Kinesis read and write privileges
 * @param streamName
 *        The Amazon Kinesis stream to delete
 */
public static void deleteStream(AmazonKinesisClient kinesisClient, String streamName) {
    if (streamExists(kinesisClient, streamName)) {
        DeleteStreamRequest deleteStreamRequest = new DeleteStreamRequest();
        deleteStreamRequest.setStreamName(streamName);
        kinesisClient.deleteStream(deleteStreamRequest);
        LOG.info("Deleting stream " + streamName);
    } else {
        LOG.warn("Stream " + streamName + " does not exist");
    }
}
 

@Test
public void testGetRecordsRetry() throws Exception {
	Properties kinesisConsumerConfig = new Properties();
	kinesisConsumerConfig.setProperty(ConsumerConfigConstants.AWS_REGION, "us-east-1");

	final GetRecordsResult expectedResult = new GetRecordsResult();
	MutableInt retries = new MutableInt();
	final Throwable[] retriableExceptions = new Throwable[] {
		new AmazonKinesisException("mock"),
	};

	AmazonKinesisClient mockClient = mock(AmazonKinesisClient.class);
	Mockito.when(mockClient.getRecords(any())).thenAnswer(new Answer<GetRecordsResult>() {
		@Override
		public GetRecordsResult answer(InvocationOnMock invocation) throws Throwable{
			if (retries.intValue() < retriableExceptions.length) {
				retries.increment();
				throw retriableExceptions[retries.intValue() - 1];
			}
			return expectedResult;
		}
	});

	KinesisProxy kinesisProxy = new KinesisProxy(kinesisConsumerConfig);
	Whitebox.getField(KinesisProxy.class, "kinesisClient").set(kinesisProxy, mockClient);

	GetRecordsResult result = kinesisProxy.getRecords("fakeShardIterator", 1);
	assertEquals(retriableExceptions.length, retries.intValue());
	assertEquals(expectedResult, result);
}
 

private void process(){
    amazonKinesisClient = new AmazonKinesisClient(new SampleAWSCredentialProvider());
    amazonKinesisClient.setEndpoint(kinesisEndpoint);
    while(true) {
        genearateMessage();
        try {
            Thread.sleep(5000);
        } catch (InterruptedException e) {
            System.exit(-1);
        }
    }
}
 

public BatchedClickEventsToKinesis(BlockingQueue<ClickEvent> inputQueue) {
    super(inputQueue);
    kinesis = new AmazonKinesisClient().withRegion(
            Regions.fromName(REGION));
    entries = new ArrayList<>();
    dataSize = 0;
}
 

public AggregatingClickEventsToKinesis(
        BlockingQueue<ClickEvent> inputQueue) {
    super(inputQueue);
    kinesis = new AmazonKinesisClient().withRegion(
            Regions.fromName(REGION));
    reset();
}
 

protected MetricsEmittingBasicClickEventsToKinesis(
        BlockingQueue<ClickEvent> inputQueue) {
    super(inputQueue);
    kinesis = new AmazonKinesisClient().withRegion(
            Regions.fromName(REGION));
    cw = new AmazonCloudWatchClient().withRegion(Regions.fromName(REGION));
}
 

/**
 * @param name The name of the client, used for debugging purposes
 * @param streamName The name of the stream to send data to
 * @param threads The number of threads to put in the pool
 * @param region The region that the kinesis stream is in
 */
public ProducerClient(String name, String streamName,
		int threads, Region region) {
	
	kinesisClient = new AmazonKinesisClient();
	kinesisClient.setRegion(region);
	
	eventsQueue = new LinkedBlockingQueue<Event>();

	this.name = name;
	this.canRun = new AtomicBoolean(true);
	this.threads = threads;
	this.streamName = streamName;

}
 

/**
 * Deletes an Amazon Kinesis stream if it exists.
 * 
 * @param kinesisClient
 *        The {@link AmazonKinesisClient} with Amazon Kinesis read and write privileges
 * @param streamName
 *        The Amazon Kinesis stream to delete
 */
public static void deleteStream(AmazonKinesisClient kinesisClient, String streamName) {
    if (streamExists(kinesisClient, streamName)) {
        DeleteStreamRequest deleteStreamRequest = new DeleteStreamRequest();
        deleteStreamRequest.setStreamName(streamName);
        kinesisClient.deleteStream(deleteStreamRequest);
        LOG.info("Deleting stream " + streamName);
    } else {
        LOG.warn("Stream " + streamName + " does not exist");
    }
}
 

/**
 * Deletes the output stream specified by config.KINESIS_OUTPUT_STREAM
 * 
 * @param config
 *        The configuration containing the stream name and {@link AWSCredentialsProvider}
 */
public static void deleteOutputStream(KinesisConnectorConfiguration config) {
    AmazonKinesisClient kinesisClient = new AmazonKinesisClient(config.AWS_CREDENTIALS_PROVIDER);
    kinesisClient.setRegion(RegionUtils.getRegion(config.REGION_NAME));
    if (config.KINESIS_ENDPOINT != null) {
        kinesisClient.setEndpoint(config.KINESIS_ENDPOINT);
    }
    deleteStream(kinesisClient, config.KINESIS_OUTPUT_STREAM);
}
 

/**
 * Deletes the input stream specified by config.KINESIS_INPUT_STREAM
 * 
 * @param config
 *        The configuration containing the stream name and {@link AWSCredentialsProvider}
 */
public static void deleteInputStream(KinesisConnectorConfiguration config) {
    AmazonKinesisClient kinesisClient = new AmazonKinesisClient(config.AWS_CREDENTIALS_PROVIDER);
    kinesisClient.setRegion(RegionUtils.getRegion(config.REGION_NAME));
    if (config.KINESIS_ENDPOINT != null) {
        kinesisClient.setEndpoint(config.KINESIS_ENDPOINT);
    }
    deleteStream(kinesisClient, config.KINESIS_INPUT_STREAM);
}