com.amazonaws.services.kinesis.AmazonKinesisClientBuilder Java Examples

/**
 * Creates an Amazon Kinesis Client.
 * @param configProps configuration properties containing the access key, secret key, and region
 * @param awsClientConfig preconfigured AWS SDK client configuration
 * @return a new Amazon Kinesis Client
 */
public static AmazonKinesis createKinesisClient(Properties configProps, ClientConfiguration awsClientConfig) {
	// set a Flink-specific user agent
	awsClientConfig.setUserAgentPrefix(String.format(USER_AGENT_FORMAT,
			EnvironmentInformation.getVersion(),
			EnvironmentInformation.getRevisionInformation().commitId));

	// utilize automatic refreshment of credentials by directly passing the AWSCredentialsProvider
	AmazonKinesisClientBuilder builder = AmazonKinesisClientBuilder.standard()
			.withCredentials(AWSUtil.getCredentialsProvider(configProps))
			.withClientConfiguration(awsClientConfig);

	if (configProps.containsKey(AWSConfigConstants.AWS_ENDPOINT)) {
		// Set signingRegion as null, to facilitate mocking Kinesis for local tests
		builder.withEndpointConfiguration(new AwsClientBuilder.EndpointConfiguration(
												configProps.getProperty(AWSConfigConstants.AWS_ENDPOINT),
												null));
	} else {
		builder.withRegion(Regions.fromName(configProps.getProperty(AWSConfigConstants.AWS_REGION)));
	}
	return builder.build();
}
 

/**
 * Creates an Amazon Kinesis Client.
 * @param configProps configuration properties containing the access key, secret key, and region
 * @param awsClientConfig preconfigured AWS SDK client configuration
 * @return a new Amazon Kinesis Client
 */
public static AmazonKinesis createKinesisClient(Properties configProps, ClientConfiguration awsClientConfig) {
	// set a Flink-specific user agent
	awsClientConfig.setUserAgentPrefix(String.format(USER_AGENT_FORMAT,
			EnvironmentInformation.getVersion(),
			EnvironmentInformation.getRevisionInformation().commitId));

	// utilize automatic refreshment of credentials by directly passing the AWSCredentialsProvider
	AmazonKinesisClientBuilder builder = AmazonKinesisClientBuilder.standard()
			.withCredentials(AWSUtil.getCredentialsProvider(configProps))
			.withClientConfiguration(awsClientConfig);

	if (configProps.containsKey(AWSConfigConstants.AWS_ENDPOINT)) {
		// Set signingRegion as null, to facilitate mocking Kinesis for local tests
		builder.withEndpointConfiguration(new AwsClientBuilder.EndpointConfiguration(
												configProps.getProperty(AWSConfigConstants.AWS_ENDPOINT),
												null));
	} else {
		builder.withRegion(Regions.fromName(configProps.getProperty(AWSConfigConstants.AWS_REGION)));
	}
	return builder.build();
}
 

/**
 * Creates an Amazon Kinesis Client.
 * @param configProps configuration properties containing the access key, secret key, and region
 * @param awsClientConfig preconfigured AWS SDK client configuration
 * @return a new Amazon Kinesis Client
 */
public static AmazonKinesis createKinesisClient(Properties configProps, ClientConfiguration awsClientConfig) {
	// set a Flink-specific user agent
	awsClientConfig.setUserAgentPrefix(String.format(USER_AGENT_FORMAT,
			EnvironmentInformation.getVersion(),
			EnvironmentInformation.getRevisionInformation().commitId));

	// utilize automatic refreshment of credentials by directly passing the AWSCredentialsProvider
	AmazonKinesisClientBuilder builder = AmazonKinesisClientBuilder.standard()
			.withCredentials(AWSUtil.getCredentialsProvider(configProps))
			.withClientConfiguration(awsClientConfig);

	if (configProps.containsKey(AWSConfigConstants.AWS_ENDPOINT)) {
		// If an endpoint is specified, we give preference to using an endpoint and use the region property to
		// sign the request.
		builder.withEndpointConfiguration(new AwsClientBuilder.EndpointConfiguration(
			configProps.getProperty(AWSConfigConstants.AWS_ENDPOINT),
			configProps.getProperty(AWSConfigConstants.AWS_REGION)));
	} else {
		builder.withRegion(Regions.fromName(configProps.getProperty(AWSConfigConstants.AWS_REGION)));
	}
	return builder.build();
}
 

@Override
public AmazonKinesis getClient() {
    ClientConfiguration clientConfiguration = new ClientConfiguration();
    clientConfiguration.setProtocol(Protocol.HTTP);

    return AmazonKinesisClientBuilder
            .standard()
            .withEndpointConfiguration(getContainer().getEndpointConfiguration(LocalStackContainer.Service.KINESIS))
            .withCredentials(getContainer().getDefaultCredentialsProvider())
            .withClientConfiguration(clientConfiguration)
            .build();
}
 

public static AmazonKinesis newKinesisClient() {
    LOG.debug("Creating a new AWS Kinesis client");
    AmazonKinesisClientBuilder clientBuilder = AmazonKinesisClientBuilder.standard();

    String awsInstanceType = System.getProperty("aws-service.kinesis.instance.type");
    String region = getRegion();

    if (awsInstanceType == null || awsInstanceType.equals("local-aws-container")) {
        String amazonHost = System.getProperty(AWSConfigs.AMAZON_AWS_HOST);

        LOG.debug("Creating a new AWS Kinesis client to access {}", amazonHost);

        ClientConfiguration clientConfiguration = new ClientConfiguration();
        clientConfiguration.setProtocol(Protocol.HTTP);

        clientBuilder
                .withEndpointConfiguration(new AwsClientBuilder.EndpointConfiguration(amazonHost, region))
                .withClientConfiguration(clientConfiguration)
                .withCredentials(new TestAWSCredentialsProvider("accesskey", "secretkey"));
    } else {
        clientBuilder
            .withRegion(region)
            .withCredentials(new TestAWSCredentialsProvider());
    }

    return clientBuilder.build();
}
 

@Override
public AmazonKinesis amazonKinesis() {
    return decorateWithConfigsAndBuild(
        AmazonKinesisClientBuilder.standard(),
        LocalstackDocker::getEndpointKinesis
    );
}
 

private static AmazonKinesis setupKinesisClient() {
    var endpointConfiguration = new AwsClientBuilder.EndpointConfiguration(KINESIS_SERVICE_ENDPOINT, "us-west-1");

    return AmazonKinesisClientBuilder.standard()
            .withCredentials(kinesisContainer.getDefaultCredentialsProvider())
            .withEndpointConfiguration(endpointConfiguration)
            .build();
}
 

@Override
public AmazonKinesis getKinesisClient() {
  AmazonKinesisClientBuilder clientBuilder =
      AmazonKinesisClientBuilder.standard().withCredentials(getCredentialsProvider());
  if (serviceEndpoint == null) {
    clientBuilder.withRegion(region);
  } else {
    clientBuilder.withEndpointConfiguration(
        new AwsClientBuilder.EndpointConfiguration(serviceEndpoint, region.getName()));
  }
  return clientBuilder.build();
}
 

private static AmazonKinesis getKinesisClient(ClientConfiguration awsClientConfig, KinesisConfigBean conf) throws StageException {

    AmazonKinesisClientBuilder builder = AmazonKinesisClientBuilder
        .standard()
        .withClientConfiguration(checkNotNull(awsClientConfig))
        .withCredentials(AWSUtil.getCredentialsProvider(conf.awsConfig));

    if (AwsRegion.OTHER == conf.region) {
      builder.withEndpointConfiguration(new AwsClientBuilder.EndpointConfiguration(conf.endpoint, null));
    } else {
      builder.withRegion(conf.region.getId());
    }

    return builder.build();
  }
 

@Override
public AmazonKinesis getKinesisClient() {
    AmazonKinesisClientBuilder clientBuilder =
            AmazonKinesisClientBuilder.standard().withCredentials(getCredentialsProvier());
    if (specifyEndpoint) {
        clientBuilder
                .withEndpointConfiguration(new AwsClientBuilder.EndpointConfiguration(endpoint, region.getName()));
    } else {
        clientBuilder.setRegion(region.getName());
    }
    return clientBuilder.build();
}
 

private static AmazonKinesis createClientWithCredentials(Properties props) throws AmazonClientException {
	AWSCredentialsProvider credentialsProvider = new EnvironmentVariableCredentialsProvider();
	return AmazonKinesisClientBuilder.standard()
		.withCredentials(credentialsProvider)
		.withEndpointConfiguration(
			new AwsClientBuilder.EndpointConfiguration(
				props.getProperty(ConsumerConfigConstants.AWS_ENDPOINT), "us-east-1"))
		.build();
}
 

public static AmazonKinesisClient createKinesisClient() {
    BasicCredentialsProvider credentials = BasicCredentialsProvider.standard();
    AmazonKinesisClient client = !credentials.isValid() ? null : (AmazonKinesisClient)
            AmazonKinesisClientBuilder.standard()
            .withCredentials(credentials)
            .withRegion("eu-west-1").build();
    return client;
}
 

public KinesisConfig(String address) throws Exception {

    	boolean create = true;
        // Chop off kineses://
        address = address.substring(10);

        // Substitute from environment variales;
        address = Configuration.substitute(address);

        String parts[] = address.split("&");
        for (int i = 0; i < parts.length; i++) {
            String t[] = parts[i].trim().split("=");
            if (t.length != 2) {
                throw new Exception("Not a proper parameter (a=b)");
            }
            t[0] = t[0].trim();
            t[1] = t[1].trim();
            switch (t[0]) {
                case "aws_access_key":
                    aws_access_key = t[1].trim();
                    break;
                case "aws_secret_key":
                    aws_secret_key = t[1].trim();
                    break;
                case "aws_region":
                    aws_region = t[1].trim();
                    break;
                case "topic":
                case "stream":
                    stream = t[1].trim();
                    break;
                case "partition":
                    partition = t[1].trim();
                    break;
                case "iterator_type":
                    iterator_type = t[1].trim();
                    break;
                case "record_limit":
                    record_limit = Integer.parseInt(t[1].trim());
                    break;
                case "sleep_time":
                    sleep_time = Integer.parseInt(t[1].trim());
                    break;
                case "create":
                    create = Boolean.parseBoolean(t[1].trim());
                    break;
            }
        }

        if (aws_access_key == null)
            throw new Exception("aws_access_key is not defined");

        if (aws_secret_key == null)
            throw new Exception("aws_secret_key is not defined");

        if (stream == null)
            throw new Exception("No stream defined");

        awsCreds = new BasicAWSCredentials(aws_access_key, aws_secret_key);

        amazonKinesis = AmazonKinesisClientBuilder
                .standard().withRegion(Regions.fromName(aws_region))
                .withCredentials(new AWSStaticCredentialsProvider(awsCreds))
                .build();
        
        if (create) {
            createTopicIfNotActive(stream);
        }
    }
 

public LambdaKinesisSdkWriteHandler() {
    this.kinesisClient = AmazonKinesisClientBuilder.standard()
            .withRegion(System.getenv("AWS_REGION"))
            .build();
}
 

public static Map<String,List<DataStreamVH>> fetchDataStreamInfo(BasicSessionCredentials temporaryCredentials, String skipRegions,String accountId,String accountName) {
    
    Map<String,List<DataStreamVH>> dataStream = new LinkedHashMap<>();
    AmazonKinesis amazonKinesis;
    String expPrefix = InventoryConstants.ERROR_PREFIX_CODE+accountId + "\",\"Message\": \"Exception in fetching info for resource\" ,\"type\": \"datastream\"" ;
    for(Region region : RegionUtils.getRegions()) { 
        try{
            if(!skipRegions.contains(region.getName())){
                amazonKinesis = AmazonKinesisClientBuilder.standard().withCredentials(new AWSStaticCredentialsProvider(temporaryCredentials)).withRegion(region.getName()).build();
                ListStreamsResult listStreamsResult = amazonKinesis.listStreams();
                List<String> streamNamesTemp = listStreamsResult.getStreamNames();
                List<String> streamNames = new ArrayList<>(streamNamesTemp);
                while (listStreamsResult.isHasMoreStreams() && !streamNamesTemp.isEmpty()) {
                    listStreamsResult = amazonKinesis.listStreams(streamNamesTemp.get(streamNamesTemp.size() - 1));
                    streamNamesTemp = listStreamsResult.getStreamNames();
                    streamNames.addAll(streamNamesTemp);
                }
                
                List<DataStreamVH> dataStreamList = new ArrayList<>();
                for(String streamName : streamNames) {
                    StreamDescription streamDescription = amazonKinesis.describeStream(new DescribeStreamRequest().withStreamName(streamName)).getStreamDescription();
                    ListTagsForStreamResult listTagsForStreamResult = amazonKinesis.listTagsForStream(new ListTagsForStreamRequest().withStreamName(streamName));
                    List<com.amazonaws.services.kinesis.model.Tag> tagsTemp = listTagsForStreamResult.getTags();
                    List<com.amazonaws.services.kinesis.model.Tag> tags = new ArrayList<>(tagsTemp);
                    while (listTagsForStreamResult.isHasMoreTags() && !tagsTemp.isEmpty()) {
                        listTagsForStreamResult = amazonKinesis.listTagsForStream(new ListTagsForStreamRequest().withExclusiveStartTagKey(tagsTemp.get(tagsTemp.size() - 1).getKey()));
                        tagsTemp = listTagsForStreamResult.getTags();
                        tags.addAll(tagsTemp);
                    }
                    dataStreamList.add(new DataStreamVH(streamDescription, tags));
                }
                if( !dataStreamList.isEmpty() ) {
                    log.debug(InventoryConstants.ACCOUNT + accountId +" Type : datastream "+region.getName() + " >> "+dataStreamList.size());
                    dataStream.put(accountId+delimiter+accountName+delimiter+region.getName(),dataStreamList);
                }
            }
        } catch(Exception e){
            log.warn(expPrefix+ region.getName()+InventoryConstants.ERROR_CAUSE +e.getMessage()+"\"}");
            ErrorManageUtil.uploadError(accountId, region.getName(),"datastream",e.getMessage());
        }
    }
    return dataStream;
}
 

public WatermarkTracker(String region, String streamName) {
  this.streamName = streamName;
  this.kinesisClient = AmazonKinesisClientBuilder.standard().withRegion(region).build();
}