Java Code Examples for com.amazonaws.services.kinesis.producer.KinesisProducerConfiguration#setRegion()

/**
 * Validate configuration properties for {@link FlinkKinesisProducer},
 * and return a constructed KinesisProducerConfiguration.
 */
public static KinesisProducerConfiguration getValidatedProducerConfiguration(Properties config) {
	checkNotNull(config, "config can not be null");

	validateAwsConfiguration(config);

	if (!config.containsKey(AWSConfigConstants.AWS_REGION)) {
		// per requirement in Amazon Kinesis Producer Library
		throw new IllegalArgumentException(String.format("For FlinkKinesisProducer AWS region ('%s') must be set in the config.", AWSConfigConstants.AWS_REGION));
	}

	KinesisProducerConfiguration kpc = KinesisProducerConfiguration.fromProperties(config);
	kpc.setRegion(config.getProperty(AWSConfigConstants.AWS_REGION));

	kpc.setCredentialsProvider(AWSUtil.getCredentialsProvider(config));

	// we explicitly lower the credential refresh delay (default is 5 seconds)
	// to avoid an ignorable interruption warning that occurs when shutting down the
	// KPL client. See https://github.com/awslabs/amazon-kinesis-producer/issues/10.
	kpc.setCredentialsRefreshDelay(100);

	// Override default values if they aren't specified by users
	if (!config.containsKey(RATE_LIMIT)) {
		kpc.setRateLimit(DEFAULT_RATE_LIMIT);
	}
	if (!config.containsKey(THREADING_MODEL)) {
		kpc.setThreadingModel(DEFAULT_THREADING_MODEL);
	}
	if (!config.containsKey(THREAD_POOL_SIZE)) {
		kpc.setThreadPoolSize(DEFAULT_THREAD_POOL_SIZE);
	}

	return kpc;
}
 

/**
 * Validate configuration properties for {@link FlinkKinesisProducer},
 * and return a constructed KinesisProducerConfiguration.
 */
public static KinesisProducerConfiguration getValidatedProducerConfiguration(Properties config) {
	checkNotNull(config, "config can not be null");

	validateAwsConfiguration(config);

	if (!config.containsKey(AWSConfigConstants.AWS_REGION)) {
		// per requirement in Amazon Kinesis Producer Library
		throw new IllegalArgumentException(String.format("For FlinkKinesisProducer AWS region ('%s') must be set in the config.", AWSConfigConstants.AWS_REGION));
	}

	KinesisProducerConfiguration kpc = KinesisProducerConfiguration.fromProperties(config);
	kpc.setRegion(config.getProperty(AWSConfigConstants.AWS_REGION));

	kpc.setCredentialsProvider(AWSUtil.getCredentialsProvider(config));

	// we explicitly lower the credential refresh delay (default is 5 seconds)
	// to avoid an ignorable interruption warning that occurs when shutting down the
	// KPL client. See https://github.com/awslabs/amazon-kinesis-producer/issues/10.
	kpc.setCredentialsRefreshDelay(100);

	// Override default values if they aren't specified by users
	if (!config.containsKey(RATE_LIMIT)) {
		kpc.setRateLimit(DEFAULT_RATE_LIMIT);
	}
	if (!config.containsKey(THREADING_MODEL)) {
		kpc.setThreadingModel(DEFAULT_THREADING_MODEL);
	}
	if (!config.containsKey(THREAD_POOL_SIZE)) {
		kpc.setThreadPoolSize(DEFAULT_THREAD_POOL_SIZE);
	}

	return kpc;
}
 

private KinesisProducer getKinesisProducer() {
	KinesisProducerConfiguration config = new KinesisProducerConfiguration();
	config.setRegion(regionName);
	config.setCredentialsProvider(new DefaultAWSCredentialsProviderChain());
	config.setMaxConnections(maxConnections);

	config.setAggregationEnabled(aggregration);

	// Limits the maximum allowed put rate for a shard, as a percentage of
	// the
	// backend limits.
	config.setRateLimit(rateLimit);

	// Maximum amount of time (milliseconds) a record may spend being
	// buffered
	// before it gets sent. Records may be sent sooner than this depending
	// on the
	// other buffering limits
	config.setRecordMaxBufferedTime(maxBufferedTime);

	// Set a time-to-live on records (milliseconds). Records that do not get
	// successfully put within the limit are failed.
	config.setRecordTtl(ttl);

	// Controls the number of metrics that are uploaded to CloudWatch.
	// Expected pattern: none|summary|detailed
	config.setMetricsLevel(metricsLevel);

	// Controls the granularity of metrics that are uploaded to CloudWatch.
	// Greater granularity produces more metrics.
	// Expected pattern: global|stream|shard
	config.setMetricsGranularity(metricsGranuality);

	// The namespace to upload metrics under.
	config.setMetricsNamespace(metricsNameSpace);

	return new KinesisProducer(config);

}
 

/**
 * Here'll walk through some of the config options and create an instance of
 * KinesisProducer, which will be used to put records.
 *
 * @param region The region of the Kinesis stream being used.
 *
 * @return KinesisProducer instance used to put records.
 */
private static KinesisProducer getKinesisProducer(final String region) {
    KinesisProducerConfiguration config = new KinesisProducerConfiguration();
    config.setRegion(region);
    config.setCredentialsProvider(new DefaultAWSCredentialsProviderChain());
    config.setMaxConnections(1);
    config.setRequestTimeout(60000);
    config.setRecordMaxBufferedTime(2000);
    config.setAggregationEnabled(false);

    KinesisProducer producer = new KinesisProducer(config);
    return producer;
}
 

/**
 * Given an AWS region, generate a configured Kinesis Producer.
 */
private static KinesisProducer getKinesisProducer(String region)
{
    KinesisProducerConfiguration config = new KinesisProducerConfiguration();
    config.setRegion(region);
    config.setMaxConnections(1);
    config.setRequestTimeout(60000);
    config.setRecordMaxBufferedTime(15000);
    config.setAggregationEnabled(true);

    return new KinesisProducer(config);
}
 

@Override
public void open(Map<String, Object> config, SinkContext sinkContext) throws Exception {
    kinesisSinkConfig = KinesisSinkConfig.load(config);
    this.sinkContext = sinkContext;

    checkArgument(isNotBlank(kinesisSinkConfig.getAwsKinesisStreamName()), "empty kinesis-stream name");
    checkArgument(isNotBlank(kinesisSinkConfig.getAwsEndpoint()) || 
                  isNotBlank(kinesisSinkConfig.getAwsRegion()), 
                  "Either the aws-end-point or aws-region must be set");
    checkArgument(isNotBlank(kinesisSinkConfig.getAwsCredentialPluginParam()), "empty aws-credential param");

    KinesisProducerConfiguration kinesisConfig = new KinesisProducerConfiguration();
    kinesisConfig.setKinesisEndpoint(kinesisSinkConfig.getAwsEndpoint());
    kinesisConfig.setRegion(kinesisSinkConfig.getAwsRegion());
    kinesisConfig.setThreadingModel(ThreadingModel.POOLED);
    kinesisConfig.setThreadPoolSize(4);
    kinesisConfig.setCollectionMaxCount(1);
    AWSCredentialsProvider credentialsProvider = createCredentialProvider(
            kinesisSinkConfig.getAwsCredentialPluginName(),
            kinesisSinkConfig.getAwsCredentialPluginParam())
        .getCredentialProvider();
    kinesisConfig.setCredentialsProvider(credentialsProvider);

    this.streamName = kinesisSinkConfig.getAwsKinesisStreamName();
    this.kinesisProducer = new KinesisProducer(kinesisConfig);
    IS_PUBLISH_FAILED.set(this, FALSE);

    LOG.info("Kinesis sink started. {}", (ReflectionToStringBuilder.toString(kinesisConfig, ToStringStyle.SHORT_PREFIX_STYLE)));
}
 

/**
 * Validate configuration properties for {@link FlinkKinesisProducer},
 * and return a constructed KinesisProducerConfiguration.
 */
public static KinesisProducerConfiguration getValidatedProducerConfiguration(Properties config) {
	checkNotNull(config, "config can not be null");

	validateAwsConfiguration(config);

	if (!config.containsKey(AWSConfigConstants.AWS_REGION)) {
		// per requirement in Amazon Kinesis Producer Library
		throw new IllegalArgumentException(String.format("For FlinkKinesisProducer AWS region ('%s') must be set in the config.", AWSConfigConstants.AWS_REGION));
	}

	KinesisProducerConfiguration kpc = KinesisProducerConfiguration.fromProperties(config);
	kpc.setRegion(config.getProperty(AWSConfigConstants.AWS_REGION));

	kpc.setCredentialsProvider(AWSUtil.getCredentialsProvider(config));

	// we explicitly lower the credential refresh delay (default is 5 seconds)
	// to avoid an ignorable interruption warning that occurs when shutting down the
	// KPL client. See https://github.com/awslabs/amazon-kinesis-producer/issues/10.
	kpc.setCredentialsRefreshDelay(100);

	// Override default values if they aren't specified by users
	if (!config.containsKey(RATE_LIMIT)) {
		kpc.setRateLimit(DEFAULT_RATE_LIMIT);
	}
	if (!config.containsKey(THREADING_MODEL)) {
		kpc.setThreadingModel(DEFAULT_THREADING_MODEL);
	}
	if (!config.containsKey(THREAD_POOL_SIZE)) {
		kpc.setThreadPoolSize(DEFAULT_THREAD_POOL_SIZE);
	}

	return kpc;
}
 

@Override
public IKinesisProducer createKinesisProducer(KinesisProducerConfiguration config) {
  config.setRegion(region.getName());
  config.setCredentialsProvider(getCredentialsProvider());
  return new KinesisProducer(config);
}