Java Code Examples for org.apache.flink.streaming.connectors.kinesis.util.KinesisConfigUtil#getValidatedProducerConfiguration()

The following examples show how to use org.apache.flink.streaming.connectors.kinesis.util.KinesisConfigUtil#getValidatedProducerConfiguration() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: FlinkKinesisProducer.java    From Flink-CEPplus with Apache License 2.0 4 votes vote down vote up 
@Override
public void open(Configuration parameters) throws Exception {
	super.open(parameters);

	// check and pass the configuration properties
	KinesisProducerConfiguration producerConfig = KinesisConfigUtil.getValidatedProducerConfiguration(configProps);

	producer = getKinesisProducer(producerConfig);

	final MetricGroup kinesisMectricGroup = getRuntimeContext().getMetricGroup().addGroup(KINESIS_PRODUCER_METRIC_GROUP);
	this.backpressureCycles = kinesisMectricGroup.counter(METRIC_BACKPRESSURE_CYCLES);
	kinesisMectricGroup.gauge(METRIC_OUTSTANDING_RECORDS_COUNT, producer::getOutstandingRecordsCount);

	backpressureLatch = new TimeoutLatch();
	callback = new FutureCallback<UserRecordResult>() {
		@Override
		public void onSuccess(UserRecordResult result) {
			backpressureLatch.trigger();
			if (!result.isSuccessful()) {
				if (failOnError) {
					// only remember the first thrown exception
					if (thrownException == null) {
						thrownException = new RuntimeException("Record was not sent successful");
					}
				} else {
					LOG.warn("Record was not sent successful");
				}
			}
		}

		@Override
		public void onFailure(Throwable t) {
			backpressureLatch.trigger();
			if (failOnError) {
				thrownException = t;
			} else {
				LOG.warn("An exception occurred while processing a record", t);
			}
		}
	};

	if (this.customPartitioner != null) {
		this.customPartitioner.initialize(getRuntimeContext().getIndexOfThisSubtask(), getRuntimeContext().getNumberOfParallelSubtasks());
	}

	LOG.info("Started Kinesis producer instance for region '{}'", producerConfig.getRegion());
}
Example 2
Source File: FlinkKinesisProducer.java    From flink with Apache License 2.0 4 votes vote down vote up 
@Override
public void open(Configuration parameters) throws Exception {
	super.open(parameters);

	// check and pass the configuration properties
	KinesisProducerConfiguration producerConfig = KinesisConfigUtil.getValidatedProducerConfiguration(configProps);

	producer = getKinesisProducer(producerConfig);

	final MetricGroup kinesisMectricGroup = getRuntimeContext().getMetricGroup().addGroup(KINESIS_PRODUCER_METRIC_GROUP);
	this.backpressureCycles = kinesisMectricGroup.counter(METRIC_BACKPRESSURE_CYCLES);
	kinesisMectricGroup.gauge(METRIC_OUTSTANDING_RECORDS_COUNT, producer::getOutstandingRecordsCount);

	backpressureLatch = new TimeoutLatch();
	callback = new FutureCallback<UserRecordResult>() {
		@Override
		public void onSuccess(UserRecordResult result) {
			backpressureLatch.trigger();
			if (!result.isSuccessful()) {
				if (failOnError) {
					// only remember the first thrown exception
					if (thrownException == null) {
						thrownException = new RuntimeException("Record was not sent successful");
					}
				} else {
					LOG.warn("Record was not sent successful");
				}
			}
		}

		@Override
		public void onFailure(Throwable t) {
			backpressureLatch.trigger();
			if (failOnError) {
				thrownException = t;
			} else {
				LOG.warn("An exception occurred while processing a record", t);
			}
		}
	};

	if (this.customPartitioner != null) {
		this.customPartitioner.initialize(getRuntimeContext().getIndexOfThisSubtask(), getRuntimeContext().getNumberOfParallelSubtasks());
	}

	LOG.info("Started Kinesis producer instance for region '{}'", producerConfig.getRegion());
}
Example 3
Source File: FlinkKinesisProducer.java    From flink with Apache License 2.0 4 votes vote down vote up 
@Override
public void open(Configuration parameters) throws Exception {
	super.open(parameters);

	schema.open(() -> getRuntimeContext().getMetricGroup().addGroup("user"));

	// check and pass the configuration properties
	KinesisProducerConfiguration producerConfig = KinesisConfigUtil.getValidatedProducerConfiguration(configProps);

	producer = getKinesisProducer(producerConfig);

	final MetricGroup kinesisMectricGroup = getRuntimeContext().getMetricGroup().addGroup(KINESIS_PRODUCER_METRIC_GROUP);
	this.backpressureCycles = kinesisMectricGroup.counter(METRIC_BACKPRESSURE_CYCLES);
	kinesisMectricGroup.gauge(METRIC_OUTSTANDING_RECORDS_COUNT, producer::getOutstandingRecordsCount);

	backpressureLatch = new TimeoutLatch();
	callback = new FutureCallback<UserRecordResult>() {
		@Override
		public void onSuccess(UserRecordResult result) {
			backpressureLatch.trigger();
			if (!result.isSuccessful()) {
				if (failOnError) {
					// only remember the first thrown exception
					if (thrownException == null) {
						thrownException = new RuntimeException("Record was not sent successful");
					}
				} else {
					LOG.warn("Record was not sent successful");
				}
			}
		}

		@Override
		public void onFailure(Throwable t) {
			backpressureLatch.trigger();
			if (failOnError) {
				thrownException = t;
			} else {
				LOG.warn("An exception occurred while processing a record", t);
			}
		}
	};

	if (this.customPartitioner != null) {
		this.customPartitioner.initialize(getRuntimeContext().getIndexOfThisSubtask(), getRuntimeContext().getNumberOfParallelSubtasks());
	}

	LOG.info("Started Kinesis producer instance for region '{}'", producerConfig.getRegion());
}