Java Code Examples for org.apache.kafka.clients.producer.ProducerConfig#TRANSACTION_TIMEOUT_CONFIG

The following examples show how to use org.apache.kafka.clients.producer.ProducerConfig#TRANSACTION_TIMEOUT_CONFIG . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example 1

Source File: FlinkKafkaProducer.java From Flink-CEPplus with Apache License 2.0

4 votes

/**
 * Creates a FlinkKafkaProducer for a given topic. The sink produces its input to
 * the topic. It accepts a keyed {@link KeyedSerializationSchema} and possibly a custom {@link FlinkKafkaPartitioner}.
 *
 * <p>If a partitioner is not provided, written records will be partitioned by the attached key of each
 * record (as determined by {@link KeyedSerializationSchema#serializeKey(Object)}). If written records do not
 * have a key (i.e., {@link KeyedSerializationSchema#serializeKey(Object)} returns {@code null}), they
 * will be distributed to Kafka partitions in a round-robin fashion.
 *
 * @param defaultTopicId The default topic to write data to
 * @param serializationSchema A serializable serialization schema for turning user objects into a kafka-consumable byte[] supporting key/value messages
 * @param producerConfig Configuration properties for the KafkaProducer. 'bootstrap.servers.' is the only required argument.
 * @param customPartitioner A serializable partitioner for assigning messages to Kafka partitions.
 *                          If a partitioner is not provided, records will be partitioned by the key of each record
 *                          (determined by {@link KeyedSerializationSchema#serializeKey(Object)}). If the keys
 *                          are {@code null}, then records will be distributed to Kafka partitions in a
 *                          round-robin fashion.
 * @param semantic Defines semantic that will be used by this producer (see {@link FlinkKafkaProducer.Semantic}).
 * @param kafkaProducersPoolSize Overwrite default KafkaProducers pool size (see {@link FlinkKafkaProducer.Semantic#EXACTLY_ONCE}).
 */
public FlinkKafkaProducer(
	String defaultTopicId,
	KeyedSerializationSchema<IN> serializationSchema,
	Properties producerConfig,
	Optional<FlinkKafkaPartitioner<IN>> customPartitioner,
	FlinkKafkaProducer.Semantic semantic,
	int kafkaProducersPoolSize) {
	super(new FlinkKafkaProducer.TransactionStateSerializer(), new FlinkKafkaProducer.ContextStateSerializer());

	this.defaultTopicId = checkNotNull(defaultTopicId, "defaultTopicId is null");
	this.schema = checkNotNull(serializationSchema, "serializationSchema is null");
	this.producerConfig = checkNotNull(producerConfig, "producerConfig is null");
	this.flinkKafkaPartitioner = checkNotNull(customPartitioner, "customPartitioner is null").orElse(null);
	this.semantic = checkNotNull(semantic, "semantic is null");
	this.kafkaProducersPoolSize = kafkaProducersPoolSize;
	checkState(kafkaProducersPoolSize > 0, "kafkaProducersPoolSize must be non empty");

	ClosureCleaner.clean(this.flinkKafkaPartitioner, ExecutionConfig.ClosureCleanerLevel.RECURSIVE, true);
	ClosureCleaner.ensureSerializable(serializationSchema);

	// set the producer configuration properties for kafka record key value serializers.
	if (!producerConfig.containsKey(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG)) {
		this.producerConfig.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, ByteArraySerializer.class.getName());
	} else {
		LOG.warn("Overwriting the '{}' is not recommended", ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG);
	}

	if (!producerConfig.containsKey(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG)) {
		this.producerConfig.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, ByteArraySerializer.class.getName());
	} else {
		LOG.warn("Overwriting the '{}' is not recommended", ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG);
	}

	// eagerly ensure that bootstrap servers are set.
	if (!this.producerConfig.containsKey(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG)) {
		throw new IllegalArgumentException(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG + " must be supplied in the producer config properties.");
	}

	if (!producerConfig.containsKey(ProducerConfig.TRANSACTION_TIMEOUT_CONFIG)) {
		long timeout = DEFAULT_KAFKA_TRANSACTION_TIMEOUT.toMilliseconds();
		checkState(timeout < Integer.MAX_VALUE && timeout > 0, "timeout does not fit into 32 bit integer");
		this.producerConfig.put(ProducerConfig.TRANSACTION_TIMEOUT_CONFIG, (int) timeout);
		LOG.warn("Property [{}] not specified. Setting it to {}", ProducerConfig.TRANSACTION_TIMEOUT_CONFIG, DEFAULT_KAFKA_TRANSACTION_TIMEOUT);
	}

	// Enable transactionTimeoutWarnings to avoid silent data loss
	// See KAFKA-6119 (affects versions 0.11.0.0 and 0.11.0.1):
	// The KafkaProducer may not throw an exception if the transaction failed to commit
	if (semantic == FlinkKafkaProducer.Semantic.EXACTLY_ONCE) {
		final Object object = this.producerConfig.get(ProducerConfig.TRANSACTION_TIMEOUT_CONFIG);
		final long transactionTimeout;
		if (object instanceof String && StringUtils.isNumeric((String) object)) {
			transactionTimeout = Long.parseLong((String) object);
		} else if (object instanceof Number) {
			transactionTimeout = ((Number) object).longValue();
		} else {
			throw new IllegalArgumentException(ProducerConfig.TRANSACTION_TIMEOUT_CONFIG
				+ " must be numeric, was " + object);
		}
		super.setTransactionTimeout(transactionTimeout);
		super.enableTransactionTimeoutWarnings(0.8);
	}

	this.topicPartitionsMap = new HashMap<>();
}

Example 2

Source File: FlinkKafkaProducer011.java From Flink-CEPplus with Apache License 2.0

4 votes

/**
 * Creates a FlinkKafkaProducer for a given topic. The sink produces its input to
 * the topic. It accepts a keyed {@link KeyedSerializationSchema} and possibly a custom {@link FlinkKafkaPartitioner}.
 *
 * <p>If a partitioner is not provided, written records will be partitioned by the attached key of each
 * record (as determined by {@link KeyedSerializationSchema#serializeKey(Object)}). If written records do not
 * have a key (i.e., {@link KeyedSerializationSchema#serializeKey(Object)} returns {@code null}), they
 * will be distributed to Kafka partitions in a round-robin fashion.
 *
 * @param defaultTopicId The default topic to write data to
 * @param serializationSchema A serializable serialization schema for turning user objects into a kafka-consumable byte[] supporting key/value messages
 * @param producerConfig Configuration properties for the KafkaProducer. 'bootstrap.servers.' is the only required argument.
 * @param customPartitioner A serializable partitioner for assigning messages to Kafka partitions.
 *                          If a partitioner is not provided, records will be partitioned by the key of each record
 *                          (determined by {@link KeyedSerializationSchema#serializeKey(Object)}). If the keys
 *                          are {@code null}, then records will be distributed to Kafka partitions in a
 *                          round-robin fashion.
 * @param semantic Defines semantic that will be used by this producer (see {@link Semantic}).
 * @param kafkaProducersPoolSize Overwrite default KafkaProducers pool size (see {@link Semantic#EXACTLY_ONCE}).
 */
public FlinkKafkaProducer011(
		String defaultTopicId,
		KeyedSerializationSchema<IN> serializationSchema,
		Properties producerConfig,
		Optional<FlinkKafkaPartitioner<IN>> customPartitioner,
		Semantic semantic,
		int kafkaProducersPoolSize) {
	super(new TransactionStateSerializer(), new ContextStateSerializer());

	this.defaultTopicId = checkNotNull(defaultTopicId, "defaultTopicId is null");
	this.schema = checkNotNull(serializationSchema, "serializationSchema is null");
	this.producerConfig = checkNotNull(producerConfig, "producerConfig is null");
	this.flinkKafkaPartitioner = checkNotNull(customPartitioner, "customPartitioner is null").orElse(null);
	this.semantic = checkNotNull(semantic, "semantic is null");
	this.kafkaProducersPoolSize = kafkaProducersPoolSize;
	checkState(kafkaProducersPoolSize > 0, "kafkaProducersPoolSize must be non empty");

	ClosureCleaner.clean(this.flinkKafkaPartitioner, ExecutionConfig.ClosureCleanerLevel.RECURSIVE, true);
	ClosureCleaner.ensureSerializable(serializationSchema);

	// set the producer configuration properties for kafka record key value serializers.
	if (!producerConfig.containsKey(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG)) {
		this.producerConfig.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, ByteArraySerializer.class.getName());
	} else {
		LOG.warn("Overwriting the '{}' is not recommended", ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG);
	}

	if (!producerConfig.containsKey(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG)) {
		this.producerConfig.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, ByteArraySerializer.class.getName());
	} else {
		LOG.warn("Overwriting the '{}' is not recommended", ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG);
	}

	// eagerly ensure that bootstrap servers are set.
	if (!this.producerConfig.containsKey(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG)) {
		throw new IllegalArgumentException(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG + " must be supplied in the producer config properties.");
	}

	if (!producerConfig.containsKey(ProducerConfig.TRANSACTION_TIMEOUT_CONFIG)) {
		long timeout = DEFAULT_KAFKA_TRANSACTION_TIMEOUT.toMilliseconds();
		checkState(timeout < Integer.MAX_VALUE && timeout > 0, "timeout does not fit into 32 bit integer");
		this.producerConfig.put(ProducerConfig.TRANSACTION_TIMEOUT_CONFIG, (int) timeout);
		LOG.warn("Property [{}] not specified. Setting it to {}", ProducerConfig.TRANSACTION_TIMEOUT_CONFIG, DEFAULT_KAFKA_TRANSACTION_TIMEOUT);
	}

	// Enable transactionTimeoutWarnings to avoid silent data loss
	// See KAFKA-6119 (affects versions 0.11.0.0 and 0.11.0.1):
	// The KafkaProducer may not throw an exception if the transaction failed to commit
	if (semantic == Semantic.EXACTLY_ONCE) {
		final Object object = this.producerConfig.get(ProducerConfig.TRANSACTION_TIMEOUT_CONFIG);
		final long transactionTimeout;
		if (object instanceof String && StringUtils.isNumeric((String) object)) {
			transactionTimeout = Long.parseLong((String) object);
		} else if (object instanceof Number) {
			transactionTimeout = ((Number) object).longValue();
		} else {
			throw new IllegalArgumentException(ProducerConfig.TRANSACTION_TIMEOUT_CONFIG
				+ " must be numeric, was " + object);
		}
		super.setTransactionTimeout(transactionTimeout);
		super.enableTransactionTimeoutWarnings(0.8);
	}

	this.topicPartitionsMap = new HashMap<>();
}

Example 3

Source File: FlinkKafkaProducer011.java From flink with Apache License 2.0

4 votes

/**
 * Creates a FlinkKafkaProducer for a given topic. The sink produces its input to
 * the topic. It accepts a keyed {@link KeyedSerializationSchema} and possibly a custom {@link FlinkKafkaPartitioner}.
 *
 * <p>If a partitioner is not provided, written records will be partitioned by the attached key of each
 * record (as determined by {@link KeyedSerializationSchema#serializeKey(Object)}). If written records do not
 * have a key (i.e., {@link KeyedSerializationSchema#serializeKey(Object)} returns {@code null}), they
 * will be distributed to Kafka partitions in a round-robin fashion.
 *
 * @param defaultTopicId The default topic to write data to
 * @param serializationSchema A serializable serialization schema for turning user objects into a kafka-consumable byte[] supporting key/value messages
 * @param producerConfig Configuration properties for the KafkaProducer. 'bootstrap.servers.' is the only required argument.
 * @param customPartitioner A serializable partitioner for assigning messages to Kafka partitions.
 *                          If a partitioner is not provided, records will be partitioned by the key of each record
 *                          (determined by {@link KeyedSerializationSchema#serializeKey(Object)}). If the keys
 *                          are {@code null}, then records will be distributed to Kafka partitions in a
 *                          round-robin fashion.
 * @param semantic Defines semantic that will be used by this producer (see {@link Semantic}).
 * @param kafkaProducersPoolSize Overwrite default KafkaProducers pool size (see {@link Semantic#EXACTLY_ONCE}).
 */
public FlinkKafkaProducer011(
		String defaultTopicId,
		KeyedSerializationSchema<IN> serializationSchema,
		Properties producerConfig,
		Optional<FlinkKafkaPartitioner<IN>> customPartitioner,
		Semantic semantic,
		int kafkaProducersPoolSize) {
	super(new TransactionStateSerializer(), new ContextStateSerializer());

	this.defaultTopicId = checkNotNull(defaultTopicId, "defaultTopicId is null");
	this.schema = checkNotNull(serializationSchema, "serializationSchema is null");
	this.producerConfig = checkNotNull(producerConfig, "producerConfig is null");
	this.flinkKafkaPartitioner = checkNotNull(customPartitioner, "customPartitioner is null").orElse(null);
	this.semantic = checkNotNull(semantic, "semantic is null");
	this.kafkaProducersPoolSize = kafkaProducersPoolSize;
	checkState(kafkaProducersPoolSize > 0, "kafkaProducersPoolSize must be non empty");

	ClosureCleaner.clean(this.flinkKafkaPartitioner, ExecutionConfig.ClosureCleanerLevel.RECURSIVE, true);
	ClosureCleaner.ensureSerializable(serializationSchema);

	// set the producer configuration properties for kafka record key value serializers.
	if (!producerConfig.containsKey(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG)) {
		this.producerConfig.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, ByteArraySerializer.class.getName());
	} else {
		LOG.warn("Overwriting the '{}' is not recommended", ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG);
	}

	if (!producerConfig.containsKey(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG)) {
		this.producerConfig.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, ByteArraySerializer.class.getName());
	} else {
		LOG.warn("Overwriting the '{}' is not recommended", ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG);
	}

	// eagerly ensure that bootstrap servers are set.
	if (!this.producerConfig.containsKey(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG)) {
		throw new IllegalArgumentException(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG + " must be supplied in the producer config properties.");
	}

	if (!producerConfig.containsKey(ProducerConfig.TRANSACTION_TIMEOUT_CONFIG)) {
		long timeout = DEFAULT_KAFKA_TRANSACTION_TIMEOUT.toMilliseconds();
		checkState(timeout < Integer.MAX_VALUE && timeout > 0, "timeout does not fit into 32 bit integer");
		this.producerConfig.put(ProducerConfig.TRANSACTION_TIMEOUT_CONFIG, (int) timeout);
		LOG.warn("Property [{}] not specified. Setting it to {}", ProducerConfig.TRANSACTION_TIMEOUT_CONFIG, DEFAULT_KAFKA_TRANSACTION_TIMEOUT);
	}

	// Enable transactionTimeoutWarnings to avoid silent data loss
	// See KAFKA-6119 (affects versions 0.11.0.0 and 0.11.0.1):
	// The KafkaProducer may not throw an exception if the transaction failed to commit
	if (semantic == Semantic.EXACTLY_ONCE) {
		final Object object = this.producerConfig.get(ProducerConfig.TRANSACTION_TIMEOUT_CONFIG);
		final long transactionTimeout;
		if (object instanceof String && StringUtils.isNumeric((String) object)) {
			transactionTimeout = Long.parseLong((String) object);
		} else if (object instanceof Number) {
			transactionTimeout = ((Number) object).longValue();
		} else {
			throw new IllegalArgumentException(ProducerConfig.TRANSACTION_TIMEOUT_CONFIG
				+ " must be numeric, was " + object);
		}
		super.setTransactionTimeout(transactionTimeout);
		super.enableTransactionTimeoutWarnings(0.8);
	}

	this.topicPartitionsMap = new HashMap<>();
}

Example 4

Source File: FlinkKafkaProducer011.java From flink with Apache License 2.0

4 votes

/**
 * Creates a FlinkKafkaProducer for a given topic. The sink produces its input to
 * the topic. It accepts a keyed {@link KeyedSerializationSchema} and possibly a custom {@link FlinkKafkaPartitioner}.
 *
 * <p>If a partitioner is not provided, written records will be partitioned by the attached key of each
 * record (as determined by {@link KeyedSerializationSchema#serializeKey(Object)}). If written records do not
 * have a key (i.e., {@link KeyedSerializationSchema#serializeKey(Object)} returns {@code null}), they
 * will be distributed to Kafka partitions in a round-robin fashion.
 *
 * @param defaultTopicId The default topic to write data to
 * @param serializationSchema A serializable serialization schema for turning user objects into a kafka-consumable byte[] supporting key/value messages
 * @param producerConfig Configuration properties for the KafkaProducer. 'bootstrap.servers.' is the only required argument.
 * @param customPartitioner A serializable partitioner for assigning messages to Kafka partitions.
 *                          If a partitioner is not provided, records will be partitioned by the key of each record
 *                          (determined by {@link KeyedSerializationSchema#serializeKey(Object)}). If the keys
 *                          are {@code null}, then records will be distributed to Kafka partitions in a
 *                          round-robin fashion.
 * @param semantic Defines semantic that will be used by this producer (see {@link Semantic}).
 * @param kafkaProducersPoolSize Overwrite default KafkaProducers pool size (see {@link Semantic#EXACTLY_ONCE}).
 */
public FlinkKafkaProducer011(
		String defaultTopicId,
		KeyedSerializationSchema<IN> serializationSchema,
		Properties producerConfig,
		Optional<FlinkKafkaPartitioner<IN>> customPartitioner,
		Semantic semantic,
		int kafkaProducersPoolSize) {
	super(new TransactionStateSerializer(), new ContextStateSerializer());

	this.defaultTopicId = checkNotNull(defaultTopicId, "defaultTopicId is null");
	this.schema = checkNotNull(serializationSchema, "serializationSchema is null");
	this.producerConfig = checkNotNull(producerConfig, "producerConfig is null");
	this.flinkKafkaPartitioner = checkNotNull(customPartitioner, "customPartitioner is null").orElse(null);
	this.semantic = checkNotNull(semantic, "semantic is null");
	this.kafkaProducersPoolSize = kafkaProducersPoolSize;
	checkState(kafkaProducersPoolSize > 0, "kafkaProducersPoolSize must be non empty");

	ClosureCleaner.clean(this.flinkKafkaPartitioner, ExecutionConfig.ClosureCleanerLevel.RECURSIVE, true);
	ClosureCleaner.ensureSerializable(serializationSchema);

	// set the producer configuration properties for kafka record key value serializers.
	if (!producerConfig.containsKey(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG)) {
		this.producerConfig.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, ByteArraySerializer.class.getName());
	} else {
		LOG.warn("Overwriting the '{}' is not recommended", ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG);
	}

	if (!producerConfig.containsKey(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG)) {
		this.producerConfig.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, ByteArraySerializer.class.getName());
	} else {
		LOG.warn("Overwriting the '{}' is not recommended", ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG);
	}

	// eagerly ensure that bootstrap servers are set.
	if (!this.producerConfig.containsKey(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG)) {
		throw new IllegalArgumentException(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG + " must be supplied in the producer config properties.");
	}

	if (!producerConfig.containsKey(ProducerConfig.TRANSACTION_TIMEOUT_CONFIG)) {
		long timeout = DEFAULT_KAFKA_TRANSACTION_TIMEOUT.toMilliseconds();
		checkState(timeout < Integer.MAX_VALUE && timeout > 0, "timeout does not fit into 32 bit integer");
		this.producerConfig.put(ProducerConfig.TRANSACTION_TIMEOUT_CONFIG, (int) timeout);
		LOG.warn("Property [{}] not specified. Setting it to {}", ProducerConfig.TRANSACTION_TIMEOUT_CONFIG, DEFAULT_KAFKA_TRANSACTION_TIMEOUT);
	}

	// Enable transactionTimeoutWarnings to avoid silent data loss
	// See KAFKA-6119 (affects versions 0.11.0.0 and 0.11.0.1):
	// The KafkaProducer may not throw an exception if the transaction failed to commit
	if (semantic == Semantic.EXACTLY_ONCE) {
		final Object object = this.producerConfig.get(ProducerConfig.TRANSACTION_TIMEOUT_CONFIG);
		final long transactionTimeout;
		if (object instanceof String && StringUtils.isNumeric((String) object)) {
			transactionTimeout = Long.parseLong((String) object);
		} else if (object instanceof Number) {
			transactionTimeout = ((Number) object).longValue();
		} else {
			throw new IllegalArgumentException(ProducerConfig.TRANSACTION_TIMEOUT_CONFIG
				+ " must be numeric, was " + object);
		}
		super.setTransactionTimeout(transactionTimeout);
		super.enableTransactionTimeoutWarnings(0.8);
	}

	this.topicPartitionsMap = new HashMap<>();
}