org.apache.flink.streaming.connectors.kafka.internal.FlinkKafkaProducer Java Examples

@Override
protected void recoverAndCommit(KafkaTransactionState transaction) {
	if (transaction.isTransactional()) {
		try (
			FlinkKafkaProducer<byte[], byte[]> producer =
				initTransactionalProducer(transaction.transactionalId, false)) {
			producer.resumeTransaction(transaction.producerId, transaction.epoch);
			producer.commitTransaction();
		} catch (InvalidTxnStateException | ProducerFencedException ex) {
			// That means we have committed this transaction before.
			LOG.warn("Encountered error {} while recovering transaction {}. " +
					"Presumably this transaction has been already committed before",
				ex,
				transaction);
		}
	}
}
 

@Override
protected KafkaTransactionState beginTransaction() throws FlinkKafka011Exception {
	switch (semantic) {
		case EXACTLY_ONCE:
			FlinkKafkaProducer<byte[], byte[]> producer = createTransactionalProducer();
			producer.beginTransaction();
			return new KafkaTransactionState(producer.getTransactionalId(), producer);
		case AT_LEAST_ONCE:
		case NONE:
			// Do not create new producer on each beginTransaction() if it is not necessary
			final KafkaTransactionState currentTransaction = currentTransaction();
			if (currentTransaction != null && currentTransaction.producer != null) {
				return new KafkaTransactionState(currentTransaction.producer);
			}
			return new KafkaTransactionState(initNonTransactionalProducer(true));
		default:
			throw new UnsupportedOperationException("Not implemented semantic");
	}
}
 

private void abortTransactions(Set<String> transactionalIds) {
	transactionalIds.parallelStream().forEach(transactionalId -> {
		// don't mess with the original configuration or any other properties of the
		// original object
		// -> create an internal kafka producer on our own and do not rely on
		//    initTransactionalProducer().
		final Properties myConfig = new Properties();
		myConfig.putAll(producerConfig);
		initTransactionalProducerConfig(myConfig, transactionalId);
		try (FlinkKafkaProducer<byte[], byte[]> kafkaProducer =
				new FlinkKafkaProducer<>(myConfig)) {
			// it suffices to call initTransactions - this will abort any lingering transactions
			kafkaProducer.initTransactions();
		}
	});
}
 

private void abortTransactions(Set<String> transactionalIds) {
	transactionalIds.parallelStream().forEach(transactionalId -> {
		// don't mess with the original configuration or any other properties of the
		// original object
		// -> create an internal kafka producer on our own and do not rely on
		//    initTransactionalProducer().
		final Properties myConfig = new Properties();
		myConfig.putAll(producerConfig);
		initTransactionalProducerConfig(myConfig, transactionalId);
		try (FlinkKafkaProducer<byte[], byte[]> kafkaProducer =
				new FlinkKafkaProducer<>(myConfig)) {
			// it suffices to call initTransactions - this will abort any lingering transactions
			kafkaProducer.initTransactions();
		}
	});
}
 

@Override
protected KafkaTransactionState beginTransaction() throws FlinkKafka011Exception {
	switch (semantic) {
		case EXACTLY_ONCE:
			FlinkKafkaProducer<byte[], byte[]> producer = createTransactionalProducer();
			producer.beginTransaction();
			return new KafkaTransactionState(producer.getTransactionalId(), producer);
		case AT_LEAST_ONCE:
		case NONE:
			// Do not create new producer on each beginTransaction() if it is not necessary
			final KafkaTransactionState currentTransaction = currentTransaction();
			if (currentTransaction != null && currentTransaction.producer != null) {
				return new KafkaTransactionState(currentTransaction.producer);
			}
			return new KafkaTransactionState(initNonTransactionalProducer(true));
		default:
			throw new UnsupportedOperationException("Not implemented semantic");
	}
}
 

@Override
protected void recoverAndCommit(KafkaTransactionState transaction) {
	if (transaction.isTransactional()) {
		try (
			FlinkKafkaProducer<byte[], byte[]> producer =
				initTransactionalProducer(transaction.transactionalId, false)) {
			producer.resumeTransaction(transaction.producerId, transaction.epoch);
			producer.commitTransaction();
		} catch (InvalidTxnStateException | ProducerFencedException ex) {
			// That means we have committed this transaction before.
			LOG.warn("Encountered error {} while recovering transaction {}. " +
					"Presumably this transaction has been already committed before",
				ex,
				transaction);
		}
	}
}
 

@Override
protected KafkaTransactionState beginTransaction() throws FlinkKafka011Exception {
	switch (semantic) {
		case EXACTLY_ONCE:
			FlinkKafkaProducer<byte[], byte[]> producer = createTransactionalProducer();
			producer.beginTransaction();
			return new KafkaTransactionState(producer.getTransactionalId(), producer);
		case AT_LEAST_ONCE:
		case NONE:
			// Do not create new producer on each beginTransaction() if it is not necessary
			final KafkaTransactionState currentTransaction = currentTransaction();
			if (currentTransaction != null && currentTransaction.producer != null) {
				return new KafkaTransactionState(currentTransaction.producer);
			}
			return new KafkaTransactionState(initNonTransactionalProducer(true));
		default:
			throw new UnsupportedOperationException("Not implemented semantic");
	}
}
 

@Override
protected void recoverAndCommit(KafkaTransactionState transaction) {
	if (transaction.isTransactional()) {
		try (
			FlinkKafkaProducer<byte[], byte[]> producer =
				initTransactionalProducer(transaction.transactionalId, false)) {
			producer.resumeTransaction(transaction.producerId, transaction.epoch);
			producer.commitTransaction();
		} catch (InvalidTxnStateException | ProducerFencedException ex) {
			// That means we have committed this transaction before.
			LOG.warn("Encountered error {} while recovering transaction {}. " +
					"Presumably this transaction has been already committed before",
				ex,
				transaction);
		}
	}
}
 

KafkaTransactionState(
		@Nullable String transactionalId,
		long producerId,
		short epoch,
		FlinkKafkaProducer<byte[], byte[]> producer) {
	this.transactionalId = transactionalId;
	this.producerId = producerId;
	this.epoch = epoch;
	this.producer = producer;
}
 

private FlinkKafkaProducer<String, String> getClosedProducer(String topicName) {
	FlinkKafkaProducer<String, String> kafkaProducer = new FlinkKafkaProducer<>(extraProperties);
	kafkaProducer.initTransactions();
	kafkaProducer.beginTransaction();
	kafkaProducer.send(new ProducerRecord<>(topicName, "42", "42"));
	kafkaProducer.close();
	return kafkaProducer;
}
 

@Test(timeout = 30000L)
public void testHappyPath() throws IOException {
	String topicName = "flink-kafka-producer-happy-path";
	try (Producer<String, String> kafkaProducer = new FlinkKafkaProducer<>(extraProperties)) {
		kafkaProducer.initTransactions();
		kafkaProducer.beginTransaction();
		kafkaProducer.send(new ProducerRecord<>(topicName, "42", "42"));
		kafkaProducer.commitTransaction();
	}
	assertRecord(topicName, "42", "42");
	deleteTestTopic(topicName);
}
 

@Test(timeout = 30000L)
public void testResumeTransaction() throws IOException {
	String topicName = "flink-kafka-producer-resume-transaction";
	try (FlinkKafkaProducer<String, String> kafkaProducer = new FlinkKafkaProducer<>(extraProperties)) {
		kafkaProducer.initTransactions();
		kafkaProducer.beginTransaction();
		kafkaProducer.send(new ProducerRecord<>(topicName, "42", "42"));
		kafkaProducer.flush();
		long producerId = kafkaProducer.getProducerId();
		short epoch = kafkaProducer.getEpoch();

		try (FlinkKafkaProducer<String, String> resumeProducer = new FlinkKafkaProducer<>(extraProperties)) {
			resumeProducer.resumeTransaction(producerId, epoch);
			resumeProducer.commitTransaction();
		}

		assertRecord(topicName, "42", "42");

		// this shouldn't throw - in case of network split, old producer might attempt to commit it's transaction
		kafkaProducer.commitTransaction();

		// this shouldn't fail also, for same reason as above
		try (FlinkKafkaProducer<String, String> resumeProducer = new FlinkKafkaProducer<>(extraProperties)) {
			resumeProducer.resumeTransaction(producerId, epoch);
			resumeProducer.commitTransaction();
		}
	}
	deleteTestTopic(topicName);
}
 

@Test(timeout = 30000L, expected = IllegalStateException.class)
public void testBeginTransactionAfterClosed() {
	FlinkKafkaProducer<String, String> kafkaProducer = new FlinkKafkaProducer<>(extraProperties);
	kafkaProducer.initTransactions();
	kafkaProducer.close();
	kafkaProducer.beginTransaction();
}
 

@Test(timeout = 30000L, expected = IllegalStateException.class)
public void testAbortTransactionAfterClosed() {
	String topicName = "testAbortTransactionAfterClosed";
	FlinkKafkaProducer<String, String> kafkaProducer = getClosedProducer(topicName);
	kafkaProducer.abortTransaction();
	kafkaProducer.resumeTransaction(0L, (short) 1);
}
 

private FlinkKafkaProducer<String, String> getClosedProducer(String topicName) {
	FlinkKafkaProducer<String, String> kafkaProducer = new FlinkKafkaProducer<>(extraProperties);
	kafkaProducer.initTransactions();
	kafkaProducer.beginTransaction();
	kafkaProducer.send(new ProducerRecord<>(topicName, "42", "42"));
	kafkaProducer.close();
	return kafkaProducer;
}
 

@Override
protected void recoverAndAbort(KafkaTransactionState transaction) {
	if (transaction.isTransactional()) {
		try (
			FlinkKafkaProducer<byte[], byte[]> producer =
				initTransactionalProducer(transaction.transactionalId, false)) {
			producer.initTransactions();
		}
	}
}
 

/**
 * For each checkpoint we create new {@link FlinkKafkaProducer} so that new transactions will not clash
 * with transactions created during previous checkpoints ({@code producer.initTransactions()} assures that we
 * obtain new producerId and epoch counters).
 */
private FlinkKafkaProducer<byte[], byte[]> createTransactionalProducer() throws FlinkKafka011Exception {
	String transactionalId = availableTransactionalIds.poll();
	if (transactionalId == null) {
		throw new FlinkKafka011Exception(
			FlinkKafka011ErrorCode.PRODUCERS_POOL_EMPTY,
			"Too many ongoing snapshots. Increase kafka producers pool size or decrease number of concurrent checkpoints.");
	}
	FlinkKafkaProducer<byte[], byte[]> producer = initTransactionalProducer(transactionalId, true);
	producer.initTransactions();
	return producer;
}
 

@VisibleForTesting
@Internal
public KafkaTransactionState(
		@Nullable String transactionalId,
		long producerId,
		short epoch,
		FlinkKafkaProducer<byte[], byte[]> producer) {
	this.transactionalId = transactionalId;
	this.producerId = producerId;
	this.epoch = epoch;
	this.producer = producer;
}
 

@Test(timeout = 30000L)
public void testHappyPath() throws IOException {
	String topicName = "flink-kafka-producer-happy-path";
	try (Producer<String, String> kafkaProducer = new FlinkKafkaProducer<>(extraProperties)) {
		kafkaProducer.initTransactions();
		kafkaProducer.beginTransaction();
		kafkaProducer.send(new ProducerRecord<>(topicName, "42", "42"));
		kafkaProducer.commitTransaction();
	}
	assertRecord(topicName, "42", "42");
	deleteTestTopic(topicName);
}
 

@Test(timeout = 30000L)
public void testResumeTransaction() throws IOException {
	String topicName = "flink-kafka-producer-resume-transaction";
	try (FlinkKafkaProducer<String, String> kafkaProducer = new FlinkKafkaProducer<>(extraProperties)) {
		kafkaProducer.initTransactions();
		kafkaProducer.beginTransaction();
		kafkaProducer.send(new ProducerRecord<>(topicName, "42", "42"));
		kafkaProducer.flush();
		long producerId = kafkaProducer.getProducerId();
		short epoch = kafkaProducer.getEpoch();

		try (FlinkKafkaProducer<String, String> resumeProducer = new FlinkKafkaProducer<>(extraProperties)) {
			resumeProducer.resumeTransaction(producerId, epoch);
			resumeProducer.commitTransaction();
		}

		assertRecord(topicName, "42", "42");

		// this shouldn't throw - in case of network split, old producer might attempt to commit it's transaction
		kafkaProducer.commitTransaction();

		// this shouldn't fail also, for same reason as above
		try (FlinkKafkaProducer<String, String> resumeProducer = new FlinkKafkaProducer<>(extraProperties)) {
			resumeProducer.resumeTransaction(producerId, epoch);
			resumeProducer.commitTransaction();
		}
	}
	deleteTestTopic(topicName);
}
 

@Test(timeout = 30000L, expected = IllegalStateException.class)
public void testBeginTransactionAfterClosed() {
	FlinkKafkaProducer<String, String> kafkaProducer = new FlinkKafkaProducer<>(extraProperties);
	kafkaProducer.initTransactions();
	kafkaProducer.close();
	kafkaProducer.beginTransaction();
}
 

@Test(timeout = 30000L, expected = IllegalStateException.class)
public void testAbortTransactionAfterClosed() {
	String topicName = "testAbortTransactionAfterClosed";
	FlinkKafkaProducer<String, String> kafkaProducer = getClosedProducer(topicName);
	kafkaProducer.abortTransaction();
	kafkaProducer.resumeTransaction(0L, (short) 1);
}
 

@Test(timeout = 30000L)
public void testProducerWhenCommitEmptyPartitionsToOutdatedTxnCoordinator() throws Exception {
	String topic = "flink-kafka-producer-txn-coordinator-changed";
	createTestTopic(topic, 1, 2);
	try (Producer<String, String> kafkaProducer = new FlinkKafkaProducer<>(extraProperties)) {
		kafkaProducer.initTransactions();
		kafkaProducer.beginTransaction();
		restartBroker(kafkaServer.getLeaderToShutDown("__transaction_state"));
		kafkaProducer.flush();
		kafkaProducer.commitTransaction();
	}
	deleteTestTopic(topic);
}
 

@Override
protected void recoverAndAbort(KafkaTransactionState transaction) {
	if (transaction.isTransactional()) {
		try (
			FlinkKafkaProducer<byte[], byte[]> producer =
				initTransactionalProducer(transaction.transactionalId, false)) {
			producer.initTransactions();
		}
	}
}
 

KafkaTransactionState(
		@Nullable String transactionalId,
		long producerId,
		short epoch,
		FlinkKafkaProducer<byte[], byte[]> producer) {
	this.transactionalId = transactionalId;
	this.producerId = producerId;
	this.epoch = epoch;
	this.producer = producer;
}
 

@Test(timeout = 30000L)
public void testHappyPath() throws IOException {
	String topicName = "flink-kafka-producer-happy-path";
	try (Producer<String, String> kafkaProducer = new FlinkKafkaProducer<>(extraProperties)) {
		kafkaProducer.initTransactions();
		kafkaProducer.beginTransaction();
		kafkaProducer.send(new ProducerRecord<>(topicName, "42", "42"));
		kafkaProducer.commitTransaction();
	}
	assertRecord(topicName, "42", "42");
	deleteTestTopic(topicName);
}
 

@Test(timeout = 30000L)
public void testResumeTransaction() throws IOException {
	String topicName = "flink-kafka-producer-resume-transaction";
	try (FlinkKafkaProducer<String, String> kafkaProducer = new FlinkKafkaProducer<>(extraProperties)) {
		kafkaProducer.initTransactions();
		kafkaProducer.beginTransaction();
		kafkaProducer.send(new ProducerRecord<>(topicName, "42", "42"));
		kafkaProducer.flush();
		long producerId = kafkaProducer.getProducerId();
		short epoch = kafkaProducer.getEpoch();

		try (FlinkKafkaProducer<String, String> resumeProducer = new FlinkKafkaProducer<>(extraProperties)) {
			resumeProducer.resumeTransaction(producerId, epoch);
			resumeProducer.commitTransaction();
		}

		assertRecord(topicName, "42", "42");

		// this shouldn't throw - in case of network split, old producer might attempt to commit it's transaction
		kafkaProducer.commitTransaction();

		// this shouldn't fail also, for same reason as above
		try (FlinkKafkaProducer<String, String> resumeProducer = new FlinkKafkaProducer<>(extraProperties)) {
			resumeProducer.resumeTransaction(producerId, epoch);
			resumeProducer.commitTransaction();
		}
	}
	deleteTestTopic(topicName);
}
 

@Override
protected void recoverAndAbort(KafkaTransactionState transaction) {
	if (transaction.isTransactional()) {
		try (
			FlinkKafkaProducer<byte[], byte[]> producer =
				initTransactionalProducer(transaction.transactionalId, false)) {
			producer.initTransactions();
		}
	}
}
 

/**
 * For each checkpoint we create new {@link FlinkKafkaProducer} so that new transactions will not clash
 * with transactions created during previous checkpoints ({@code producer.initTransactions()} assures that we
 * obtain new producerId and epoch counters).
 */
private FlinkKafkaProducer<byte[], byte[]> createTransactionalProducer() throws FlinkKafka011Exception {
	String transactionalId = availableTransactionalIds.poll();
	if (transactionalId == null) {
		throw new FlinkKafka011Exception(
			FlinkKafka011ErrorCode.PRODUCERS_POOL_EMPTY,
			"Too many ongoing snapshots. Increase kafka producers pool size or decrease number of concurrent checkpoints.");
	}
	FlinkKafkaProducer<byte[], byte[]> producer = initTransactionalProducer(transactionalId, true);
	producer.initTransactions();
	return producer;
}
 

/**
 * For each checkpoint we create new {@link FlinkKafkaProducer} so that new transactions will not clash
 * with transactions created during previous checkpoints ({@code producer.initTransactions()} assures that we
 * obtain new producerId and epoch counters).
 */
private FlinkKafkaProducer<byte[], byte[]> createTransactionalProducer() throws FlinkKafka011Exception {
	String transactionalId = availableTransactionalIds.poll();
	if (transactionalId == null) {
		throw new FlinkKafka011Exception(
			FlinkKafka011ErrorCode.PRODUCERS_POOL_EMPTY,
			"Too many ongoing snapshots. Increase kafka producers pool size or decrease number of concurrent checkpoints.");
	}
	FlinkKafkaProducer<byte[], byte[]> producer = initTransactionalProducer(transactionalId, true);
	producer.initTransactions();
	return producer;
}