org.apache.flink.streaming.connectors.kafka.internal.FlinkKafkaInternalProducer Java Examples

@Override
protected void recoverAndCommit(FlinkKafkaProducer.KafkaTransactionState transaction) {
	if (transaction.isTransactional()) {
		try (
			FlinkKafkaInternalProducer<byte[], byte[]> producer =
				initTransactionalProducer(transaction.transactionalId, false)) {
			producer.resumeTransaction(transaction.producerId, transaction.epoch);
			producer.commitTransaction();
		} catch (InvalidTxnStateException | ProducerFencedException ex) {
			// That means we have committed this transaction before.
			LOG.warn("Encountered error {} while recovering transaction {}. " +
					"Presumably this transaction has been already committed before",
				ex,
				transaction);
		}
	}
}
 

private void abortTransactions(final Set<String> transactionalIds) {
	transactionalIds.parallelStream().forEach(transactionalId -> {
		// don't mess with the original configuration or any other properties of the
		// original object
		// -> create an internal kafka producer on our own and do not rely on
		//    initTransactionalProducer().
		final Properties myConfig = new Properties();
		myConfig.putAll(producerConfig);
		initTransactionalProducerConfig(myConfig, transactionalId);
		try (FlinkKafkaInternalProducer<byte[], byte[]> kafkaProducer =
				new FlinkKafkaInternalProducer<>(myConfig)) {
			// it suffices to call initTransactions - this will abort any lingering transactions
			kafkaProducer.initTransactions();
		}
	});
}
 

@Override
protected void recoverAndCommit(FlinkKafkaProducer.KafkaTransactionState transaction) {
	if (transaction.isTransactional()) {
		try (
			FlinkKafkaInternalProducer<byte[], byte[]> producer =
				initTransactionalProducer(transaction.transactionalId, false)) {
			producer.resumeTransaction(transaction.producerId, transaction.epoch);
			producer.commitTransaction();
		} catch (InvalidTxnStateException | ProducerFencedException ex) {
			// That means we have committed this transaction before.
			LOG.warn("Encountered error {} while recovering transaction {}. " +
					"Presumably this transaction has been already committed before",
				ex,
				transaction);
		}
	}
}
 

@Override
protected FlinkKafkaProducer.KafkaTransactionState beginTransaction() throws FlinkKafkaException {
	switch (semantic) {
		case EXACTLY_ONCE:
			FlinkKafkaInternalProducer<byte[], byte[]> producer = createTransactionalProducer();
			producer.beginTransaction();
			return new FlinkKafkaProducer.KafkaTransactionState(producer.getTransactionalId(), producer);
		case AT_LEAST_ONCE:
		case NONE:
			// Do not create new producer on each beginTransaction() if it is not necessary
			final FlinkKafkaProducer.KafkaTransactionState currentTransaction = currentTransaction();
			if (currentTransaction != null && currentTransaction.producer != null) {
				return new FlinkKafkaProducer.KafkaTransactionState(currentTransaction.producer);
			}
			return new FlinkKafkaProducer.KafkaTransactionState(initNonTransactionalProducer(true));
		default:
			throw new UnsupportedOperationException("Not implemented semantic");
	}
}
 

@Test(timeout = 30000L)
public void testHappyPath() throws IOException {
	String topicName = "flink-kafka-producer-happy-path";

	Producer<String, String> kafkaProducer = new FlinkKafkaInternalProducer<>(extraProperties);
	try {
		kafkaProducer.initTransactions();
		kafkaProducer.beginTransaction();
		kafkaProducer.send(new ProducerRecord<>(topicName, "42", "42"));
		kafkaProducer.commitTransaction();
	} finally {
		kafkaProducer.close(Duration.ofSeconds(5));
	}
	assertRecord(topicName, "42", "42");
	deleteTestTopic(topicName);
}
 

@Test(timeout = 30000L)
public void testProducerWhenCommitEmptyPartitionsToOutdatedTxnCoordinator() throws Exception {
	String topic = "flink-kafka-producer-txn-coordinator-changed";
	createTestTopic(topic, 1, 2);
	Producer<String, String> kafkaProducer = new FlinkKafkaInternalProducer<>(extraProperties);
	try {
		kafkaProducer.initTransactions();
		kafkaProducer.beginTransaction();
		restartBroker(kafkaServer.getLeaderToShutDown("__transaction_state"));
		kafkaProducer.flush();
		kafkaProducer.commitTransaction();
	} finally {
		kafkaProducer.close(Duration.ofSeconds(5));
	}
	deleteTestTopic(topic);
}
 

@Override
protected FlinkKafkaProducer.KafkaTransactionState beginTransaction() throws FlinkKafkaException {
	switch (semantic) {
		case EXACTLY_ONCE:
			FlinkKafkaInternalProducer<byte[], byte[]> producer = createTransactionalProducer();
			producer.beginTransaction();
			return new FlinkKafkaProducer.KafkaTransactionState(producer.getTransactionalId(), producer);
		case AT_LEAST_ONCE:
		case NONE:
			// Do not create new producer on each beginTransaction() if it is not necessary
			final FlinkKafkaProducer.KafkaTransactionState currentTransaction = currentTransaction();
			if (currentTransaction != null && currentTransaction.producer != null) {
				return new FlinkKafkaProducer.KafkaTransactionState(currentTransaction.producer);
			}
			return new FlinkKafkaProducer.KafkaTransactionState(initNonTransactionalProducer(true));
		default:
			throw new UnsupportedOperationException("Not implemented semantic");
	}
}
 

@Override
protected FlinkKafkaProducer.KafkaTransactionState beginTransaction() throws FlinkKafkaException {
	switch (semantic) {
		case EXACTLY_ONCE:
			FlinkKafkaInternalProducer<byte[], byte[]> producer = createTransactionalProducer();
			producer.beginTransaction();
			return new FlinkKafkaProducer.KafkaTransactionState(producer.getTransactionalId(), producer);
		case AT_LEAST_ONCE:
		case NONE:
			// Do not create new producer on each beginTransaction() if it is not necessary
			final FlinkKafkaProducer.KafkaTransactionState currentTransaction = currentTransaction();
			if (currentTransaction != null && currentTransaction.producer != null) {
				return new FlinkKafkaProducer.KafkaTransactionState(currentTransaction.producer);
			}
			return new FlinkKafkaProducer.KafkaTransactionState(initNonTransactionalProducer(true));
		default:
			throw new UnsupportedOperationException("Not implemented semantic");
	}
}
 

@Override
protected void recoverAndCommit(FlinkKafkaProducer.KafkaTransactionState transaction) {
	if (transaction.isTransactional()) {
		FlinkKafkaInternalProducer<byte[], byte[]> producer = null;
		try {
			producer =
				initTransactionalProducer(transaction.transactionalId, false);
			producer.resumeTransaction(transaction.producerId, transaction.epoch);
			producer.commitTransaction();
		} catch (InvalidTxnStateException | ProducerFencedException ex) {
			// That means we have committed this transaction before.
			LOG.warn("Encountered error {} while recovering transaction {}. " +
					"Presumably this transaction has been already committed before",
				ex,
				transaction);
		} finally {
			if (producer != null) {
				producer.close(0, TimeUnit.SECONDS);
			}
		}
	}
}
 

@Test(timeout = 30000L, expected = IllegalStateException.class)
public void testBeginTransactionAfterClosed() {
	FlinkKafkaInternalProducer<String, String> kafkaProducer = new FlinkKafkaInternalProducer<>(extraProperties);
	kafkaProducer.initTransactions();
	kafkaProducer.close(Duration.ofSeconds(5));
	kafkaProducer.beginTransaction();
}
 

/**
 * For each checkpoint we create new {@link FlinkKafkaInternalProducer} so that new transactions will not clash
 * with transactions created during previous checkpoints ({@code producer.initTransactions()} assures that we
 * obtain new producerId and epoch counters).
 */
private FlinkKafkaInternalProducer<byte[], byte[]> createTransactionalProducer() throws FlinkKafkaException {
	String transactionalId = availableTransactionalIds.poll();
	if (transactionalId == null) {
		throw new FlinkKafkaException(
			FlinkKafkaErrorCode.PRODUCERS_POOL_EMPTY,
			"Too many ongoing snapshots. Increase kafka producers pool size or decrease number of concurrent checkpoints.");
	}
	FlinkKafkaInternalProducer<byte[], byte[]> producer = initTransactionalProducer(transactionalId, true);
	producer.initTransactions();
	return producer;
}
 

private void abortTransactions(Set<String> transactionalIds) {
	for (String transactionalId : transactionalIds) {
		try (FlinkKafkaInternalProducer<byte[], byte[]> kafkaProducer =
			initTransactionalProducer(transactionalId, false)) {
			// it suffice to call initTransactions - this will abort any lingering transactions
			kafkaProducer.initTransactions();
		}
	}
}
 

@Override
protected void recoverAndAbort(FlinkKafkaProducer.KafkaTransactionState transaction) {
	if (transaction.isTransactional()) {
		try (
			FlinkKafkaInternalProducer<byte[], byte[]> producer =
				initTransactionalProducer(transaction.transactionalId, false)) {
			producer.initTransactions();
		}
	}
}
 

/**
 * For each checkpoint we create new {@link FlinkKafkaInternalProducer} so that new transactions will not clash
 * with transactions created during previous checkpoints ({@code producer.initTransactions()} assures that we
 * obtain new producerId and epoch counters).
 */
private FlinkKafkaInternalProducer<byte[], byte[]> createTransactionalProducer() throws FlinkKafkaException {
	String transactionalId = availableTransactionalIds.poll();
	if (transactionalId == null) {
		throw new FlinkKafkaException(
			FlinkKafkaErrorCode.PRODUCERS_POOL_EMPTY,
			"Too many ongoing snapshots. Increase kafka producers pool size or decrease number of concurrent checkpoints.");
	}
	FlinkKafkaInternalProducer<byte[], byte[]> producer = initTransactionalProducer(transactionalId, true);
	producer.initTransactions();
	return producer;
}
 

@Test(timeout = 30000L)
public void testHappyPath() throws IOException {
	String topicName = "flink-kafka-producer-happy-path";
	try (Producer<String, String> kafkaProducer = new FlinkKafkaInternalProducer<>(extraProperties)) {
		kafkaProducer.initTransactions();
		kafkaProducer.beginTransaction();
		kafkaProducer.send(new ProducerRecord<>(topicName, "42", "42"));
		kafkaProducer.commitTransaction();
	}
	assertRecord(topicName, "42", "42");
	deleteTestTopic(topicName);
}
 

KafkaTransactionState(
	@Nullable String transactionalId,
	long producerId,
	short epoch,
	FlinkKafkaInternalProducer<byte[], byte[]> producer) {
	this.transactionalId = transactionalId;
	this.producerId = producerId;
	this.epoch = epoch;
	this.producer = producer;
}
 

@Test(timeout = 30000L, expected = IllegalStateException.class)
public void testAbortTransactionAfterClosed() {
	String topicName = "testAbortTransactionAfterClosed";
	FlinkKafkaInternalProducer<String, String> kafkaProducer = getClosedProducer(topicName);
	kafkaProducer.abortTransaction();
	kafkaProducer.resumeTransaction(0L, (short) 1);
}
 

private FlinkKafkaInternalProducer<String, String> getClosedProducer(String topicName) {
	FlinkKafkaInternalProducer<String, String> kafkaProducer = new FlinkKafkaInternalProducer<>(extraProperties);
	kafkaProducer.initTransactions();
	kafkaProducer.beginTransaction();
	kafkaProducer.send(new ProducerRecord<>(topicName, "42", "42"));
	kafkaProducer.close(Duration.ofSeconds(5));
	return kafkaProducer;
}
 

@Test(timeout = 30000L)
public void testResumeTransaction() throws IOException {
	String topicName = "flink-kafka-producer-resume-transaction";
	try (FlinkKafkaInternalProducer<String, String> kafkaProducer = new FlinkKafkaInternalProducer<>(extraProperties)) {
		kafkaProducer.initTransactions();
		kafkaProducer.beginTransaction();
		kafkaProducer.send(new ProducerRecord<>(topicName, "42", "42"));
		kafkaProducer.flush();
		long producerId = kafkaProducer.getProducerId();
		short epoch = kafkaProducer.getEpoch();

		try (FlinkKafkaInternalProducer<String, String> resumeProducer = new FlinkKafkaInternalProducer<>(extraProperties)) {
			resumeProducer.resumeTransaction(producerId, epoch);
			resumeProducer.commitTransaction();
		}

		assertRecord(topicName, "42", "42");

		// this shouldn't throw - in case of network split, old producer might attempt to commit it's transaction
		kafkaProducer.commitTransaction();

		// this shouldn't fail also, for same reason as above
		try (FlinkKafkaInternalProducer<String, String> resumeProducer = new FlinkKafkaInternalProducer<>(extraProperties)) {
			resumeProducer.resumeTransaction(producerId, epoch);
			resumeProducer.commitTransaction();
		}
	}
	deleteTestTopic(topicName);
}
 

@Test(timeout = 30000L)
public void testResumeTransaction() throws IOException {
	String topicName = "flink-kafka-producer-resume-transaction";
	FlinkKafkaInternalProducer<String, String> kafkaProducer = new FlinkKafkaInternalProducer<>(extraProperties);
	try {
		kafkaProducer.initTransactions();
		kafkaProducer.beginTransaction();
		kafkaProducer.send(new ProducerRecord<>(topicName, "42", "42"));
		kafkaProducer.flush();
		long producerId = kafkaProducer.getProducerId();
		short epoch = kafkaProducer.getEpoch();

		FlinkKafkaInternalProducer<String, String> resumeProducer = new FlinkKafkaInternalProducer<>(extraProperties);
		try {
			resumeProducer.resumeTransaction(producerId, epoch);
			resumeProducer.commitTransaction();
		} finally {
			resumeProducer.close(Duration.ofSeconds(5));
		}

		assertRecord(topicName, "42", "42");

		// this shouldn't throw - in case of network split, old producer might attempt to commit it's transaction
		kafkaProducer.commitTransaction();

		// this shouldn't fail also, for same reason as above
		resumeProducer = new FlinkKafkaInternalProducer<>(extraProperties);
		try {
			resumeProducer.resumeTransaction(producerId, epoch);
			resumeProducer.commitTransaction();
		} finally {
			resumeProducer.close(Duration.ofSeconds(5));
		}
	} finally {
		kafkaProducer.close(Duration.ofSeconds(5));
	}
	deleteTestTopic(topicName);
}
 

KafkaTransactionState(
	@Nullable String transactionalId,
	long producerId,
	short epoch,
	FlinkKafkaInternalProducer<byte[], byte[]> producer) {
	this.transactionalId = transactionalId;
	this.producerId = producerId;
	this.epoch = epoch;
	this.producer = producer;
}
 

@Test(timeout = 30000L)
public void testHappyPath() throws IOException {
	String topicName = "flink-kafka-producer-happy-path";
	try (Producer<String, String> kafkaProducer = new FlinkKafkaInternalProducer<>(extraProperties)) {
		kafkaProducer.initTransactions();
		kafkaProducer.beginTransaction();
		kafkaProducer.send(new ProducerRecord<>(topicName, "42", "42"));
		kafkaProducer.commitTransaction();
	}
	assertRecord(topicName, "42", "42");
	deleteTestTopic(topicName);
}
 

@Test(timeout = 30000L)
public void testResumeTransaction() throws IOException {
	String topicName = "flink-kafka-producer-resume-transaction";
	try (FlinkKafkaInternalProducer<String, String> kafkaProducer = new FlinkKafkaInternalProducer<>(extraProperties)) {
		kafkaProducer.initTransactions();
		kafkaProducer.beginTransaction();
		kafkaProducer.send(new ProducerRecord<>(topicName, "42", "42"));
		kafkaProducer.flush();
		long producerId = kafkaProducer.getProducerId();
		short epoch = kafkaProducer.getEpoch();

		try (FlinkKafkaInternalProducer<String, String> resumeProducer = new FlinkKafkaInternalProducer<>(extraProperties)) {
			resumeProducer.resumeTransaction(producerId, epoch);
			resumeProducer.commitTransaction();
		}

		assertRecord(topicName, "42", "42");

		// this shouldn't throw - in case of network split, old producer might attempt to commit it's transaction
		kafkaProducer.commitTransaction();

		// this shouldn't fail also, for same reason as above
		try (FlinkKafkaInternalProducer<String, String> resumeProducer = new FlinkKafkaInternalProducer<>(extraProperties)) {
			resumeProducer.resumeTransaction(producerId, epoch);
			resumeProducer.commitTransaction();
		}
	}
	deleteTestTopic(topicName);
}
 

/**
 * For each checkpoint we create new {@link FlinkKafkaInternalProducer} so that new transactions will not clash
 * with transactions created during previous checkpoints ({@code producer.initTransactions()} assures that we
 * obtain new producerId and epoch counters).
 */
private FlinkKafkaInternalProducer<byte[], byte[]> createTransactionalProducer() throws FlinkKafkaException {
	String transactionalId = availableTransactionalIds.poll();
	if (transactionalId == null) {
		throw new FlinkKafkaException(
			FlinkKafkaErrorCode.PRODUCERS_POOL_EMPTY,
			"Too many ongoing snapshots. Increase kafka producers pool size or decrease number of concurrent checkpoints.");
	}
	FlinkKafkaInternalProducer<byte[], byte[]> producer = initTransactionalProducer(transactionalId, true);
	producer.initTransactions();
	return producer;
}
 

private void abortTransactions(final Set<String> transactionalIds) {
	final ClassLoader classLoader = Thread.currentThread().getContextClassLoader();
	transactionalIds.parallelStream().forEach(transactionalId -> {
		// The parallelStream executes the consumer in a separated thread pool.
		// Because the consumer(e.g. Kafka) uses the context classloader to construct some class
		// we should set the correct classloader for it.
		try (TemporaryClassLoaderContext ignored = TemporaryClassLoaderContext.of(classLoader)) {
			// don't mess with the original configuration or any other properties of the
			// original object
			// -> create an internal kafka producer on our own and do not rely on
			//    initTransactionalProducer().
			final Properties myConfig = new Properties();
			myConfig.putAll(producerConfig);
			initTransactionalProducerConfig(myConfig, transactionalId);
			FlinkKafkaInternalProducer<byte[], byte[]> kafkaProducer = null;
			try {
				kafkaProducer =
						new FlinkKafkaInternalProducer<>(myConfig);
				// it suffices to call initTransactions - this will abort any lingering transactions
				kafkaProducer.initTransactions();
			} finally {
				if (kafkaProducer != null) {
					kafkaProducer.close(Duration.ofSeconds(0));
				}
			}
		}
	});
}
 

@Test(timeout = 30000L, expected = IllegalStateException.class)
public void testBeginTransactionAfterClosed() {
	FlinkKafkaInternalProducer<String, String> kafkaProducer = new FlinkKafkaInternalProducer<>(extraProperties);
	kafkaProducer.initTransactions();
	kafkaProducer.close();
	kafkaProducer.beginTransaction();
}
 

@VisibleForTesting
public KafkaTransactionState(
	@Nullable String transactionalId,
	long producerId,
	short epoch,
	FlinkKafkaInternalProducer<byte[], byte[]> producer) {
	this.transactionalId = transactionalId;
	this.producerId = producerId;
	this.epoch = epoch;
	this.producer = producer;
}
 

@Test(timeout = 30000L, expected = IllegalStateException.class)
public void testAbortTransactionAfterClosed() {
	String topicName = "testAbortTransactionAfterClosed";
	FlinkKafkaInternalProducer<String, String> kafkaProducer = getClosedProducer(topicName);
	kafkaProducer.abortTransaction();
	kafkaProducer.resumeTransaction(0L, (short) 1);
}
 

private FlinkKafkaInternalProducer<String, String> getClosedProducer(String topicName) {
	FlinkKafkaInternalProducer<String, String> kafkaProducer = new FlinkKafkaInternalProducer<>(extraProperties);
	kafkaProducer.initTransactions();
	kafkaProducer.beginTransaction();
	kafkaProducer.send(new ProducerRecord<>(topicName, "42", "42"));
	kafkaProducer.close();
	return kafkaProducer;
}
 

@Override
protected void recoverAndAbort(FlinkKafkaProducer.KafkaTransactionState transaction) {
	if (transaction.isTransactional()) {
		FlinkKafkaInternalProducer<byte[], byte[]> producer = null;
		try {
			producer =
					initTransactionalProducer(transaction.transactionalId, false);
			producer.initTransactions();
		} finally {
			if (producer != null) {
				producer.close(0, TimeUnit.SECONDS);
			}
		}
	}
}