Java Code Examples for org.apache.flink.streaming.connectors.kafka.internal.FlinkKafkaInternalProducer#initTransactions()

/**
 * For each checkpoint we create new {@link FlinkKafkaInternalProducer} so that new transactions will not clash
 * with transactions created during previous checkpoints ({@code producer.initTransactions()} assures that we
 * obtain new producerId and epoch counters).
 */
private FlinkKafkaInternalProducer<byte[], byte[]> createTransactionalProducer() throws FlinkKafkaException {
	String transactionalId = availableTransactionalIds.poll();
	if (transactionalId == null) {
		throw new FlinkKafkaException(
			FlinkKafkaErrorCode.PRODUCERS_POOL_EMPTY,
			"Too many ongoing snapshots. Increase kafka producers pool size or decrease number of concurrent checkpoints.");
	}
	FlinkKafkaInternalProducer<byte[], byte[]> producer = initTransactionalProducer(transactionalId, true);
	producer.initTransactions();
	return producer;
}
 

/**
 * For each checkpoint we create new {@link FlinkKafkaInternalProducer} so that new transactions will not clash
 * with transactions created during previous checkpoints ({@code producer.initTransactions()} assures that we
 * obtain new producerId and epoch counters).
 */
private FlinkKafkaInternalProducer<byte[], byte[]> createTransactionalProducer() throws FlinkKafkaException {
	String transactionalId = availableTransactionalIds.poll();
	if (transactionalId == null) {
		throw new FlinkKafkaException(
			FlinkKafkaErrorCode.PRODUCERS_POOL_EMPTY,
			"Too many ongoing snapshots. Increase kafka producers pool size or decrease number of concurrent checkpoints.");
	}
	FlinkKafkaInternalProducer<byte[], byte[]> producer = initTransactionalProducer(transactionalId, true);
	producer.initTransactions();
	return producer;
}
 

@Test(timeout = 30000L, expected = IllegalStateException.class)
public void testBeginTransactionAfterClosed() {
	FlinkKafkaInternalProducer<String, String> kafkaProducer = new FlinkKafkaInternalProducer<>(extraProperties);
	kafkaProducer.initTransactions();
	kafkaProducer.close();
	kafkaProducer.beginTransaction();
}
 

private FlinkKafkaInternalProducer<String, String> getClosedProducer(String topicName) {
	FlinkKafkaInternalProducer<String, String> kafkaProducer = new FlinkKafkaInternalProducer<>(extraProperties);
	kafkaProducer.initTransactions();
	kafkaProducer.beginTransaction();
	kafkaProducer.send(new ProducerRecord<>(topicName, "42", "42"));
	kafkaProducer.close();
	return kafkaProducer;
}
 

@Override
protected void recoverAndAbort(FlinkKafkaProducer.KafkaTransactionState transaction) {
	if (transaction.isTransactional()) {
		FlinkKafkaInternalProducer<byte[], byte[]> producer = null;
		try {
			producer =
					initTransactionalProducer(transaction.transactionalId, false);
			producer.initTransactions();
		} finally {
			if (producer != null) {
				producer.close(0, TimeUnit.SECONDS);
			}
		}
	}
}
 

/**
 * For each checkpoint we create new {@link FlinkKafkaInternalProducer} so that new transactions will not clash
 * with transactions created during previous checkpoints ({@code producer.initTransactions()} assures that we
 * obtain new producerId and epoch counters).
 */
private FlinkKafkaInternalProducer<byte[], byte[]> createTransactionalProducer() throws FlinkKafkaException {
	String transactionalId = availableTransactionalIds.poll();
	if (transactionalId == null) {
		throw new FlinkKafkaException(
			FlinkKafkaErrorCode.PRODUCERS_POOL_EMPTY,
			"Too many ongoing snapshots. Increase kafka producers pool size or decrease number of concurrent checkpoints.");
	}
	FlinkKafkaInternalProducer<byte[], byte[]> producer = initTransactionalProducer(transactionalId, true);
	producer.initTransactions();
	return producer;
}
 

@Test(timeout = 30000L)
public void testResumeTransaction() throws IOException {
	String topicName = "flink-kafka-producer-resume-transaction";
	FlinkKafkaInternalProducer<String, String> kafkaProducer = new FlinkKafkaInternalProducer<>(extraProperties);
	try {
		kafkaProducer.initTransactions();
		kafkaProducer.beginTransaction();
		kafkaProducer.send(new ProducerRecord<>(topicName, "42", "42"));
		kafkaProducer.flush();
		long producerId = kafkaProducer.getProducerId();
		short epoch = kafkaProducer.getEpoch();

		FlinkKafkaInternalProducer<String, String> resumeProducer = new FlinkKafkaInternalProducer<>(extraProperties);
		try {
			resumeProducer.resumeTransaction(producerId, epoch);
			resumeProducer.commitTransaction();
		} finally {
			resumeProducer.close(Duration.ofSeconds(5));
		}

		assertRecord(topicName, "42", "42");

		// this shouldn't throw - in case of network split, old producer might attempt to commit it's transaction
		kafkaProducer.commitTransaction();

		// this shouldn't fail also, for same reason as above
		resumeProducer = new FlinkKafkaInternalProducer<>(extraProperties);
		try {
			resumeProducer.resumeTransaction(producerId, epoch);
			resumeProducer.commitTransaction();
		} finally {
			resumeProducer.close(Duration.ofSeconds(5));
		}
	} finally {
		kafkaProducer.close(Duration.ofSeconds(5));
	}
	deleteTestTopic(topicName);
}
 

@Test(timeout = 30000L, expected = IllegalStateException.class)
public void testBeginTransactionAfterClosed() {
	FlinkKafkaInternalProducer<String, String> kafkaProducer = new FlinkKafkaInternalProducer<>(extraProperties);
	kafkaProducer.initTransactions();
	kafkaProducer.close(Duration.ofSeconds(5));
	kafkaProducer.beginTransaction();
}
 

private FlinkKafkaInternalProducer<String, String> getClosedProducer(String topicName) {
	FlinkKafkaInternalProducer<String, String> kafkaProducer = new FlinkKafkaInternalProducer<>(extraProperties);
	kafkaProducer.initTransactions();
	kafkaProducer.beginTransaction();
	kafkaProducer.send(new ProducerRecord<>(topicName, "42", "42"));
	kafkaProducer.close(Duration.ofSeconds(5));
	return kafkaProducer;
}
 

@Test(timeout = 30000L, expected = IllegalStateException.class)
public void testInitTransactionsAfterClosed() {
	FlinkKafkaInternalProducer<String, String> kafkaProducer = new FlinkKafkaInternalProducer<>(extraProperties);
	kafkaProducer.close();
	kafkaProducer.initTransactions();
}
 

@Test(timeout = 30000L, expected = IllegalStateException.class)
public void testInitTransactionsAfterClosed() {
	FlinkKafkaInternalProducer<String, String> kafkaProducer = new FlinkKafkaInternalProducer<>(extraProperties);
	kafkaProducer.close(Duration.ofSeconds(5));
	kafkaProducer.initTransactions();
}