Java Code Examples for org.apache.flink.streaming.util.OneInputStreamOperatorTestHarness#notifyOfCompletedCheckpoint()

@Test
public void testRecoverCommittedTransaction() throws Exception {
	String topic = "flink-kafka-producer-recover-committed-transaction";

	OneInputStreamOperatorTestHarness<Integer, Object> testHarness = createTestHarness(topic);

	testHarness.setup();
	testHarness.open(); // producerA - start transaction (txn) 0
	testHarness.processElement(42, 0); // producerA - write 42 in txn 0
	OperatorSubtaskState checkpoint0 = testHarness.snapshot(0, 1); // producerA - pre commit txn 0, producerB - start txn 1
	testHarness.processElement(43, 2); // producerB - write 43 in txn 1
	testHarness.notifyOfCompletedCheckpoint(0); // producerA - commit txn 0 and return to the pool
	testHarness.snapshot(1, 3); // producerB - pre txn 1,  producerA - start txn 2
	testHarness.processElement(44, 4); // producerA - write 44 in txn 2
	testHarness.close(); // producerA - abort txn 2

	testHarness = createTestHarness(topic);
	testHarness.initializeState(checkpoint0); // recover state 0 - producerA recover and commit txn 0
	testHarness.close();

	assertExactlyOnceForTopic(createProperties(), topic, 0, Arrays.asList(42));

	deleteTestTopic(topic);
	checkProducerLeak();
}
 

@Test
public void testRecoverCommittedTransaction() throws Exception {
	String topic = "flink-kafka-producer-recover-committed-transaction";

	OneInputStreamOperatorTestHarness<Integer, Object> testHarness = createTestHarness(topic);

	testHarness.setup();
	testHarness.open(); // producerA - start transaction (txn) 0
	testHarness.processElement(42, 0); // producerA - write 42 in txn 0
	OperatorSubtaskState checkpoint0 = testHarness.snapshot(0, 1); // producerA - pre commit txn 0, producerB - start txn 1
	testHarness.processElement(43, 2); // producerB - write 43 in txn 1
	testHarness.notifyOfCompletedCheckpoint(0); // producerA - commit txn 0 and return to the pool
	testHarness.snapshot(1, 3); // producerB - pre txn 1,  producerA - start txn 2
	testHarness.processElement(44, 4); // producerA - write 44 in txn 2
	testHarness.close(); // producerA - abort txn 2

	testHarness = createTestHarness(topic);
	testHarness.initializeState(checkpoint0); // recover state 0 - producerA recover and commit txn 0
	testHarness.close();

	assertExactlyOnceForTopic(createProperties(), topic, 0, Arrays.asList(42));

	deleteTestTopic(topic);
	checkProducerLeak();
}
 

@Test
public void testRecoverCommittedTransaction() throws Exception {
	String topic = "flink-kafka-producer-recover-committed-transaction";

	OneInputStreamOperatorTestHarness<Integer, Object> testHarness = createTestHarness(topic);

	testHarness.setup();
	testHarness.open(); // producerA - start transaction (txn) 0
	testHarness.processElement(42, 0); // producerA - write 42 in txn 0
	OperatorSubtaskState checkpoint0 = testHarness.snapshot(0, 1); // producerA - pre commit txn 0, producerB - start txn 1
	testHarness.processElement(43, 2); // producerB - write 43 in txn 1
	testHarness.notifyOfCompletedCheckpoint(0); // producerA - commit txn 0 and return to the pool
	testHarness.snapshot(1, 3); // producerB - pre txn 1,  producerA - start txn 2
	testHarness.processElement(44, 4); // producerA - write 44 in txn 2
	testHarness.close(); // producerA - abort txn 2

	testHarness = createTestHarness(topic);
	testHarness.initializeState(checkpoint0); // recover state 0 - producerA recover and commit txn 0
	testHarness.close();

	assertExactlyOnceForTopic(createProperties(), topic, 0, Arrays.asList(42));

	deleteTestTopic(topic);
	checkProducerLeak();
}
 

@Test
public void testDataPersistenceUponMissedNotify() throws Exception {
	S sink = createSink();

	OneInputStreamOperatorTestHarness<IN, IN> testHarness =
			new OneInputStreamOperatorTestHarness<>(sink);

	testHarness.open();

	int elementCounter = 1;
	int snapshotCount = 0;

	for (int x = 0; x < 20; x++) {
		testHarness.processElement(new StreamRecord<>(generateValue(elementCounter, 0)));
		elementCounter++;
	}

	testHarness.snapshot(snapshotCount++, 0);
	testHarness.notifyOfCompletedCheckpoint(snapshotCount - 1);

	for (int x = 0; x < 20; x++) {
		testHarness.processElement(new StreamRecord<>(generateValue(elementCounter, 1)));
		elementCounter++;
	}

	testHarness.snapshot(snapshotCount++, 0);

	for (int x = 0; x < 20; x++) {
		testHarness.processElement(new StreamRecord<>(generateValue(elementCounter, 2)));
		elementCounter++;
	}

	testHarness.snapshot(snapshotCount++, 0);
	testHarness.notifyOfCompletedCheckpoint(snapshotCount - 1);

	verifyResultsDataPersistenceUponMissedNotify(sink);
}
 

@Test
public void testIdealCircumstances() throws Exception {
	S sink = createSink();

	OneInputStreamOperatorTestHarness<IN, IN> testHarness =
			new OneInputStreamOperatorTestHarness<>(sink);

	testHarness.open();

	int elementCounter = 1;
	int snapshotCount = 0;

	for (int x = 0; x < 20; x++) {
		testHarness.processElement(new StreamRecord<>(generateValue(elementCounter, 0)));
		elementCounter++;
	}

	testHarness.snapshot(snapshotCount++, 0);
	testHarness.notifyOfCompletedCheckpoint(snapshotCount - 1);

	for (int x = 0; x < 20; x++) {
		testHarness.processElement(new StreamRecord<>(generateValue(elementCounter, 1)));
		elementCounter++;
	}

	testHarness.snapshot(snapshotCount++, 0);
	testHarness.notifyOfCompletedCheckpoint(snapshotCount - 1);

	for (int x = 0; x < 20; x++) {
		testHarness.processElement(new StreamRecord<>(generateValue(elementCounter, 2)));
		elementCounter++;
	}

	testHarness.snapshot(snapshotCount++, 0);
	testHarness.notifyOfCompletedCheckpoint(snapshotCount - 1);

	verifyResultsIdealCircumstances(sink);
}
 

@Test
public void testIdealCircumstances() throws Exception {
	S sink = createSink();

	OneInputStreamOperatorTestHarness<IN, IN> testHarness =
			new OneInputStreamOperatorTestHarness<>(sink);

	testHarness.open();

	int elementCounter = 1;
	int snapshotCount = 0;

	for (int x = 0; x < 20; x++) {
		testHarness.processElement(new StreamRecord<>(generateValue(elementCounter, 0)));
		elementCounter++;
	}

	testHarness.snapshot(snapshotCount++, 0);
	testHarness.notifyOfCompletedCheckpoint(snapshotCount - 1);

	for (int x = 0; x < 20; x++) {
		testHarness.processElement(new StreamRecord<>(generateValue(elementCounter, 1)));
		elementCounter++;
	}

	testHarness.snapshot(snapshotCount++, 0);
	testHarness.notifyOfCompletedCheckpoint(snapshotCount - 1);

	for (int x = 0; x < 20; x++) {
		testHarness.processElement(new StreamRecord<>(generateValue(elementCounter, 2)));
		elementCounter++;
	}

	testHarness.snapshot(snapshotCount++, 0);
	testHarness.notifyOfCompletedCheckpoint(snapshotCount - 1);

	verifyResultsIdealCircumstances(sink);
}
 

@Test
public void testInactivityPeriodWithLateNotify() throws Exception {
	final File outDir = tempFolder.newFolder();

	OneInputStreamOperatorTestHarness<String, Object> testHarness = createRescalingTestSink(outDir, 1, 0, 100);
	testHarness.setup();
	testHarness.open();

	testHarness.setProcessingTime(0L);

	testHarness.processElement(new StreamRecord<>("test1", 1L));
	testHarness.processElement(new StreamRecord<>("test2", 1L));
	checkLocalFs(outDir, 2, 0 , 0, 0);

	testHarness.setProcessingTime(101L);	// put some in pending
	checkLocalFs(outDir, 0, 2, 0, 0);

	testHarness.snapshot(0, 0);				// put them in pending for 0
	checkLocalFs(outDir, 0, 2, 0, 0);

	testHarness.processElement(new StreamRecord<>("test3", 1L));
	testHarness.processElement(new StreamRecord<>("test4", 1L));

	testHarness.setProcessingTime(202L);	// put some in pending

	testHarness.snapshot(1, 0);				// put them in pending for 1
	checkLocalFs(outDir, 0, 4, 0, 0);

	testHarness.notifyOfCompletedCheckpoint(0);	// put the pending for 0 to the "committed" state
	checkLocalFs(outDir, 0, 2, 2, 0);

	testHarness.notifyOfCompletedCheckpoint(1); // put the pending for 1 to the "committed" state
	checkLocalFs(outDir, 0, 0, 4, 0);
}
 

@Test
public void testRolloverInterval() throws Exception {
	final File outDir = tempFolder.newFolder();

	OneInputStreamOperatorTestHarness<String, Object> testHarness = createRescalingTestSinkWithRollover(outDir, 1, 0, 1000L, 100L);
	testHarness.setup();
	testHarness.open();

	testHarness.setProcessingTime(0L);

	testHarness.processElement(new StreamRecord<>("test1", 1L));
	checkLocalFs(outDir, 1, 0, 0, 0);

	// invoke rollover based on rollover interval
	testHarness.setProcessingTime(101L);
	testHarness.processElement(new StreamRecord<>("test1", 2L));
	checkLocalFs(outDir, 1, 1, 0, 0);

	testHarness.snapshot(0, 0);
	testHarness.notifyOfCompletedCheckpoint(0);
	checkLocalFs(outDir, 1, 0, 1, 0);

	// move the in-progress file to pending
	testHarness.setProcessingTime(3000L);
	testHarness.snapshot(1, 1);
	checkLocalFs(outDir, 0, 1, 1, 0);

	// move the pending file to "committed"
	testHarness.notifyOfCompletedCheckpoint(1);
	testHarness.close();

	checkLocalFs(outDir, 0, 0, 2, 0);
}
 

private void testPartFiles(
		OneInputStreamOperatorTestHarness<Tuple2<String, Integer>, Object> testHarness,
		File outDir,
		String partFileName1,
		String partFileName2) throws Exception {

	testHarness.setup();
	testHarness.open();

	// this creates a new bucket "test1" and part-0-0
	testHarness.processElement(new StreamRecord<>(Tuple2.of("test1", 1), 1L));
	TestUtils.checkLocalFs(outDir, 1, 0);

	// we take a checkpoint so we roll.
	testHarness.snapshot(1L, 1L);

	// these will close part-0-0 and open part-0-1
	testHarness.processElement(new StreamRecord<>(Tuple2.of("test1", 2), 2L));
	testHarness.processElement(new StreamRecord<>(Tuple2.of("test1", 3), 3L));

	// we take a checkpoint so we roll again.
	testHarness.snapshot(2L, 2L);

	TestUtils.checkLocalFs(outDir, 2, 0);

	Map<File, String> contents = TestUtils.getFileContentByPath(outDir);
	int fileCounter = 0;
	for (Map.Entry<File, String> fileContents : contents.entrySet()) {
		if (fileContents.getKey().getName().contains(partFileName1)) {
			fileCounter++;
			Assert.assertEquals("test1@1\n", fileContents.getValue());
		} else if (fileContents.getKey().getName().contains(partFileName2)) {
			fileCounter++;
			Assert.assertEquals("test1@2\ntest1@3\n", fileContents.getValue());
		}
	}
	Assert.assertEquals(2L, fileCounter);

	// we acknowledge the latest checkpoint, so everything should be published.
	testHarness.notifyOfCompletedCheckpoint(2L);

	TestUtils.checkLocalFs(outDir, 0, 2);
}
 

/**
 * This tests checks whether FlinkKafkaProducer011 correctly aborts lingering transactions after a failure,
 * which happened before first checkpoint and was followed up by reducing the parallelism.
 * If such transactions were left alone lingering it consumers would be unable to read committed records
 * that were created after this lingering transaction.
 */
@Test
public void testScaleDownBeforeFirstCheckpoint() throws Exception {
	String topic = "scale-down-before-first-checkpoint";

	List<AutoCloseable> operatorsToClose = new ArrayList<>();
	int preScaleDownParallelism = Math.max(2, FlinkKafkaProducer011.SAFE_SCALE_DOWN_FACTOR);
	for (int subtaskIndex = 0; subtaskIndex < preScaleDownParallelism; subtaskIndex++) {
		OneInputStreamOperatorTestHarness<Integer, Object> preScaleDownOperator = createTestHarness(
			topic,
			preScaleDownParallelism,
			preScaleDownParallelism,
			subtaskIndex,
			EXACTLY_ONCE);

		preScaleDownOperator.setup();
		preScaleDownOperator.open();
		preScaleDownOperator.processElement(subtaskIndex * 2, 0);
		preScaleDownOperator.snapshot(0, 1);
		preScaleDownOperator.processElement(subtaskIndex * 2 + 1, 2);

		operatorsToClose.add(preScaleDownOperator);
	}

	// do not close previous testHarnesses to make sure that closing do not clean up something (in case of failure
	// there might not be any close)

	// After previous failure simulate restarting application with smaller parallelism
	OneInputStreamOperatorTestHarness<Integer, Object> postScaleDownOperator1 = createTestHarness(topic, 1, 1, 0, EXACTLY_ONCE);

	postScaleDownOperator1.setup();
	postScaleDownOperator1.open();

	// write and commit more records, after potentially lingering transactions
	postScaleDownOperator1.processElement(46, 7);
	postScaleDownOperator1.snapshot(4, 8);
	postScaleDownOperator1.processElement(47, 9);
	postScaleDownOperator1.notifyOfCompletedCheckpoint(4);

	//now we should have:
	// - records 42, 43, 44 and 45 in aborted transactions
	// - committed transaction with record 46
	// - pending transaction with record 47
	assertExactlyOnceForTopic(createProperties(), topic, 0, Arrays.asList(46));

	postScaleDownOperator1.close();
	// ignore ProducerFencedExceptions, because postScaleDownOperator1 could reuse transactional ids.
	for (AutoCloseable operatorToClose : operatorsToClose) {
		closeIgnoringProducerFenced(operatorToClose);
	}
	deleteTestTopic(topic);
	checkProducerLeak();
}
 

@Test
/**
 * Verifies that exceptions thrown by a committer do not fail a job and lead to an abort of notify()
 * and later retry of the affected checkpoints.
 */
public void testCommitterException() throws Exception {

	ListSink2 sink = new ListSink2();

	OneInputStreamOperatorTestHarness<Tuple1<Integer>, Tuple1<Integer>> testHarness =
			new OneInputStreamOperatorTestHarness<>(sink);

	testHarness.open();

	int elementCounter = 1;

	for (int x = 0; x < 10; x++) {
		testHarness.processElement(new StreamRecord<>(generateValue(elementCounter, 0)));
		elementCounter++;
	}

	testHarness.snapshot(0, 0);
	testHarness.notifyOfCompletedCheckpoint(0);

	//isCommitted should have failed, thus sendValues() should never have been called
	Assert.assertEquals(0, sink.values.size());

	for (int x = 0; x < 11; x++) {
		testHarness.processElement(new StreamRecord<>(generateValue(elementCounter, 1)));
		elementCounter++;
	}

	testHarness.snapshot(1, 0);
	testHarness.notifyOfCompletedCheckpoint(1);

	//previous CP should be retried, but will fail the CP commit. Second CP should be skipped.
	Assert.assertEquals(10, sink.values.size());

	for (int x = 0; x < 12; x++) {
		testHarness.processElement(new StreamRecord<>(generateValue(elementCounter, 2)));
		elementCounter++;
	}

	testHarness.snapshot(2, 0);
	testHarness.notifyOfCompletedCheckpoint(2);

	//all CP's should be retried and succeed; since one CP was written twice we have 2 * 10 + 11 + 12 = 43 values
	Assert.assertEquals(43, sink.values.size());
}
 

@Test
public void testDataDiscardingUponRestore() throws Exception {
	S sink = createSink();

	OneInputStreamOperatorTestHarness<IN, IN> testHarness =
			new OneInputStreamOperatorTestHarness<>(sink);

	testHarness.open();

	int elementCounter = 1;
	int snapshotCount = 0;

	for (int x = 0; x < 20; x++) {
		testHarness.processElement(new StreamRecord<>(generateValue(elementCounter, 0)));
		elementCounter++;
	}

	OperatorSubtaskState latestSnapshot = testHarness.snapshot(snapshotCount++, 0);
	testHarness.notifyOfCompletedCheckpoint(snapshotCount - 1);

	for (int x = 0; x < 20; x++) {
		testHarness.processElement(new StreamRecord<>(generateValue(elementCounter, 1)));
		elementCounter++;
	}

	testHarness.close();

	sink = createSink();

	testHarness = new OneInputStreamOperatorTestHarness<>(sink);

	testHarness.setup();
	testHarness.initializeState(latestSnapshot);
	testHarness.open();

	for (int x = 0; x < 20; x++) {
		testHarness.processElement(new StreamRecord<>(generateValue(elementCounter, 2)));
		elementCounter++;
	}

	testHarness.snapshot(snapshotCount++, 0);
	testHarness.notifyOfCompletedCheckpoint(snapshotCount - 1);

	verifyResultsDataDiscardingUponRestore(sink);
}
 

@Test
public void testScalingDown() throws Exception {
	S sink1 = createSink();
	OneInputStreamOperatorTestHarness<IN, IN> testHarness1 =
		new OneInputStreamOperatorTestHarness<>(sink1, maxParallelism, 2, 0);
	testHarness1.open();

	S sink2 = createSink();
	OneInputStreamOperatorTestHarness<IN, IN> testHarness2 =
		new OneInputStreamOperatorTestHarness<>(sink2, maxParallelism, 2, 1);
	testHarness2.open();

	int elementCounter = 1;
	int snapshotCount = 0;

	for (int x = 0; x < 10; x++) {
		testHarness1.processElement(new StreamRecord<>(generateValue(elementCounter, 0)));
		elementCounter++;
	}

	for (int x = 0; x < 11; x++) {
		testHarness2.processElement(new StreamRecord<>(generateValue(elementCounter, 0)));
		elementCounter++;
	}

	// snapshot at checkpoint 0 for testHarness1 and testHarness 2
	OperatorSubtaskState snapshot1 = testHarness1.snapshot(snapshotCount, 0);
	OperatorSubtaskState snapshot2 = testHarness2.snapshot(snapshotCount, 0);

	// merge the two partial states
	OperatorSubtaskState mergedSnapshot = AbstractStreamOperatorTestHarness
		.repackageState(snapshot1, snapshot2);

	testHarness1.close();
	testHarness2.close();

	// and create a third instance that operates alone but
	// has the merged state of the previous 2 instances

	OperatorSubtaskState initState = AbstractStreamOperatorTestHarness.repartitionOperatorState(
		mergedSnapshot, maxParallelism, 2, 1, 0);

	S sink3 = createSink();
	OneInputStreamOperatorTestHarness<IN, IN> mergedTestHarness =
		new OneInputStreamOperatorTestHarness<>(sink3, maxParallelism, 1, 0);

	mergedTestHarness.setup();
	mergedTestHarness.initializeState(initState);
	mergedTestHarness.open();

	for (int x = 0; x < 12; x++) {
		mergedTestHarness.processElement(new StreamRecord<>(generateValue(elementCounter, 0)));
		elementCounter++;
	}

	snapshotCount++;
	mergedTestHarness.snapshot(snapshotCount, 1);
	mergedTestHarness.notifyOfCompletedCheckpoint(snapshotCount);

	verifyResultsWhenReScaling(sink3, 1, 33);
	mergedTestHarness.close();
}
 

private void testPartFilesWithIntegerBucketer(
		OneInputStreamOperatorTestHarness<Tuple2<String, Integer>, Object> testHarness,
		File outDir,
		String partFileName1,
		String partFileName2,
		String partFileName3) throws Exception {

	testHarness.setup();
	testHarness.open();

	// this creates a new bucket "test1" and part-0-0
	testHarness.processElement(new StreamRecord<>(Tuple2.of("test1", 1), 1L));
	TestUtils.checkLocalFs(outDir, 1, 0);

	// we take a checkpoint so we roll.
	testHarness.snapshot(1L, 1L);

	// these will close part-0-0 and open part-0-1 and part-0-2
	testHarness.processElement(new StreamRecord<>(Tuple2.of("test1", 2), 2L));
	testHarness.processElement(new StreamRecord<>(Tuple2.of("test1", 3), 3L));

	// we take a checkpoint so we roll again.
	testHarness.snapshot(2L, 2L);

	TestUtils.checkLocalFs(outDir, 3, 0);

	Map<File, String> contents = TestUtils.getFileContentByPath(outDir);
	int fileCounter = 0;
	for (Map.Entry<File, String> fileContents : contents.entrySet()) {
		if (fileContents.getKey().getName().contains(partFileName1)) {
			fileCounter++;
			Assert.assertEquals("test1@1\n", fileContents.getValue());
			Assert.assertEquals("1", fileContents.getKey().getParentFile().getName());
		} else if (fileContents.getKey().getName().contains(partFileName2)) {
			fileCounter++;
			Assert.assertEquals("test1@2\n", fileContents.getValue());
			Assert.assertEquals("2", fileContents.getKey().getParentFile().getName());
		} else if (fileContents.getKey().getName().contains(partFileName3)) {
			fileCounter++;
			Assert.assertEquals("test1@3\n", fileContents.getValue());
			Assert.assertEquals("3", fileContents.getKey().getParentFile().getName());
		}
	}
	Assert.assertEquals(3L, fileCounter);

	// we acknowledge the latest checkpoint, so everything should be published.
	testHarness.notifyOfCompletedCheckpoint(2L);

	TestUtils.checkLocalFs(outDir, 0, 3);
}
 

@Test
public void testDataDiscardingUponRestore() throws Exception {
	S sink = createSink();

	OneInputStreamOperatorTestHarness<IN, IN> testHarness =
			new OneInputStreamOperatorTestHarness<>(sink);

	testHarness.open();

	int elementCounter = 1;
	int snapshotCount = 0;

	for (int x = 0; x < 20; x++) {
		testHarness.processElement(new StreamRecord<>(generateValue(elementCounter, 0)));
		elementCounter++;
	}

	OperatorSubtaskState latestSnapshot = testHarness.snapshot(snapshotCount++, 0);
	testHarness.notifyOfCompletedCheckpoint(snapshotCount - 1);

	for (int x = 0; x < 20; x++) {
		testHarness.processElement(new StreamRecord<>(generateValue(elementCounter, 1)));
		elementCounter++;
	}

	testHarness.close();

	sink = createSink();

	testHarness = new OneInputStreamOperatorTestHarness<>(sink);

	testHarness.setup();
	testHarness.initializeState(latestSnapshot);
	testHarness.open();

	for (int x = 0; x < 20; x++) {
		testHarness.processElement(new StreamRecord<>(generateValue(elementCounter, 2)));
		elementCounter++;
	}

	testHarness.snapshot(snapshotCount++, 0);
	testHarness.notifyOfCompletedCheckpoint(snapshotCount - 1);

	verifyResultsDataDiscardingUponRestore(sink);
}
 

/**
 * This tests checks whether FlinkKafkaProducer correctly aborts lingering transactions after a failure.
 * If such transactions were left alone lingering it consumers would be unable to read committed records
 * that were created after this lingering transaction.
 */
@Test
public void testFailBeforeNotifyAndResumeWorkAfterwards() throws Exception {
	String topic = "flink-kafka-producer-fail-before-notify";

	OneInputStreamOperatorTestHarness<Integer, Object> testHarness1 = createTestHarness(topic);
	checkProducerLeak();
	testHarness1.setup();
	testHarness1.open();
	testHarness1.processElement(42, 0);
	testHarness1.snapshot(0, 1);
	testHarness1.processElement(43, 2);
	OperatorSubtaskState snapshot1 = testHarness1.snapshot(1, 3);

	testHarness1.processElement(44, 4);
	testHarness1.snapshot(2, 5);
	testHarness1.processElement(45, 6);

	// do not close previous testHarness to make sure that closing do not clean up something (in case of failure
	// there might not be any close)
	OneInputStreamOperatorTestHarness<Integer, Object> testHarness2 = createTestHarness(topic);
	testHarness2.setup();
	// restore from snapshot1, transactions with records 44 and 45 should be aborted
	testHarness2.initializeState(snapshot1);
	testHarness2.open();

	// write and commit more records, after potentially lingering transactions
	testHarness2.processElement(46, 7);
	testHarness2.snapshot(4, 8);
	testHarness2.processElement(47, 9);
	testHarness2.notifyOfCompletedCheckpoint(4);

	//now we should have:
	// - records 42 and 43 in committed transactions
	// - aborted transactions with records 44 and 45
	// - committed transaction with record 46
	// - pending transaction with record 47
	assertExactlyOnceForTopic(createProperties(), topic, 0, Arrays.asList(42, 43, 46));

	try {
		testHarness1.close();
	} catch (Exception e) {
		// The only acceptable exception is ProducerFencedException because testHarness2 uses the same
		// transactional ID.
		if (!(e.getCause() instanceof ProducerFencedException)) {
			fail("Received unexpected exception " + e);
		}
	}
	testHarness2.close();
	deleteTestTopic(topic);
	checkProducerLeak();
}
 

/**
 * This tests checks whether FlinkKafkaProducer011 correctly aborts lingering transactions after a failure,
 * which happened before first checkpoint and was followed up by reducing the parallelism.
 * If such transactions were left alone lingering it consumers would be unable to read committed records
 * that were created after this lingering transaction.
 */
@Test
public void testScaleDownBeforeFirstCheckpoint() throws Exception {
	String topic = "scale-down-before-first-checkpoint";

	List<AutoCloseable> operatorsToClose = new ArrayList<>();
	int preScaleDownParallelism = Math.max(2, FlinkKafkaProducer011.SAFE_SCALE_DOWN_FACTOR);
	for (int subtaskIndex = 0; subtaskIndex < preScaleDownParallelism; subtaskIndex++) {
		OneInputStreamOperatorTestHarness<Integer, Object> preScaleDownOperator = createTestHarness(
			topic,
			preScaleDownParallelism,
			preScaleDownParallelism,
			subtaskIndex,
			EXACTLY_ONCE);

		preScaleDownOperator.setup();
		preScaleDownOperator.open();
		preScaleDownOperator.processElement(subtaskIndex * 2, 0);
		preScaleDownOperator.snapshot(0, 1);
		preScaleDownOperator.processElement(subtaskIndex * 2 + 1, 2);

		operatorsToClose.add(preScaleDownOperator);
	}

	// do not close previous testHarnesses to make sure that closing do not clean up something (in case of failure
	// there might not be any close)

	// After previous failure simulate restarting application with smaller parallelism
	OneInputStreamOperatorTestHarness<Integer, Object> postScaleDownOperator1 = createTestHarness(topic, 1, 1, 0, EXACTLY_ONCE);

	postScaleDownOperator1.setup();
	postScaleDownOperator1.open();

	// write and commit more records, after potentially lingering transactions
	postScaleDownOperator1.processElement(46, 7);
	postScaleDownOperator1.snapshot(4, 8);
	postScaleDownOperator1.processElement(47, 9);
	postScaleDownOperator1.notifyOfCompletedCheckpoint(4);

	//now we should have:
	// - records 42, 43, 44 and 45 in aborted transactions
	// - committed transaction with record 46
	// - pending transaction with record 47
	assertExactlyOnceForTopic(createProperties(), topic, 0, Arrays.asList(46));

	postScaleDownOperator1.close();
	// ignore ProducerFencedExceptions, because postScaleDownOperator1 could reuse transactional ids.
	for (AutoCloseable operatorToClose : operatorsToClose) {
		closeIgnoringProducerFenced(operatorToClose);
	}
	deleteTestTopic(topic);
	checkProducerLeak();
}
 

/**
 * This tests checks whether FlinkKafkaProducer correctly aborts lingering transactions after a failure,
 * which happened before first checkpoint and was followed up by reducing the parallelism.
 * If such transactions were left alone lingering it consumers would be unable to read committed records
 * that were created after this lingering transaction.
 */
@Test
public void testScaleDownBeforeFirstCheckpoint() throws Exception {
	String topic = "scale-down-before-first-checkpoint";

	List<AutoCloseable> operatorsToClose = new ArrayList<>();
	int preScaleDownParallelism = Math.max(2, FlinkKafkaProducer.SAFE_SCALE_DOWN_FACTOR);
	for (int subtaskIndex = 0; subtaskIndex < preScaleDownParallelism; subtaskIndex++) {
		OneInputStreamOperatorTestHarness<Integer, Object> preScaleDownOperator = createTestHarness(
			topic,
			preScaleDownParallelism,
			preScaleDownParallelism,
			subtaskIndex,
			FlinkKafkaProducer.Semantic.EXACTLY_ONCE);

		preScaleDownOperator.setup();
		preScaleDownOperator.open();
		preScaleDownOperator.processElement(subtaskIndex * 2, 0);
		preScaleDownOperator.snapshot(0, 1);
		preScaleDownOperator.processElement(subtaskIndex * 2 + 1, 2);

		operatorsToClose.add(preScaleDownOperator);
	}

	// do not close previous testHarnesses to make sure that closing do not clean up something (in case of failure
	// there might not be any close)

	// After previous failure simulate restarting application with smaller parallelism
	OneInputStreamOperatorTestHarness<Integer, Object> postScaleDownOperator1 = createTestHarness(topic, 1, 1, 0, FlinkKafkaProducer.Semantic.EXACTLY_ONCE);

	postScaleDownOperator1.setup();
	postScaleDownOperator1.open();

	// write and commit more records, after potentially lingering transactions
	postScaleDownOperator1.processElement(46, 7);
	postScaleDownOperator1.snapshot(4, 8);
	postScaleDownOperator1.processElement(47, 9);
	postScaleDownOperator1.notifyOfCompletedCheckpoint(4);

	//now we should have:
	// - records 42, 43, 44 and 45 in aborted transactions
	// - committed transaction with record 46
	// - pending transaction with record 47
	assertExactlyOnceForTopic(createProperties(), topic, 0, Arrays.asList(46));

	postScaleDownOperator1.close();
	// ignore ProducerFencedExceptions, because postScaleDownOperator1 could reuse transactional ids.
	for (AutoCloseable operatorToClose : operatorsToClose) {
		closeIgnoringProducerFenced(operatorToClose);
	}
	deleteTestTopic(topic);
	checkProducerLeak();
}
 

@Test
/**
 * Verifies that exceptions thrown by a committer do not fail a job and lead to an abort of notify()
 * and later retry of the affected checkpoints.
 */
public void testCommitterException() throws Exception {

	ListSink2 sink = new ListSink2();

	OneInputStreamOperatorTestHarness<Tuple1<Integer>, Tuple1<Integer>> testHarness =
			new OneInputStreamOperatorTestHarness<>(sink);

	testHarness.open();

	int elementCounter = 1;

	for (int x = 0; x < 10; x++) {
		testHarness.processElement(new StreamRecord<>(generateValue(elementCounter, 0)));
		elementCounter++;
	}

	testHarness.snapshot(0, 0);
	testHarness.notifyOfCompletedCheckpoint(0);

	//isCommitted should have failed, thus sendValues() should never have been called
	Assert.assertEquals(0, sink.values.size());

	for (int x = 0; x < 11; x++) {
		testHarness.processElement(new StreamRecord<>(generateValue(elementCounter, 1)));
		elementCounter++;
	}

	testHarness.snapshot(1, 0);
	testHarness.notifyOfCompletedCheckpoint(1);

	//previous CP should be retried, but will fail the CP commit. Second CP should be skipped.
	Assert.assertEquals(10, sink.values.size());

	for (int x = 0; x < 12; x++) {
		testHarness.processElement(new StreamRecord<>(generateValue(elementCounter, 2)));
		elementCounter++;
	}

	testHarness.snapshot(2, 0);
	testHarness.notifyOfCompletedCheckpoint(2);

	//all CP's should be retried and succeed; since one CP was written twice we have 2 * 10 + 11 + 12 = 43 values
	Assert.assertEquals(43, sink.values.size());
}
 

/**
 * This tests checks whether FlinkKafkaProducer correctly aborts lingering transactions after a failure,
 * which happened before first checkpoint and was followed up by reducing the parallelism.
 * If such transactions were left alone lingering it consumers would be unable to read committed records
 * that were created after this lingering transaction.
 */
@Test
public void testScaleDownBeforeFirstCheckpoint() throws Exception {
	String topic = "scale-down-before-first-checkpoint";

	List<AutoCloseable> operatorsToClose = new ArrayList<>();
	int preScaleDownParallelism = Math.max(2, FlinkKafkaProducer.SAFE_SCALE_DOWN_FACTOR);
	for (int subtaskIndex = 0; subtaskIndex < preScaleDownParallelism; subtaskIndex++) {
		OneInputStreamOperatorTestHarness<Integer, Object> preScaleDownOperator = createTestHarness(
			topic,
			preScaleDownParallelism,
			preScaleDownParallelism,
			subtaskIndex,
			FlinkKafkaProducer.Semantic.EXACTLY_ONCE);

		preScaleDownOperator.setup();
		preScaleDownOperator.open();
		preScaleDownOperator.processElement(subtaskIndex * 2, 0);
		preScaleDownOperator.snapshot(0, 1);
		preScaleDownOperator.processElement(subtaskIndex * 2 + 1, 2);

		operatorsToClose.add(preScaleDownOperator);
	}

	// do not close previous testHarnesses to make sure that closing do not clean up something (in case of failure
	// there might not be any close)

	// After previous failure simulate restarting application with smaller parallelism
	OneInputStreamOperatorTestHarness<Integer, Object> postScaleDownOperator1 = createTestHarness(topic, 1, 1, 0, FlinkKafkaProducer.Semantic.EXACTLY_ONCE);

	postScaleDownOperator1.setup();
	postScaleDownOperator1.open();

	// write and commit more records, after potentially lingering transactions
	postScaleDownOperator1.processElement(46, 7);
	postScaleDownOperator1.snapshot(4, 8);
	postScaleDownOperator1.processElement(47, 9);
	postScaleDownOperator1.notifyOfCompletedCheckpoint(4);

	//now we should have:
	// - records 42, 43, 44 and 45 in aborted transactions
	// - committed transaction with record 46
	// - pending transaction with record 47
	assertExactlyOnceForTopic(createProperties(), topic, 0, Arrays.asList(46));

	postScaleDownOperator1.close();
	// ignore ProducerFencedExceptions, because postScaleDownOperator1 could reuse transactional ids.
	for (AutoCloseable operatorToClose : operatorsToClose) {
		closeIgnoringProducerFenced(operatorToClose);
	}
	deleteTestTopic(topic);
	checkProducerLeak();
}