Java Code Examples for org.apache.flink.streaming.util.OneInputStreamOperatorTestHarness#getCheckpointLock()

/**
 * FLINK-5652
 * Tests that registered timers are properly canceled upon completion of a
 * {@link StreamElement} in order to avoid resource leaks because TriggerTasks hold
 * a reference on the StreamRecordQueueEntry.
 */
@Test
public void testTimeoutCleanup() throws Exception {
	OneInputStreamOperatorTestHarness<Integer, Integer> harness =
		createTestHarness(new MyAsyncFunction(), TIMEOUT, 1, AsyncDataStream.OutputMode.UNORDERED);

	harness.open();

	synchronized (harness.getCheckpointLock()) {
		harness.processElement(42, 1L);
	}

	synchronized (harness.getCheckpointLock()) {
		harness.endInput();
		harness.close();
	}

	// check that we actually outputted the result of the single input
	assertEquals(Arrays.asList(new StreamRecord(42 * 2, 1L)), new ArrayList<>(harness.getOutput()));

	// check that we have cancelled our registered timeout
	assertEquals(0, harness.getProcessingTimeService().getNumActiveTimers());
}
 

@Test
public void testTemporalLeftAsyncJoin() throws Exception {
	OneInputStreamOperatorTestHarness<RowData, RowData> testHarness = createHarness(
		JoinType.LEFT_JOIN,
		FilterOnTable.WITHOUT_FILTER);

	testHarness.open();

	synchronized (testHarness.getCheckpointLock()) {
		testHarness.processElement(insertRecord(1, "a"));
		testHarness.processElement(insertRecord(2, "b"));
		testHarness.processElement(insertRecord(3, "c"));
		testHarness.processElement(insertRecord(4, "d"));
		testHarness.processElement(insertRecord(5, "e"));
	}

	// wait until all async collectors in the buffer have been emitted out.
	synchronized (testHarness.getCheckpointLock()) {
		testHarness.endInput();
		testHarness.close();
	}

	List<Object> expectedOutput = new ArrayList<>();
	expectedOutput.add(insertRecord(1, "a", 1, "Julian"));
	expectedOutput.add(insertRecord(2, "b", null, null));
	expectedOutput.add(insertRecord(3, "c", 3, "Jark"));
	expectedOutput.add(insertRecord(3, "c", 3, "Jackson"));
	expectedOutput.add(insertRecord(4, "d", 4, "Fabian"));
	expectedOutput.add(insertRecord(5, "e", null, null));

	assertor.assertOutputEquals("output wrong.", expectedOutput, testHarness.getOutput());
}
 

private void testUserExceptionHandling(AsyncDataStream.OutputMode outputMode) throws Exception {
	UserExceptionAsyncFunction asyncWaitFunction = new UserExceptionAsyncFunction();
	long timeout = 2000L;

	AsyncWaitOperator<Integer, Integer> asyncWaitOperator = new AsyncWaitOperator<>(
		asyncWaitFunction,
		TIMEOUT,
		2,
		outputMode);

	final MockEnvironment mockEnvironment = createMockEnvironment();
	mockEnvironment.setExpectedExternalFailureCause(Throwable.class);

	OneInputStreamOperatorTestHarness<Integer, Integer> harness = new OneInputStreamOperatorTestHarness<>(
		asyncWaitOperator,
		IntSerializer.INSTANCE,
		mockEnvironment);

	harness.open();

	synchronized (harness.getCheckpointLock()) {
		harness.processElement(1, 1L);
	}

	synchronized (harness.getCheckpointLock()) {
		harness.close();
	}

	assertTrue(harness.getEnvironment().getActualExternalFailureCause().isPresent());
}
 

private void testUserExceptionHandling(AsyncDataStream.OutputMode outputMode) throws Exception {
	UserExceptionAsyncFunction asyncWaitFunction = new UserExceptionAsyncFunction();
	long timeout = 2000L;

	AsyncWaitOperator<Integer, Integer> asyncWaitOperator = new AsyncWaitOperator<>(
		asyncWaitFunction,
		TIMEOUT,
		2,
		outputMode);

	final MockEnvironment mockEnvironment = createMockEnvironment();
	mockEnvironment.setExpectedExternalFailureCause(Throwable.class);

	OneInputStreamOperatorTestHarness<Integer, Integer> harness = new OneInputStreamOperatorTestHarness<>(
		asyncWaitOperator,
		IntSerializer.INSTANCE,
		mockEnvironment);

	harness.open();

	synchronized (harness.getCheckpointLock()) {
		harness.processElement(1, 1L);
	}

	synchronized (harness.getCheckpointLock()) {
		harness.close();
	}

	assertTrue(harness.getEnvironment().getActualExternalFailureCause().isPresent());
}
 

private void testTimeoutExceptionHandling(AsyncDataStream.OutputMode outputMode) throws Exception {
	AsyncFunction<Integer, Integer> asyncFunction = new NoOpAsyncFunction<>();
	long timeout = 10L; // 1 milli second

	AsyncWaitOperator<Integer, Integer> asyncWaitOperator = new AsyncWaitOperator<>(
		asyncFunction,
		timeout,
		2,
		outputMode);

	final MockEnvironment mockEnvironment = createMockEnvironment();
	mockEnvironment.setExpectedExternalFailureCause(Throwable.class);

	OneInputStreamOperatorTestHarness<Integer, Integer> harness = new OneInputStreamOperatorTestHarness<>(
		asyncWaitOperator,
		IntSerializer.INSTANCE,
		mockEnvironment);

	harness.open();

	synchronized (harness.getCheckpointLock()) {
		harness.processElement(1, 1L);
	}

	harness.setProcessingTime(10L);

	synchronized (harness.getCheckpointLock()) {
		harness.close();
	}
}
 

@Test
public void testTemporalLeftAsyncJoin() throws Exception {
	OneInputStreamOperatorTestHarness<BaseRow, BaseRow> testHarness = createHarness(
		JoinType.LEFT_JOIN,
		FilterOnTable.WITHOUT_FILTER);

	testHarness.open();

	synchronized (testHarness.getCheckpointLock()) {
		testHarness.processElement(record(1, "a"));
		testHarness.processElement(record(2, "b"));
		testHarness.processElement(record(3, "c"));
		testHarness.processElement(record(4, "d"));
		testHarness.processElement(record(5, "e"));
	}

	// wait until all async collectors in the buffer have been emitted out.
	synchronized (testHarness.getCheckpointLock()) {
		testHarness.close();
	}

	List<Object> expectedOutput = new ArrayList<>();
	expectedOutput.add(record(1, "a", 1, "Julian"));
	expectedOutput.add(record(2, "b", null, null));
	expectedOutput.add(record(3, "c", 3, "Jark"));
	expectedOutput.add(record(3, "c", 3, "Jackson"));
	expectedOutput.add(record(4, "d", 4, "Fabian"));
	expectedOutput.add(record(5, "e", null, null));

	assertor.assertOutputEquals("output wrong.", expectedOutput, testHarness.getOutput());
}
 

@Test
public void testTemporalInnerAsyncJoinWithFilter() throws Exception {
	OneInputStreamOperatorTestHarness<RowData, RowData> testHarness = createHarness(
		JoinType.INNER_JOIN,
		FilterOnTable.WITH_FILTER);

	testHarness.open();

	synchronized (testHarness.getCheckpointLock()) {
		testHarness.processElement(insertRecord(1, "a"));
		testHarness.processElement(insertRecord(2, "b"));
		testHarness.processElement(insertRecord(3, "c"));
		testHarness.processElement(insertRecord(4, "d"));
		testHarness.processElement(insertRecord(5, "e"));
	}

	// wait until all async collectors in the buffer have been emitted out.
	synchronized (testHarness.getCheckpointLock()) {
		testHarness.endInput();
		testHarness.close();
	}

	List<Object> expectedOutput = new ArrayList<>();
	expectedOutput.add(insertRecord(1, "a", 1, "Julian"));
	expectedOutput.add(insertRecord(3, "c", 3, "Jackson"));
	expectedOutput.add(insertRecord(4, "d", 4, "Fabian"));

	assertor.assertOutputEquals("output wrong.", expectedOutput, testHarness.getOutput());
}
 

@Test
public void testReaderScalingUp() throws Exception {
	// simulates the scenario of scaling up from 1 to 2 instances

	final OneShotLatch waitingLatch1 = new OneShotLatch();
	final OneShotLatch triggerLatch1 = new OneShotLatch();

	BlockingFileInputFormat format1 = new BlockingFileInputFormat(
		triggerLatch1, waitingLatch1, new Path("test"), 20, 5);
	FileInputSplit[] splits = format1.createInputSplits(2);

	OneInputStreamOperatorTestHarness<TimestampedFileInputSplit, String> testHarness1 = getTestHarness(format1, 1, 0);
	testHarness1.open();

	testHarness1.processElement(new StreamRecord<>(getTimestampedSplit(0, splits[0])));
	testHarness1.processElement(new StreamRecord<>(getTimestampedSplit(1, splits[1])));

	// wait until its arrives to element 5
	if (!triggerLatch1.isTriggered()) {
		triggerLatch1.await();
	}

	OperatorSubtaskState snapshot = testHarness1.snapshot(0, 0);

	// this will be the init state for new instance-0
	OperatorSubtaskState initState1 =
		AbstractStreamOperatorTestHarness.repartitionOperatorState(snapshot, maxParallelism, 1, 2, 0);

	// this will be the init state for new instance-1
	OperatorSubtaskState initState2 =
		AbstractStreamOperatorTestHarness.repartitionOperatorState(snapshot, maxParallelism, 1, 2, 1);

	// 1) clear the output of instance so that we can compare it with one created by the new instances, and
	// 2) let the operator process the rest of its state
	testHarness1.getOutput().clear();
	waitingLatch1.trigger();

	// create the second instance and let it process the second split till element 15
	final OneShotLatch triggerLatch2 = new OneShotLatch();
	final OneShotLatch waitingLatch2 = new OneShotLatch();

	BlockingFileInputFormat format2 = new BlockingFileInputFormat(
		triggerLatch2, waitingLatch2, new Path("test"), 20, 15);

	OneInputStreamOperatorTestHarness<TimestampedFileInputSplit, String> testHarness2 = getTestHarness(format2, 2, 0);
	testHarness2.setup();
	testHarness2.initializeState(initState1);
	testHarness2.open();

	BlockingFileInputFormat format3 = new BlockingFileInputFormat(
		triggerLatch2, waitingLatch2, new Path("test"), 20, 15);

	OneInputStreamOperatorTestHarness<TimestampedFileInputSplit, String> testHarness3 = getTestHarness(format3, 2, 1);
	testHarness3.setup();
	testHarness3.initializeState(initState2);
	testHarness3.open();

	triggerLatch2.trigger();
	waitingLatch2.trigger();

	// and wait for the processing to finish
	synchronized (testHarness1.getCheckpointLock()) {
		testHarness1.close();
	}
	synchronized (testHarness2.getCheckpointLock()) {
		testHarness2.close();
	}
	synchronized (testHarness3.getCheckpointLock()) {
		testHarness3.close();
	}

	Queue<Object> expectedResult = new ArrayDeque<>();
	putElementsInQ(expectedResult, testHarness1.getOutput());

	Queue<Object> actualResult = new ArrayDeque<>();
	putElementsInQ(actualResult, testHarness2.getOutput());
	putElementsInQ(actualResult, testHarness3.getOutput());

	Assert.assertEquals(35, actualResult.size());
	Assert.assertArrayEquals(expectedResult.toArray(), actualResult.toArray());
}
 

@Test
public void testSimple() throws Exception {
	@SuppressWarnings("unchecked")
	TestMapBundleFunction func = new TestMapBundleFunction();
	CountBundleTrigger<Tuple2<String, String>> trigger = new CountBundleTrigger<>(3);
	KeySelector<Tuple2<String, String>, String> keySelector =
			(KeySelector<Tuple2<String, String>, String>) value -> value.f0;

	OneInputStreamOperatorTestHarness<Tuple2<String, String>, String> op =
			new OneInputStreamOperatorTestHarness<>(
					new MapBundleOperator<>(func, trigger, keySelector));
	op.open();
	synchronized (op.getCheckpointLock()) {
		StreamRecord<Tuple2<String, String>> input = new StreamRecord<>(null);

		input.replace(new Tuple2<>("k1", "v1"));
		op.processElement(input);

		input.replace(new Tuple2<>("k1", "v2"));
		op.processElement(input);

		assertEquals(0, func.getFinishCount());

		input.replace(new Tuple2<>("k2", "v3"));
		op.processElement(input);

		assertEquals(1, func.getFinishCount());
		assertThat(Arrays.asList("k1=v1,v2", "k2=v3"), is(func.getOutputs()));

		input.replace(new Tuple2<>("k3", "v4"));
		op.processElement(input);

		input.replace(new Tuple2<>("k4", "v5"));
		op.processElement(input);

		assertEquals(1, func.getFinishCount());

		op.close();
		assertEquals(2, func.getFinishCount());
		assertThat(Arrays.asList("k3=v4", "k4=v5"), is(func.getOutputs()));
	}
}
 

@Test
public void testReaderSnapshotRestore() throws Exception {
	String testBasePath = hdfsURI + "/" + UUID.randomUUID() + "/";

	TimestampedFileInputSplit split1 =
		new TimestampedFileInputSplit(0, 3, new Path("test/test1"), 0, 100, null);

	TimestampedFileInputSplit split2 =
		new TimestampedFileInputSplit(10, 2, new Path("test/test2"), 101, 200, null);

	TimestampedFileInputSplit split3 =
		new TimestampedFileInputSplit(10, 1, new Path("test/test2"), 0, 100, null);

	TimestampedFileInputSplit split4 =
		new TimestampedFileInputSplit(11, 0, new Path("test/test3"), 0, 100, null);

	final OneShotLatch latch = new OneShotLatch();

	BlockingFileInputFormat format = new BlockingFileInputFormat(latch, new Path(testBasePath));
	TypeInformation<FileInputSplit> typeInfo = TypeExtractor.getInputFormatTypes(format);

	ContinuousFileReaderOperator<FileInputSplit> initReader = new ContinuousFileReaderOperator<>(format);
	initReader.setOutputType(typeInfo, new ExecutionConfig());

	OneInputStreamOperatorTestHarness<TimestampedFileInputSplit, FileInputSplit> initTestInstance =
		new OneInputStreamOperatorTestHarness<>(initReader);
	initTestInstance.setTimeCharacteristic(TimeCharacteristic.EventTime);
	initTestInstance.open();

	// create some state in the reader
	initTestInstance.processElement(new StreamRecord<>(split1));
	initTestInstance.processElement(new StreamRecord<>(split2));
	initTestInstance.processElement(new StreamRecord<>(split3));
	initTestInstance.processElement(new StreamRecord<>(split4));

	// take a snapshot of the operator's state. This will be used
	// to initialize another reader and compare the results of the
	// two operators.

	final OperatorSubtaskState snapshot;
	synchronized (initTestInstance.getCheckpointLock()) {
		snapshot = initTestInstance.snapshot(0L, 0L);
	}

	ContinuousFileReaderOperator<FileInputSplit> restoredReader = new ContinuousFileReaderOperator<>(
		new BlockingFileInputFormat(latch, new Path(testBasePath)));
	restoredReader.setOutputType(typeInfo, new ExecutionConfig());

	OneInputStreamOperatorTestHarness<TimestampedFileInputSplit, FileInputSplit> restoredTestInstance  =
		new OneInputStreamOperatorTestHarness<>(restoredReader);
	restoredTestInstance.setTimeCharacteristic(TimeCharacteristic.EventTime);

	restoredTestInstance.initializeState(snapshot);
	restoredTestInstance.open();

	// now let computation start
	latch.trigger();

	// ... and wait for the operators to close gracefully

	synchronized (initTestInstance.getCheckpointLock()) {
		initTestInstance.close();
	}

	synchronized (restoredTestInstance.getCheckpointLock()) {
		restoredTestInstance.close();
	}

	FileInputSplit fsSplit1 = createSplitFromTimestampedSplit(split1);
	FileInputSplit fsSplit2 = createSplitFromTimestampedSplit(split2);
	FileInputSplit fsSplit3 = createSplitFromTimestampedSplit(split3);
	FileInputSplit fsSplit4 = createSplitFromTimestampedSplit(split4);

	// compare if the results contain what they should contain and also if
	// they are the same, as they should.

	Assert.assertTrue(initTestInstance.getOutput().contains(new StreamRecord<>(fsSplit1)));
	Assert.assertTrue(initTestInstance.getOutput().contains(new StreamRecord<>(fsSplit2)));
	Assert.assertTrue(initTestInstance.getOutput().contains(new StreamRecord<>(fsSplit3)));
	Assert.assertTrue(initTestInstance.getOutput().contains(new StreamRecord<>(fsSplit4)));

	Assert.assertArrayEquals(
		initTestInstance.getOutput().toArray(),
		restoredTestInstance.getOutput().toArray()
	);
}
 

@Test
public void testReaderRestore() throws Exception {
	File testFolder = tempFolder.newFolder();

	final OneShotLatch latch = new OneShotLatch();

	BlockingFileInputFormat format = new BlockingFileInputFormat(latch, new Path(testFolder.getAbsolutePath()));
	TypeInformation<FileInputSplit> typeInfo = TypeExtractor.getInputFormatTypes(format);

	ContinuousFileReaderOperator<FileInputSplit> initReader = new ContinuousFileReaderOperator<>(format);
	initReader.setOutputType(typeInfo, new ExecutionConfig());

	OneInputStreamOperatorTestHarness<TimestampedFileInputSplit, FileInputSplit> testHarness =
		new OneInputStreamOperatorTestHarness<>(initReader);
	testHarness.setTimeCharacteristic(TimeCharacteristic.EventTime);

	testHarness.setup();

	testHarness.initializeState(
		OperatorSnapshotUtil.getResourceFilename(
			"reader-migration-test-flink" + testMigrateVersion + "-snapshot"));

	testHarness.open();

	latch.trigger();

	// ... and wait for the operators to close gracefully

	synchronized (testHarness.getCheckpointLock()) {
		testHarness.close();
	}

	TimestampedFileInputSplit split1 =
			new TimestampedFileInputSplit(0, 3, new Path("test/test1"), 0, 100, null);

	TimestampedFileInputSplit split2 =
			new TimestampedFileInputSplit(10, 2, new Path("test/test2"), 101, 200, null);

	TimestampedFileInputSplit split3 =
			new TimestampedFileInputSplit(10, 1, new Path("test/test2"), 0, 100, null);

	TimestampedFileInputSplit split4 =
			new TimestampedFileInputSplit(11, 0, new Path("test/test3"), 0, 100, null);

	// compare if the results contain what they should contain and also if
	// they are the same, as they should.

	Assert.assertTrue(testHarness.getOutput().contains(new StreamRecord<>(split1)));
	Assert.assertTrue(testHarness.getOutput().contains(new StreamRecord<>(split2)));
	Assert.assertTrue(testHarness.getOutput().contains(new StreamRecord<>(split3)));
	Assert.assertTrue(testHarness.getOutput().contains(new StreamRecord<>(split4)));
}
 

@Test
public void testSimple() throws Exception {
	@SuppressWarnings("unchecked")
	TestMapBundleFunction func = new TestMapBundleFunction();
	CountBundleTrigger<Tuple2<String, String>> trigger = new CountBundleTrigger<>(3);
	KeySelector<Tuple2<String, String>, String> keySelector =
			(KeySelector<Tuple2<String, String>, String>) value -> value.f0;

	OneInputStreamOperatorTestHarness<Tuple2<String, String>, String> op =
			new OneInputStreamOperatorTestHarness<>(
					new MapBundleOperator<>(func, trigger, keySelector));
	op.open();
	synchronized (op.getCheckpointLock()) {
		StreamRecord<Tuple2<String, String>> input = new StreamRecord<>(null);

		input.replace(new Tuple2<>("k1", "v1"));
		op.processElement(input);

		input.replace(new Tuple2<>("k1", "v2"));
		op.processElement(input);

		assertEquals(0, func.getFinishCount());

		input.replace(new Tuple2<>("k2", "v3"));
		op.processElement(input);

		assertEquals(1, func.getFinishCount());
		assertThat(Arrays.asList("k1=v1,v2", "k2=v3"), is(func.getOutputs()));

		input.replace(new Tuple2<>("k3", "v4"));
		op.processElement(input);

		input.replace(new Tuple2<>("k4", "v5"));
		op.processElement(input);

		assertEquals(1, func.getFinishCount());

		op.close();
		assertEquals(2, func.getFinishCount());
		assertThat(Arrays.asList("k3=v4", "k4=v5"), is(func.getOutputs()));
	}
}
 

private void testAsyncTimeout(LazyAsyncFunction lazyAsyncFunction,
		Optional<Class<? extends Throwable>> expectedException,
		StreamRecord<Integer>... expectedRecords) throws Exception {
	final long timeout = 10L;

	final AsyncWaitOperator<Integer, Integer> operator = new AsyncWaitOperator<>(
		lazyAsyncFunction,
		timeout,
		2,
		AsyncDataStream.OutputMode.ORDERED);

	final MockEnvironment mockEnvironment = createMockEnvironment();
	mockEnvironment.setExpectedExternalFailureCause(Throwable.class);

	final OneInputStreamOperatorTestHarness<Integer, Integer> testHarness =
		new OneInputStreamOperatorTestHarness<>(operator, IntSerializer.INSTANCE, mockEnvironment);

	final long initialTime = 0L;
	final ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>();

	testHarness.open();

	testHarness.setProcessingTime(initialTime);

	synchronized (testHarness.getCheckpointLock()) {
		testHarness.processElement(new StreamRecord<>(1, initialTime));
		testHarness.setProcessingTime(initialTime + 5L);
		testHarness.processElement(new StreamRecord<>(2, initialTime + 5L));
	}

	// trigger the timeout of the first stream record
	testHarness.setProcessingTime(initialTime + timeout + 1L);

	// allow the second async stream record to be processed
	lazyAsyncFunction.countDown();

	// wait until all async collectors in the buffer have been emitted out.
	synchronized (testHarness.getCheckpointLock()) {
		testHarness.close();
	}

	expectedOutput.addAll(Arrays.asList(expectedRecords));

	TestHarnessUtil.assertOutputEquals("Output with watermark was not correct.", expectedOutput, testHarness.getOutput());

	if (expectedException.isPresent()) {
		assertTrue(mockEnvironment.getActualExternalFailureCause().isPresent());
		assertTrue(ExceptionUtils.findThrowable(
			mockEnvironment.getActualExternalFailureCause().get(),
			expectedException.get()).isPresent());
	}
}
 

/**
 * Manually run this to write binary snapshot data. Remove @Ignore to run.
 */
@Ignore
@Test
public void writeReaderSnapshot() throws Exception {

	File testFolder = tempFolder.newFolder();

	TimestampedFileInputSplit split1 =
			new TimestampedFileInputSplit(0, 3, new Path("test/test1"), 0, 100, null);

	TimestampedFileInputSplit split2 =
			new TimestampedFileInputSplit(10, 2, new Path("test/test2"), 101, 200, null);

	TimestampedFileInputSplit split3 =
			new TimestampedFileInputSplit(10, 1, new Path("test/test2"), 0, 100, null);

	TimestampedFileInputSplit split4 =
			new TimestampedFileInputSplit(11, 0, new Path("test/test3"), 0, 100, null);

	// this always blocks to ensure that the reader doesn't to any actual processing so that
	// we keep the state for the four splits
	final OneShotLatch blockingLatch = new OneShotLatch();
	BlockingFileInputFormat format = new BlockingFileInputFormat(blockingLatch, new Path(testFolder.getAbsolutePath()));

	TypeInformation<FileInputSplit> typeInfo = TypeExtractor.getInputFormatTypes(format);
	ContinuousFileReaderOperator<FileInputSplit> initReader = new ContinuousFileReaderOperator<>(
			format);
	initReader.setOutputType(typeInfo, new ExecutionConfig());
	OneInputStreamOperatorTestHarness<TimestampedFileInputSplit, FileInputSplit> testHarness =
			new OneInputStreamOperatorTestHarness<>(initReader);
	testHarness.setTimeCharacteristic(TimeCharacteristic.EventTime);
	testHarness.open();
	// create some state in the reader
	testHarness.processElement(new StreamRecord<>(split1));
	testHarness.processElement(new StreamRecord<>(split2));
	testHarness.processElement(new StreamRecord<>(split3));
	testHarness.processElement(new StreamRecord<>(split4));
	// take a snapshot of the operator's state. This will be used
	// to initialize another reader and compare the results of the
	// two operators.

	final OperatorSubtaskState snapshot;
	synchronized (testHarness.getCheckpointLock()) {
		snapshot = testHarness.snapshot(0L, 0L);
	}

	OperatorSnapshotUtil.writeStateHandle(snapshot, "src/test/resources/reader-migration-test-flink" + flinkGenerateSavepointVersion + "-snapshot");
}
 

private void testProcessingTime(AsyncDataStream.OutputMode mode) throws Exception {
	final OneInputStreamOperatorTestHarness<Integer, Integer> testHarness =
		createTestHarness(new MyAsyncFunction(), TIMEOUT, 6, mode);

	final long initialTime = 0L;
	final Queue<Object> expectedOutput = new ArrayDeque<>();

	testHarness.open();

	synchronized (testHarness.getCheckpointLock()) {
		testHarness.processElement(new StreamRecord<>(1, initialTime + 1));
		testHarness.processElement(new StreamRecord<>(2, initialTime + 2));
		testHarness.processElement(new StreamRecord<>(3, initialTime + 3));
		testHarness.processElement(new StreamRecord<>(4, initialTime + 4));
		testHarness.processElement(new StreamRecord<>(5, initialTime + 5));
		testHarness.processElement(new StreamRecord<>(6, initialTime + 6));
		testHarness.processElement(new StreamRecord<>(7, initialTime + 7));
		testHarness.processElement(new StreamRecord<>(8, initialTime + 8));
	}

	expectedOutput.add(new StreamRecord<>(2, initialTime + 1));
	expectedOutput.add(new StreamRecord<>(4, initialTime + 2));
	expectedOutput.add(new StreamRecord<>(6, initialTime + 3));
	expectedOutput.add(new StreamRecord<>(8, initialTime + 4));
	expectedOutput.add(new StreamRecord<>(10, initialTime + 5));
	expectedOutput.add(new StreamRecord<>(12, initialTime + 6));
	expectedOutput.add(new StreamRecord<>(14, initialTime + 7));
	expectedOutput.add(new StreamRecord<>(16, initialTime + 8));

	synchronized (testHarness.getCheckpointLock()) {
		testHarness.endInput();
		testHarness.close();
	}

	if (mode == AsyncDataStream.OutputMode.ORDERED) {
		TestHarnessUtil.assertOutputEquals("ORDERED Output was not correct.", expectedOutput, testHarness.getOutput());
	}
	else {
		TestHarnessUtil.assertOutputEqualsSorted(
				"UNORDERED Output was not correct.",
				expectedOutput,
				testHarness.getOutput(),
				new StreamRecordComparator());
	}
}
 

/**
 * Tests that the AysncWaitOperator can restart if checkpointed queue was full.
 *
 * <p>See FLINK-7949
 */
@Test(timeout = 10000)
public void testRestartWithFullQueue() throws Exception {
	int capacity = 10;

	// 1. create the snapshot which contains capacity + 1 elements
	final CompletableFuture<Void> trigger = new CompletableFuture<>();
	final ControllableAsyncFunction<Integer> controllableAsyncFunction = new ControllableAsyncFunction<>(trigger);

	final OneInputStreamOperatorTestHarness<Integer, Integer> snapshotHarness = new OneInputStreamOperatorTestHarness<>(
		new AsyncWaitOperator<>(
			controllableAsyncFunction, // the NoOpAsyncFunction is like a blocking function
			1000L,
			capacity,
			AsyncDataStream.OutputMode.ORDERED),
		IntSerializer.INSTANCE);

	snapshotHarness.open();

	final OperatorSubtaskState snapshot;

	final ArrayList<Integer> expectedOutput = new ArrayList<>(capacity + 1);

	try {
		synchronized (snapshotHarness.getCheckpointLock()) {
			for (int i = 0; i < capacity; i++) {
				snapshotHarness.processElement(i, 0L);
				expectedOutput.add(i);
			}
		}

		expectedOutput.add(capacity);

		final OneShotLatch lastElement = new OneShotLatch();

		final CheckedThread lastElementWriter = new CheckedThread() {
			@Override
			public void go() throws Exception {
				synchronized (snapshotHarness.getCheckpointLock()) {
					lastElement.trigger();
					snapshotHarness.processElement(capacity, 0L);
				}
			}
		};

		lastElementWriter.start();

		lastElement.await();

		synchronized (snapshotHarness.getCheckpointLock()) {
			// execute the snapshot within the checkpoint lock, because then it is guaranteed
			// that the lastElementWriter has written the exceeding element
			snapshot = snapshotHarness.snapshot(0L, 0L);
		}

		// trigger the computation to make the close call finish
		trigger.complete(null);
	} finally {
		synchronized (snapshotHarness.getCheckpointLock()) {
			snapshotHarness.close();
		}
	}

	// 2. restore the snapshot and check that we complete
	final OneInputStreamOperatorTestHarness<Integer, Integer> recoverHarness = new OneInputStreamOperatorTestHarness<>(
		new AsyncWaitOperator<>(
			new ControllableAsyncFunction<>(CompletableFuture.completedFuture(null)),
			1000L,
			capacity,
			AsyncDataStream.OutputMode.ORDERED),
		IntSerializer.INSTANCE);

	recoverHarness.initializeState(snapshot);

	synchronized (recoverHarness.getCheckpointLock()) {
		recoverHarness.open();
	}

	synchronized (recoverHarness.getCheckpointLock()) {
		recoverHarness.close();
	}

	final ConcurrentLinkedQueue<Object> output = recoverHarness.getOutput();

	assertThat(output.size(), Matchers.equalTo(capacity + 1));

	final ArrayList<Integer> outputElements = new ArrayList<>(capacity + 1);

	for (int i = 0; i < capacity + 1; i++) {
		StreamRecord<Integer> streamRecord = ((StreamRecord<Integer>) output.poll());
		outputElements.add(streamRecord.getValue());
	}

	assertThat(outputElements, Matchers.equalTo(expectedOutput));
}
 

private void testAsyncTimeout(LazyAsyncFunction lazyAsyncFunction,
		Optional<Class<? extends Throwable>> expectedException,
		StreamRecord<Integer>... expectedRecords) throws Exception {
	final long timeout = 10L;

	final AsyncWaitOperator<Integer, Integer> operator = new AsyncWaitOperator<>(
		lazyAsyncFunction,
		timeout,
		2,
		AsyncDataStream.OutputMode.ORDERED);

	final MockEnvironment mockEnvironment = createMockEnvironment();
	mockEnvironment.setExpectedExternalFailureCause(Throwable.class);

	final OneInputStreamOperatorTestHarness<Integer, Integer> testHarness =
		new OneInputStreamOperatorTestHarness<>(operator, IntSerializer.INSTANCE, mockEnvironment);

	final long initialTime = 0L;
	final ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>();

	testHarness.open();

	testHarness.setProcessingTime(initialTime);

	synchronized (testHarness.getCheckpointLock()) {
		testHarness.processElement(new StreamRecord<>(1, initialTime));
		testHarness.setProcessingTime(initialTime + 5L);
		testHarness.processElement(new StreamRecord<>(2, initialTime + 5L));
	}

	// trigger the timeout of the first stream record
	testHarness.setProcessingTime(initialTime + timeout + 1L);

	// allow the second async stream record to be processed
	lazyAsyncFunction.countDown();

	// wait until all async collectors in the buffer have been emitted out.
	synchronized (testHarness.getCheckpointLock()) {
		testHarness.close();
	}

	expectedOutput.addAll(Arrays.asList(expectedRecords));

	TestHarnessUtil.assertOutputEquals("Output with watermark was not correct.", expectedOutput, testHarness.getOutput());

	if (expectedException.isPresent()) {
		assertTrue(mockEnvironment.getActualExternalFailureCause().isPresent());
		assertTrue(ExceptionUtils.findThrowable(
			mockEnvironment.getActualExternalFailureCause().get(),
			expectedException.get()).isPresent());
	}
}
 

@Test
public void testReaderScalingUp() throws Exception {
	// simulates the scenario of scaling up from 1 to 2 instances

	final OneShotLatch waitingLatch1 = new OneShotLatch();
	final OneShotLatch triggerLatch1 = new OneShotLatch();

	BlockingFileInputFormat format1 = new BlockingFileInputFormat(
		triggerLatch1, waitingLatch1, new Path("test"), 20, 5);
	FileInputSplit[] splits = format1.createInputSplits(2);

	OneInputStreamOperatorTestHarness<TimestampedFileInputSplit, String> testHarness1 = getTestHarness(format1, 1, 0);
	testHarness1.open();

	testHarness1.processElement(new StreamRecord<>(getTimestampedSplit(0, splits[0])));
	testHarness1.processElement(new StreamRecord<>(getTimestampedSplit(1, splits[1])));

	// wait until its arrives to element 5
	if (!triggerLatch1.isTriggered()) {
		triggerLatch1.await();
	}

	OperatorSubtaskState snapshot = testHarness1.snapshot(0, 0);

	// this will be the init state for new instance-0
	OperatorSubtaskState initState1 =
		AbstractStreamOperatorTestHarness.repartitionOperatorState(snapshot, maxParallelism, 1, 2, 0);

	// this will be the init state for new instance-1
	OperatorSubtaskState initState2 =
		AbstractStreamOperatorTestHarness.repartitionOperatorState(snapshot, maxParallelism, 1, 2, 1);

	// 1) clear the output of instance so that we can compare it with one created by the new instances, and
	// 2) let the operator process the rest of its state
	testHarness1.getOutput().clear();
	waitingLatch1.trigger();

	// create the second instance and let it process the second split till element 15
	final OneShotLatch triggerLatch2 = new OneShotLatch();
	final OneShotLatch waitingLatch2 = new OneShotLatch();

	BlockingFileInputFormat format2 = new BlockingFileInputFormat(
		triggerLatch2, waitingLatch2, new Path("test"), 20, 15);

	OneInputStreamOperatorTestHarness<TimestampedFileInputSplit, String> testHarness2 = getTestHarness(format2, 2, 0);
	testHarness2.setup();
	testHarness2.initializeState(initState1);
	testHarness2.open();

	BlockingFileInputFormat format3 = new BlockingFileInputFormat(
		triggerLatch2, waitingLatch2, new Path("test"), 20, 15);

	OneInputStreamOperatorTestHarness<TimestampedFileInputSplit, String> testHarness3 = getTestHarness(format3, 2, 1);
	testHarness3.setup();
	testHarness3.initializeState(initState2);
	testHarness3.open();

	triggerLatch2.trigger();
	waitingLatch2.trigger();

	// and wait for the processing to finish
	synchronized (testHarness1.getCheckpointLock()) {
		testHarness1.close();
	}
	synchronized (testHarness2.getCheckpointLock()) {
		testHarness2.close();
	}
	synchronized (testHarness3.getCheckpointLock()) {
		testHarness3.close();
	}

	Queue<Object> expectedResult = new ArrayDeque<>();
	putElementsInQ(expectedResult, testHarness1.getOutput());

	Queue<Object> actualResult = new ArrayDeque<>();
	putElementsInQ(actualResult, testHarness2.getOutput());
	putElementsInQ(actualResult, testHarness3.getOutput());

	Assert.assertEquals(35, actualResult.size());
	Assert.assertArrayEquals(expectedResult.toArray(), actualResult.toArray());
}
 

@Test
public void testReaderSnapshotRestore() throws Exception {
	String testBasePath = hdfsURI + "/" + UUID.randomUUID() + "/";

	TimestampedFileInputSplit split1 =
		new TimestampedFileInputSplit(0, 3, new Path("test/test1"), 0, 100, null);

	TimestampedFileInputSplit split2 =
		new TimestampedFileInputSplit(10, 2, new Path("test/test2"), 101, 200, null);

	TimestampedFileInputSplit split3 =
		new TimestampedFileInputSplit(10, 1, new Path("test/test2"), 0, 100, null);

	TimestampedFileInputSplit split4 =
		new TimestampedFileInputSplit(11, 0, new Path("test/test3"), 0, 100, null);

	final OneShotLatch latch = new OneShotLatch();

	BlockingFileInputFormat format = new BlockingFileInputFormat(latch, new Path(testBasePath));
	TypeInformation<FileInputSplit> typeInfo = TypeExtractor.getInputFormatTypes(format);

	ContinuousFileReaderOperator<FileInputSplit> initReader = new ContinuousFileReaderOperator<>(format);
	initReader.setOutputType(typeInfo, new ExecutionConfig());

	OneInputStreamOperatorTestHarness<TimestampedFileInputSplit, FileInputSplit> initTestInstance =
		new OneInputStreamOperatorTestHarness<>(initReader);
	initTestInstance.setTimeCharacteristic(TimeCharacteristic.EventTime);
	initTestInstance.open();

	// create some state in the reader
	initTestInstance.processElement(new StreamRecord<>(split1));
	initTestInstance.processElement(new StreamRecord<>(split2));
	initTestInstance.processElement(new StreamRecord<>(split3));
	initTestInstance.processElement(new StreamRecord<>(split4));

	// take a snapshot of the operator's state. This will be used
	// to initialize another reader and compare the results of the
	// two operators.

	final OperatorSubtaskState snapshot;
	synchronized (initTestInstance.getCheckpointLock()) {
		snapshot = initTestInstance.snapshot(0L, 0L);
	}

	ContinuousFileReaderOperator<FileInputSplit> restoredReader = new ContinuousFileReaderOperator<>(
		new BlockingFileInputFormat(latch, new Path(testBasePath)));
	restoredReader.setOutputType(typeInfo, new ExecutionConfig());

	OneInputStreamOperatorTestHarness<TimestampedFileInputSplit, FileInputSplit> restoredTestInstance  =
		new OneInputStreamOperatorTestHarness<>(restoredReader);
	restoredTestInstance.setTimeCharacteristic(TimeCharacteristic.EventTime);

	restoredTestInstance.initializeState(snapshot);
	restoredTestInstance.open();

	// now let computation start
	latch.trigger();

	// ... and wait for the operators to close gracefully

	synchronized (initTestInstance.getCheckpointLock()) {
		initTestInstance.close();
	}

	synchronized (restoredTestInstance.getCheckpointLock()) {
		restoredTestInstance.close();
	}

	FileInputSplit fsSplit1 = createSplitFromTimestampedSplit(split1);
	FileInputSplit fsSplit2 = createSplitFromTimestampedSplit(split2);
	FileInputSplit fsSplit3 = createSplitFromTimestampedSplit(split3);
	FileInputSplit fsSplit4 = createSplitFromTimestampedSplit(split4);

	// compare if the results contain what they should contain and also if
	// they are the same, as they should.

	Assert.assertTrue(initTestInstance.getOutput().contains(new StreamRecord<>(fsSplit1)));
	Assert.assertTrue(initTestInstance.getOutput().contains(new StreamRecord<>(fsSplit2)));
	Assert.assertTrue(initTestInstance.getOutput().contains(new StreamRecord<>(fsSplit3)));
	Assert.assertTrue(initTestInstance.getOutput().contains(new StreamRecord<>(fsSplit4)));

	Assert.assertArrayEquals(
		initTestInstance.getOutput().toArray(),
		restoredTestInstance.getOutput().toArray()
	);
}
 

/**
 * Tests that the AysncWaitOperator can restart if checkpointed queue was full.
 *
 * <p>See FLINK-7949
 */
@Test(timeout = 10000)
public void testRestartWithFullQueue() throws Exception {
	final int capacity = 10;

	// 1. create the snapshot which contains capacity + 1 elements
	final CompletableFuture<Void> trigger = new CompletableFuture<>();

	final OneInputStreamOperatorTestHarness<Integer, Integer> snapshotHarness = createTestHarness(
		new ControllableAsyncFunction<>(trigger), // the NoOpAsyncFunction is like a blocking function
		1000L,
		capacity,
		AsyncDataStream.OutputMode.ORDERED);

	snapshotHarness.open();

	final OperatorSubtaskState snapshot;

	final ArrayList<Integer> expectedOutput = new ArrayList<>(capacity);

	try {
		synchronized (snapshotHarness.getCheckpointLock()) {
			for (int i = 0; i < capacity; i++) {
				snapshotHarness.processElement(i, 0L);
				expectedOutput.add(i);
			}
		}

		synchronized (snapshotHarness.getCheckpointLock()) {
			// execute the snapshot within the checkpoint lock, because then it is guaranteed
			// that the lastElementWriter has written the exceeding element
			snapshot = snapshotHarness.snapshot(0L, 0L);
		}

		// trigger the computation to make the close call finish
		trigger.complete(null);
	} finally {
		synchronized (snapshotHarness.getCheckpointLock()) {
			snapshotHarness.close();
		}
	}

	// 2. restore the snapshot and check that we complete
	final OneInputStreamOperatorTestHarness<Integer, Integer> recoverHarness = createTestHarness(
		new ControllableAsyncFunction<>(CompletableFuture.completedFuture(null)),
		1000L,
		capacity,
		AsyncDataStream.OutputMode.ORDERED);

	recoverHarness.initializeState(snapshot);

	synchronized (recoverHarness.getCheckpointLock()) {
		recoverHarness.open();
	}

	synchronized (recoverHarness.getCheckpointLock()) {
		recoverHarness.endInput();
		recoverHarness.close();
	}

	final ConcurrentLinkedQueue<Object> output = recoverHarness.getOutput();

	final List<Integer> outputElements = output.stream()
			.map(r -> ((StreamRecord<Integer>) r).getValue())
			.collect(Collectors.toList());

	assertThat(outputElements, Matchers.equalTo(expectedOutput));
}