org.apache.flink.streaming.util.TestHarnessUtil Java Examples

@Test
public void testMultiStateSupport() throws Exception {
	try (
			TwoInputStreamOperatorTestHarness<String, Integer, String> testHarness =
					getInitializedTestHarness(new FunctionWithMultipleStates(), STATE_DESCRIPTOR, STATE_DESCRIPTOR_A)
	) {
		testHarness.processElement2(new StreamRecord<>(5, 12L));
		testHarness.processElement2(new StreamRecord<>(6, 13L));

		testHarness.processElement1(new StreamRecord<>("9", 15L));

		Queue<Object> expectedBr = new ConcurrentLinkedQueue<>();
		expectedBr.add(new StreamRecord<>("9:key.6->6", 15L));
		expectedBr.add(new StreamRecord<>("9:key.5->5", 15L));
		expectedBr.add(new StreamRecord<>("9:5->value.5", 15L));
		expectedBr.add(new StreamRecord<>("9:6->value.6", 15L));

		TestHarnessUtil.assertOutputEquals("Wrong Side Output", expectedBr, testHarness.getOutput());
	}
}
 

@Test
public void testMultiStateSupport() throws Exception {
	try (
			TwoInputStreamOperatorTestHarness<String, Integer, String> testHarness =
					getInitializedTestHarness(new FunctionWithMultipleStates(), STATE_DESCRIPTOR, STATE_DESCRIPTOR_A)
	) {
		testHarness.processElement2(new StreamRecord<>(5, 12L));
		testHarness.processElement2(new StreamRecord<>(6, 13L));

		testHarness.processElement1(new StreamRecord<>("9", 15L));

		Queue<Object> expectedBr = new ConcurrentLinkedQueue<>();
		expectedBr.add(new StreamRecord<>("9:key.6->6", 15L));
		expectedBr.add(new StreamRecord<>("9:key.5->5", 15L));
		expectedBr.add(new StreamRecord<>("9:5->value.5", 15L));
		expectedBr.add(new StreamRecord<>("9:6->value.6", 15L));

		TestHarnessUtil.assertOutputEquals("Wrong Side Output", expectedBr, testHarness.getOutput());
	}
}
 

@Test
public void testAfterWatermarkProgram() throws Exception {
	WindowingStrategy strategy = fixedWindowWithAfterWatermarkTriggerStrategy;
	long initialTime = 0L;
	OneInputStreamOperatorTestHarness<WindowedValue<KV<String, Integer>>, WindowedValue<KV<String, Integer>>> testHarness =
			createTestingOperatorAndState(strategy, initialTime);
	ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>();

	expectedOutput.add(new StreamRecord<>(makeWindowedValue(strategy, KV.of("key1", 6),
			new Instant(initialTime + 1), null, PaneInfo.createPane(true, true, PaneInfo.Timing.ON_TIME)), initialTime + 1));
	expectedOutput.add(new Watermark(initialTime + 10000));

	expectedOutput.add(new StreamRecord<>(makeWindowedValue(strategy, KV.of("key1", 11),
			new Instant(initialTime + 10000), null, PaneInfo.createPane(true, true, PaneInfo.Timing.ON_TIME)), initialTime + 10000));
	expectedOutput.add(new StreamRecord<>(makeWindowedValue(strategy, KV.of("key2", 1),
			new Instant(initialTime + 19500), null, PaneInfo.createPane(true, true, PaneInfo.Timing.ON_TIME)), initialTime + 19500));
	expectedOutput.add(new Watermark(initialTime + 20000));

	TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expectedOutput, testHarness.getOutput(), new ResultSortComparator());
	testHarness.close();
}
 

@Test
public void testMap() throws Exception {
	StreamMap<Integer, String> operator = new StreamMap<Integer, String>(new Map());

	OneInputStreamOperatorTestHarness<Integer, String> testHarness = new OneInputStreamOperatorTestHarness<Integer, String>(operator);

	long initialTime = 0L;
	ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<Object>();

	testHarness.open();

	testHarness.processElement(new StreamRecord<Integer>(1, initialTime + 1));
	testHarness.processElement(new StreamRecord<Integer>(2, initialTime + 2));
	testHarness.processWatermark(new Watermark(initialTime + 2));
	testHarness.processElement(new StreamRecord<Integer>(3, initialTime + 3));

	expectedOutput.add(new StreamRecord<String>("+2", initialTime + 1));
	expectedOutput.add(new StreamRecord<String>("+3", initialTime + 2));
	expectedOutput.add(new Watermark(initialTime + 2));
	expectedOutput.add(new StreamRecord<String>("+4", initialTime + 3));

	TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput());
}
 

/**
 * This test verifies that open() and close() are correctly called by the StreamTask.
 */
@Test
@SuppressWarnings("unchecked")
public void testOpenClose() throws Exception {
	final StreamTaskTestHarness<String> testHarness = new StreamTaskTestHarness<>(
			SourceStreamTask::new, BasicTypeInfo.STRING_TYPE_INFO);

	testHarness.setupOutputForSingletonOperatorChain();

	StreamConfig streamConfig = testHarness.getStreamConfig();
	StreamSource<String, ?> sourceOperator = new StreamSource<>(new OpenCloseTestSource());
	streamConfig.setStreamOperator(sourceOperator);
	streamConfig.setOperatorID(new OperatorID());

	testHarness.invoke();
	testHarness.waitForTaskCompletion();

	Assert.assertTrue("RichFunction methods where not called.", OpenCloseTestSource.closeCalled);

	List<String> resultElements = TestHarnessUtil.getRawElementsFromOutput(testHarness.getOutput());
	Assert.assertEquals(10, resultElements.size());
}
 

@Test
public void testFullJoin() throws Exception {
	StreamOperator joinOperator = newOperator(FlinkJoinType.FULL, leftIsSmall);
	TwoInputStreamTaskTestHarness<BinaryRowData, BinaryRowData, JoinedRowData> testHarness =
			buildSortMergeJoin(joinOperator);

	ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>();
	expectedOutput.add(new StreamRecord<>(newRow("a", "02")));
	expectedOutput.add(new StreamRecord<>(newRow("b", "14")));
	expectedOutput.add(new StreamRecord<>(newRow("c", "2null")));
	expectedOutput.add(new StreamRecord<>(newRow("d", "0null")));

	testHarness.waitForTaskCompletion();
	TestHarnessUtil.assertOutputEquals("Output was not correct.",
			expectedOutput,
			transformToBinary(testHarness.getOutput()));
}
 

@Test
public void testTimestampAndWatermarkQuerying() throws Exception {

	LegacyKeyedProcessOperator<Integer, Integer, String> operator =
			new LegacyKeyedProcessOperator<>(new QueryingFlatMapFunction(TimeDomain.EVENT_TIME));

	OneInputStreamOperatorTestHarness<Integer, String> testHarness =
			new KeyedOneInputStreamOperatorTestHarness<>(operator, new IdentityKeySelector<Integer>(), BasicTypeInfo.INT_TYPE_INFO);

	testHarness.setup();
	testHarness.open();

	testHarness.processWatermark(new Watermark(17));
	testHarness.processElement(new StreamRecord<>(5, 12L));

	testHarness.processWatermark(new Watermark(42));
	testHarness.processElement(new StreamRecord<>(6, 13L));

	ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>();

	expectedOutput.add(new Watermark(17L));
	expectedOutput.add(new StreamRecord<>("5TIME:17 TS:12", 12L));
	expectedOutput.add(new Watermark(42L));
	expectedOutput.add(new StreamRecord<>("6TIME:42 TS:13", 13L));

	TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput());

	testHarness.close();
}
 

/**
 * This also verifies that the timestamps ouf side-emitted records is correct.
 */
@Test
public void testSideOutput() throws Exception {
	KeyedProcessOperator<Integer, Integer, String> operator = new KeyedProcessOperator<>(new SideOutputProcessFunction());

	OneInputStreamOperatorTestHarness<Integer, String> testHarness =
		new KeyedOneInputStreamOperatorTestHarness<>(
			operator, new IdentityKeySelector<>(), BasicTypeInfo.INT_TYPE_INFO);

	testHarness.setup();
	testHarness.open();

	testHarness.processElement(new StreamRecord<>(42, 17L /* timestamp */));

	ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>();

	expectedOutput.add(new StreamRecord<>("IN:42", 17L /* timestamp */));

	TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput());

	ConcurrentLinkedQueue<StreamRecord<Integer>> expectedIntSideOutput = new ConcurrentLinkedQueue<>();
	expectedIntSideOutput.add(new StreamRecord<>(42, 17L /* timestamp */));
	ConcurrentLinkedQueue<StreamRecord<Integer>> intSideOutput =
		testHarness.getSideOutput(SideOutputProcessFunction.INTEGER_OUTPUT_TAG);
	TestHarnessUtil.assertOutputEquals(
		"Side output was not correct.",
		expectedIntSideOutput,
		intSideOutput);

	ConcurrentLinkedQueue<StreamRecord<Long>> expectedLongSideOutput = new ConcurrentLinkedQueue<>();
	expectedLongSideOutput.add(new StreamRecord<>(42L, 17L /* timestamp */));
	ConcurrentLinkedQueue<StreamRecord<Long>> longSideOutput =
		testHarness.getSideOutput(SideOutputProcessFunction.LONG_OUTPUT_TAG);
	TestHarnessUtil.assertOutputEquals(
		"Side output was not correct.",
		expectedLongSideOutput,
		longSideOutput);

	testHarness.close();
}
 

@Test
public void testEventTimeTimers() throws Exception {

	KeyedCoProcessOperator<String, Integer, String, String> operator =
			new KeyedCoProcessOperator<>(new EventTimeTriggeringProcessFunction());

	TwoInputStreamOperatorTestHarness<Integer, String, String> testHarness =
			new KeyedTwoInputStreamOperatorTestHarness<>(
					operator,
					new IntToStringKeySelector<>(),
					new IdentityKeySelector<String>(),
					BasicTypeInfo.STRING_TYPE_INFO);

	testHarness.setup();
	testHarness.open();

	testHarness.processElement1(new StreamRecord<>(17, 42L));
	testHarness.processElement2(new StreamRecord<>("18", 42L));

	testHarness.processWatermark1(new Watermark(5));
	testHarness.processWatermark2(new Watermark(5));

	testHarness.processWatermark1(new Watermark(6));
	testHarness.processWatermark2(new Watermark(6));

	ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>();

	expectedOutput.add(new StreamRecord<>("INPUT1:17", 42L));
	expectedOutput.add(new StreamRecord<>("INPUT2:18", 42L));
	expectedOutput.add(new StreamRecord<>("17:1777", 5L));
	expectedOutput.add(new Watermark(5L));
	expectedOutput.add(new StreamRecord<>("18:1777", 6L));
	expectedOutput.add(new Watermark(6L));

	TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput());

	testHarness.close();
}
 

@Test
public void testProcessingTimeTimers() throws Exception {

	KeyedCoProcessOperator<String, Integer, String, String> operator =
			new KeyedCoProcessOperator<>(new ProcessingTimeTriggeringProcessFunction());

	TwoInputStreamOperatorTestHarness<Integer, String, String> testHarness =
			new KeyedTwoInputStreamOperatorTestHarness<>(
					operator,
					new IntToStringKeySelector<>(),
					new IdentityKeySelector<String>(),
					BasicTypeInfo.STRING_TYPE_INFO);

	testHarness.setup();
	testHarness.open();

	testHarness.processElement1(new StreamRecord<>(17));
	testHarness.processElement2(new StreamRecord<>("18"));

	testHarness.setProcessingTime(5);
	testHarness.setProcessingTime(6);

	ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>();

	expectedOutput.add(new StreamRecord<>("INPUT1:17"));
	expectedOutput.add(new StreamRecord<>("INPUT2:18"));
	expectedOutput.add(new StreamRecord<>("1777"));
	expectedOutput.add(new StreamRecord<>("1777"));

	TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput());

	testHarness.close();
}
 

/**
 * This also verifies that the timestamps ouf side-emitted records is correct.
 */
@Test
public void testSideOutput() throws Exception {
	ProcessOperator<Integer, String> operator =
		new ProcessOperator<>(new SideOutputProcessFunction());

	OneInputStreamOperatorTestHarness<Integer, String> testHarness =
		new OneInputStreamOperatorTestHarness<>(operator);

	testHarness.setup();
	testHarness.open();

	testHarness.processElement(new StreamRecord<>(42, 17L /* timestamp */));

	ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>();

	expectedOutput.add(new StreamRecord<>("IN:42", 17L /* timestamp */));

	TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput());

	ConcurrentLinkedQueue<StreamRecord<Integer>> expectedIntSideOutput = new ConcurrentLinkedQueue<>();
	expectedIntSideOutput.add(new StreamRecord<>(42, 17L /* timestamp */));
	ConcurrentLinkedQueue<StreamRecord<Integer>> intSideOutput =
		testHarness.getSideOutput(SideOutputProcessFunction.INTEGER_OUTPUT_TAG);
	TestHarnessUtil.assertOutputEquals(
		"Side output was not correct.",
		expectedIntSideOutput,
		intSideOutput);

	ConcurrentLinkedQueue<StreamRecord<Long>> expectedLongSideOutput = new ConcurrentLinkedQueue<>();
	expectedLongSideOutput.add(new StreamRecord<>(42L, 17L /* timestamp */));
	ConcurrentLinkedQueue<StreamRecord<Long>> longSideOutput =
		testHarness.getSideOutput(SideOutputProcessFunction.LONG_OUTPUT_TAG);
	TestHarnessUtil.assertOutputEquals(
		"Side output was not correct.",
		expectedLongSideOutput,
		longSideOutput);

	testHarness.close();
}
 

@Test
public void testTimestampAndWatermarkQuerying() throws Exception {

	KeyedProcessOperator<Integer, Integer, String> operator =
			new KeyedProcessOperator<>(new QueryingFlatMapFunction(TimeDomain.EVENT_TIME));

	OneInputStreamOperatorTestHarness<Integer, String> testHarness =
			new KeyedOneInputStreamOperatorTestHarness<>(operator, new IdentityKeySelector<Integer>(), BasicTypeInfo.INT_TYPE_INFO);

	testHarness.setup();
	testHarness.open();

	testHarness.processWatermark(new Watermark(17));
	testHarness.processElement(new StreamRecord<>(5, 12L));

	testHarness.processWatermark(new Watermark(42));
	testHarness.processElement(new StreamRecord<>(6, 13L));

	ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>();

	expectedOutput.add(new Watermark(17L));
	expectedOutput.add(new StreamRecord<>("5TIME:17 TS:12", 12L));
	expectedOutput.add(new Watermark(42L));
	expectedOutput.add(new StreamRecord<>("6TIME:42 TS:13", 13L));

	TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput());

	testHarness.close();
}
 

@Test
public void testTimestampAndProcessingTimeQuerying() throws Exception {

	KeyedCoProcessOperator<String, Integer, String, String> operator =
			new KeyedCoProcessOperator<>(new ProcessingTimeQueryingProcessFunction());

	TwoInputStreamOperatorTestHarness<Integer, String, String> testHarness =
			new KeyedTwoInputStreamOperatorTestHarness<>(
					operator,
					new IntToStringKeySelector<>(),
					new IdentityKeySelector<String>(),
					BasicTypeInfo.STRING_TYPE_INFO);

	testHarness.setup();
	testHarness.open();

	testHarness.setProcessingTime(17);
	testHarness.processElement1(new StreamRecord<>(5));

	testHarness.setProcessingTime(42);
	testHarness.processElement2(new StreamRecord<>("6"));

	ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>();

	expectedOutput.add(new StreamRecord<>("5PT:17 TS:null"));
	expectedOutput.add(new StreamRecord<>("6PT:42 TS:null"));

	TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput());

	testHarness.close();
}
 

/**
 * This also verifies that the timestamps ouf side-emitted records is correct.
 */
@Test
public void testSideOutput() throws Exception {
	ProcessOperator<Integer, String> operator =
		new ProcessOperator<>(new SideOutputProcessFunction());

	OneInputStreamOperatorTestHarness<Integer, String> testHarness =
		new OneInputStreamOperatorTestHarness<>(operator);

	testHarness.setup();
	testHarness.open();

	testHarness.processElement(new StreamRecord<>(42, 17L /* timestamp */));

	ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>();

	expectedOutput.add(new StreamRecord<>("IN:42", 17L /* timestamp */));

	TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput());

	ConcurrentLinkedQueue<StreamRecord<Integer>> expectedIntSideOutput = new ConcurrentLinkedQueue<>();
	expectedIntSideOutput.add(new StreamRecord<>(42, 17L /* timestamp */));
	ConcurrentLinkedQueue<StreamRecord<Integer>> intSideOutput =
		testHarness.getSideOutput(SideOutputProcessFunction.INTEGER_OUTPUT_TAG);
	TestHarnessUtil.assertOutputEquals(
		"Side output was not correct.",
		expectedIntSideOutput,
		intSideOutput);

	ConcurrentLinkedQueue<StreamRecord<Long>> expectedLongSideOutput = new ConcurrentLinkedQueue<>();
	expectedLongSideOutput.add(new StreamRecord<>(42L, 17L /* timestamp */));
	ConcurrentLinkedQueue<StreamRecord<Long>> longSideOutput =
		testHarness.getSideOutput(SideOutputProcessFunction.LONG_OUTPUT_TAG);
	TestHarnessUtil.assertOutputEquals(
		"Side output was not correct.",
		expectedLongSideOutput,
		longSideOutput);

	testHarness.close();
}
 

@Test
public void testTimestampAndWatermarkQuerying() throws Exception {

	CoProcessOperator<Integer, String, String> operator =
			new CoProcessOperator<>(new WatermarkQueryingProcessFunction());

	TwoInputStreamOperatorTestHarness<Integer, String, String> testHarness =
			new TwoInputStreamOperatorTestHarness<>(operator);

	testHarness.setup();
	testHarness.open();

	testHarness.processWatermark1(new Watermark(17));
	testHarness.processWatermark2(new Watermark(17));
	testHarness.processElement1(new StreamRecord<>(5, 12L));

	testHarness.processWatermark1(new Watermark(42));
	testHarness.processWatermark2(new Watermark(42));
	testHarness.processElement2(new StreamRecord<>("6", 13L));

	ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>();

	expectedOutput.add(new Watermark(17L));
	expectedOutput.add(new StreamRecord<>("5WM:17 TS:12", 12L));
	expectedOutput.add(new Watermark(42L));
	expectedOutput.add(new StreamRecord<>("6WM:42 TS:13", 13L));

	TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput());

	testHarness.close();
}
 

@Test
public void testFlatMap() throws Exception {
	StreamFlatMap<Integer, Integer> operator = new StreamFlatMap<Integer, Integer>(new MyFlatMap());

	OneInputStreamOperatorTestHarness<Integer, Integer> testHarness = new OneInputStreamOperatorTestHarness<Integer, Integer>(operator);

	long initialTime = 0L;
	ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<Object>();

	testHarness.open();

	testHarness.processElement(new StreamRecord<Integer>(1, initialTime + 1));
	testHarness.processElement(new StreamRecord<Integer>(2, initialTime + 2));
	testHarness.processWatermark(new Watermark(initialTime + 2));
	testHarness.processElement(new StreamRecord<Integer>(3, initialTime + 3));
	testHarness.processElement(new StreamRecord<Integer>(4, initialTime + 4));
	testHarness.processElement(new StreamRecord<Integer>(5, initialTime + 5));
	testHarness.processElement(new StreamRecord<Integer>(6, initialTime + 6));
	testHarness.processElement(new StreamRecord<Integer>(7, initialTime + 7));
	testHarness.processElement(new StreamRecord<Integer>(8, initialTime + 8));

	expectedOutput.add(new StreamRecord<Integer>(2, initialTime + 2));
	expectedOutput.add(new StreamRecord<Integer>(4, initialTime + 2));
	expectedOutput.add(new Watermark(initialTime + 2));
	expectedOutput.add(new StreamRecord<Integer>(4, initialTime + 4));
	expectedOutput.add(new StreamRecord<Integer>(16, initialTime + 4));
	expectedOutput.add(new StreamRecord<Integer>(6, initialTime + 6));
	expectedOutput.add(new StreamRecord<Integer>(36, initialTime + 6));
	expectedOutput.add(new StreamRecord<Integer>(8, initialTime + 8));
	expectedOutput.add(new StreamRecord<Integer>(64, initialTime + 8));

	TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput());
}
 

@Test
public void testTimestampAndWatermarkQuerying() throws Exception {

	LegacyKeyedCoProcessOperator<String, Integer, String, String> operator =
			new LegacyKeyedCoProcessOperator<>(new WatermarkQueryingProcessFunction());

	TwoInputStreamOperatorTestHarness<Integer, String, String> testHarness =
			new KeyedTwoInputStreamOperatorTestHarness<>(
					operator,
					new IntToStringKeySelector<>(),
					new IdentityKeySelector<String>(),
					BasicTypeInfo.STRING_TYPE_INFO);

	testHarness.setup();
	testHarness.open();

	testHarness.processWatermark1(new Watermark(17));
	testHarness.processWatermark2(new Watermark(17));
	testHarness.processElement1(new StreamRecord<>(5, 12L));

	testHarness.processWatermark1(new Watermark(42));
	testHarness.processWatermark2(new Watermark(42));
	testHarness.processElement2(new StreamRecord<>("6", 13L));

	ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>();

	expectedOutput.add(new Watermark(17L));
	expectedOutput.add(new StreamRecord<>("5WM:17 TS:12", 12L));
	expectedOutput.add(new Watermark(42L));
	expectedOutput.add(new StreamRecord<>("6WM:42 TS:13", 13L));

	TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput());

	testHarness.close();
}
 

@Test
@SuppressWarnings("unchecked")
public void testFilter() throws Exception {
	StreamFilter<Integer> operator = new StreamFilter<Integer>(new MyFilter());

	OneInputStreamOperatorTestHarness<Integer, Integer> testHarness = new OneInputStreamOperatorTestHarness<Integer, Integer>(operator);

	long initialTime = 0L;
	ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<Object>();

	testHarness.open();

	testHarness.processElement(new StreamRecord<Integer>(1, initialTime + 1));
	testHarness.processElement(new StreamRecord<Integer>(2, initialTime + 2));
	testHarness.processWatermark(new Watermark(initialTime + 2));
	testHarness.processElement(new StreamRecord<Integer>(3, initialTime + 3));
	testHarness.processElement(new StreamRecord<Integer>(4, initialTime + 4));
	testHarness.processElement(new StreamRecord<Integer>(5, initialTime + 5));
	testHarness.processElement(new StreamRecord<Integer>(6, initialTime + 6));
	testHarness.processElement(new StreamRecord<Integer>(7, initialTime + 7));

	expectedOutput.add(new StreamRecord<Integer>(2, initialTime + 2));
	expectedOutput.add(new Watermark(initialTime + 2));
	expectedOutput.add(new StreamRecord<Integer>(4, initialTime + 4));
	expectedOutput.add(new StreamRecord<Integer>(6, initialTime + 6));

	TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput());
}
 

@Test
public void testSideOutput() throws Exception {
	try (
			TwoInputStreamOperatorTestHarness<String, Integer, String> testHarness = getInitializedTestHarness(
					BasicTypeInfo.STRING_TYPE_INFO,
					new IdentityKeySelector<>(),
					new FunctionWithSideOutput())
	) {

		testHarness.processWatermark1(new Watermark(10L));
		testHarness.processWatermark2(new Watermark(10L));
		testHarness.processElement2(new StreamRecord<>(5, 12L));

		testHarness.processWatermark1(new Watermark(40L));
		testHarness.processWatermark2(new Watermark(40L));
		testHarness.processElement1(new StreamRecord<>("6", 13L));
		testHarness.processElement1(new StreamRecord<>("6", 15L));

		testHarness.processWatermark1(new Watermark(50L));
		testHarness.processWatermark2(new Watermark(50L));

		Queue<StreamRecord<String>> expectedBr = new ConcurrentLinkedQueue<>();
		expectedBr.add(new StreamRecord<>("BR:5 WM:10 TS:12", 12L));

		Queue<StreamRecord<String>> expectedNonBr = new ConcurrentLinkedQueue<>();
		expectedNonBr.add(new StreamRecord<>("NON-BR:6 WM:40 TS:13", 13L));
		expectedNonBr.add(new StreamRecord<>("NON-BR:6 WM:40 TS:15", 15L));

		TestHarnessUtil.assertOutputEquals(
				"Wrong Side Output",
				expectedBr,
				testHarness.getSideOutput(FunctionWithSideOutput.BROADCAST_TAG));

		TestHarnessUtil.assertOutputEquals(
				"Wrong Side Output",
				expectedNonBr,
				testHarness.getSideOutput(FunctionWithSideOutput.NON_BROADCAST_TAG));
	}
}
 

/**
 * Verifies that we don't have leakage between different keys.
 */
@Test
public void testEventTimeTimerWithState() throws Exception {

	KeyedProcessOperator<Integer, Integer, String> operator =
			new KeyedProcessOperator<>(new TriggeringStatefulFlatMapFunction(TimeDomain.EVENT_TIME));

	OneInputStreamOperatorTestHarness<Integer, String> testHarness =
			new KeyedOneInputStreamOperatorTestHarness<>(operator, new IdentityKeySelector<Integer>(), BasicTypeInfo.INT_TYPE_INFO);

	testHarness.setup();
	testHarness.open();

	testHarness.processWatermark(new Watermark(1));
	testHarness.processElement(new StreamRecord<>(17, 0L)); // should set timer for 6
	testHarness.processElement(new StreamRecord<>(13, 0L)); // should set timer for 6

	testHarness.processWatermark(new Watermark(2));
	testHarness.processElement(new StreamRecord<>(42, 1L)); // should set timer for 7
	testHarness.processElement(new StreamRecord<>(13, 1L)); // should delete timer

	testHarness.processWatermark(new Watermark(6));
	testHarness.processWatermark(new Watermark(7));

	ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>();

	expectedOutput.add(new Watermark(1L));
	expectedOutput.add(new StreamRecord<>("INPUT:17", 0L));
	expectedOutput.add(new StreamRecord<>("INPUT:13", 0L));
	expectedOutput.add(new Watermark(2L));
	expectedOutput.add(new StreamRecord<>("INPUT:42", 1L));
	expectedOutput.add(new StreamRecord<>("STATE:17", 6L));
	expectedOutput.add(new Watermark(6L));
	expectedOutput.add(new StreamRecord<>("STATE:42", 7L));
	expectedOutput.add(new Watermark(7L));

	TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput());

	testHarness.close();
}
 

@Test
public void testInnerHashJoin() throws Exception {

	init(false, false, true);

	testHarness.processElement(new StreamRecord<>(newRow("a", "0"), initialTime), 0, 0);
	testHarness.processElement(new StreamRecord<>(newRow("d", "0"), initialTime), 0, 0);
	testHarness.processElement(new StreamRecord<>(newRow("b", "1"), initialTime), 0, 1);

	testHarness.waitForInputProcessing();

	TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput,
			transformToBinary(testHarness.getOutput()));

	testHarness.endInput(0, 0);
	testHarness.endInput(0, 1);
	testHarness.waitForInputProcessing();

	testHarness.processElement(new StreamRecord<>(newRow("a", "2"), initialTime), 1, 1);
	expectedOutput.add(new StreamRecord<>(newRow("a", "02")));
	testHarness.waitForInputProcessing();
	TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput,
			transformToBinary(testHarness.getOutput()));

	testHarness.processElement(new StreamRecord<>(newRow("c", "2"), initialTime), 1, 1);
	testHarness.processElement(new StreamRecord<>(newRow("b", "4"), initialTime), 1, 0);
	expectedOutput.add(new StreamRecord<>(newRow("b", "14")));
	testHarness.waitForInputProcessing();
	TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput,
			transformToBinary(testHarness.getOutput()));

	testHarness.endInput(1, 0);
	testHarness.endInput(1, 1);
	testHarness.waitForTaskCompletion();
	TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput,
			transformToBinary(testHarness.getOutput()));
}
 

/**
 * Verifies that we don't have leakage between different keys.
 */
@Test
public void testProcessingTimeTimerWithState() throws Exception {

	LegacyKeyedProcessOperator<Integer, Integer, String> operator =
			new LegacyKeyedProcessOperator<>(new TriggeringStatefulFlatMapFunction(TimeDomain.PROCESSING_TIME));

	OneInputStreamOperatorTestHarness<Integer, String> testHarness =
			new KeyedOneInputStreamOperatorTestHarness<>(operator, new IdentityKeySelector<Integer>(), BasicTypeInfo.INT_TYPE_INFO);

	testHarness.setup();
	testHarness.open();

	testHarness.setProcessingTime(1);
	testHarness.processElement(new StreamRecord<>(17)); // should set timer for 6

	testHarness.setProcessingTime(2);
	testHarness.processElement(new StreamRecord<>(42)); // should set timer for 7

	testHarness.setProcessingTime(6);
	testHarness.setProcessingTime(7);

	ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>();

	expectedOutput.add(new StreamRecord<>("INPUT:17"));
	expectedOutput.add(new StreamRecord<>("INPUT:42"));
	expectedOutput.add(new StreamRecord<>("STATE:17"));
	expectedOutput.add(new StreamRecord<>("STATE:42"));

	TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput());

	testHarness.close();
}
 

@Test
public void testCoMap() throws Exception {
	CoStreamMap<Double, Integer, String> operator = new CoStreamMap<Double, Integer, String>(new MyCoMap());

	TwoInputStreamOperatorTestHarness<Double, Integer, String> testHarness = new TwoInputStreamOperatorTestHarness<Double, Integer, String>(operator);

	long initialTime = 0L;
	ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<Object>();

	testHarness.open();

	testHarness.processElement1(new StreamRecord<Double>(1.1d, initialTime + 1));
	testHarness.processElement1(new StreamRecord<Double>(1.2d, initialTime + 2));
	testHarness.processElement1(new StreamRecord<Double>(1.3d, initialTime + 3));
	testHarness.processWatermark1(new Watermark(initialTime + 3));
	testHarness.processElement1(new StreamRecord<Double>(1.4d, initialTime + 4));
	testHarness.processElement1(new StreamRecord<Double>(1.5d, initialTime + 5));

	testHarness.processElement2(new StreamRecord<Integer>(1, initialTime + 1));
	testHarness.processElement2(new StreamRecord<Integer>(2, initialTime + 2));
	testHarness.processWatermark2(new Watermark(initialTime + 2));
	testHarness.processElement2(new StreamRecord<Integer>(3, initialTime + 3));
	testHarness.processElement2(new StreamRecord<Integer>(4, initialTime + 4));
	testHarness.processElement2(new StreamRecord<Integer>(5, initialTime + 5));

	expectedOutput.add(new StreamRecord<String>("1.1", initialTime + 1));
	expectedOutput.add(new StreamRecord<String>("1.2", initialTime + 2));
	expectedOutput.add(new StreamRecord<String>("1.3", initialTime + 3));
	expectedOutput.add(new StreamRecord<String>("1.4", initialTime + 4));
	expectedOutput.add(new StreamRecord<String>("1.5", initialTime + 5));

	expectedOutput.add(new StreamRecord<String>("1", initialTime + 1));
	expectedOutput.add(new StreamRecord<String>("2", initialTime + 2));
	expectedOutput.add(new Watermark(initialTime + 2));
	expectedOutput.add(new StreamRecord<String>("3", initialTime + 3));
	expectedOutput.add(new StreamRecord<String>("4", initialTime + 4));
	expectedOutput.add(new StreamRecord<String>("5", initialTime + 5));

	TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput());
}
 

@Test
public void testNotMarkingEndOfInputWhenTaskCancelled () throws Exception {
	final StreamTaskTestHarness<String> testHarness = new StreamTaskTestHarness<>(
		SourceStreamTask::new,
		BasicTypeInfo.STRING_TYPE_INFO);

	testHarness
		.setupOperatorChain(
			new OperatorID(),
			new StreamSource<>(new CancelTestSource(
				BasicTypeInfo.STRING_TYPE_INFO.createSerializer(new ExecutionConfig()), "Hello")))
		.chain(
			new OperatorID(),
			new TestBoundedOneInputStreamOperator("Operator1"),
			BasicTypeInfo.STRING_TYPE_INFO.createSerializer(new ExecutionConfig()))
		.finish();

	StreamConfig streamConfig = testHarness.getStreamConfig();
	streamConfig.setTimeCharacteristic(TimeCharacteristic.ProcessingTime);

	ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>();

	testHarness.invoke();
	CancelTestSource.getDataProcessing().await();
	testHarness.getTask().cancel();

	try {
		testHarness.waitForTaskCompletion();
	} catch (Throwable t) {
		if (!ExceptionUtils.findThrowable(t, CancelTaskException.class).isPresent()) {
			throw t;
		}
	}

	expectedOutput.add(new StreamRecord<>("Hello"));

	TestHarnessUtil.assertOutputEquals("Output was not correct.",
		expectedOutput,
		testHarness.getOutput());
}
 

@Test
public void testSideOutput() throws Exception {
	try (
			TwoInputStreamOperatorTestHarness<String, Integer, String> testHarness = getInitializedTestHarness(
					BasicTypeInfo.STRING_TYPE_INFO,
					new IdentityKeySelector<>(),
					new FunctionWithSideOutput())
	) {

		testHarness.processWatermark1(new Watermark(10L));
		testHarness.processWatermark2(new Watermark(10L));
		testHarness.processElement2(new StreamRecord<>(5, 12L));

		testHarness.processWatermark1(new Watermark(40L));
		testHarness.processWatermark2(new Watermark(40L));
		testHarness.processElement1(new StreamRecord<>("6", 13L));
		testHarness.processElement1(new StreamRecord<>("6", 15L));

		testHarness.processWatermark1(new Watermark(50L));
		testHarness.processWatermark2(new Watermark(50L));

		Queue<StreamRecord<String>> expectedBr = new ConcurrentLinkedQueue<>();
		expectedBr.add(new StreamRecord<>("BR:5 WM:10 TS:12", 12L));

		Queue<StreamRecord<String>> expectedNonBr = new ConcurrentLinkedQueue<>();
		expectedNonBr.add(new StreamRecord<>("NON-BR:6 WM:40 TS:13", 13L));
		expectedNonBr.add(new StreamRecord<>("NON-BR:6 WM:40 TS:15", 15L));

		TestHarnessUtil.assertOutputEquals(
				"Wrong Side Output",
				expectedBr,
				testHarness.getSideOutput(FunctionWithSideOutput.BROADCAST_TAG));

		TestHarnessUtil.assertOutputEquals(
				"Wrong Side Output",
				expectedNonBr,
				testHarness.getSideOutput(FunctionWithSideOutput.NON_BROADCAST_TAG));
	}
}
 

@Test
public void testCleanupTimerWithEmptyListStateForSessionWindows() throws Exception {
	final int gapSize = 3;
	final long lateness = 10;

	ListStateDescriptor<Tuple2<String, Integer>> windowStateDesc =
		new ListStateDescriptor<>("window-contents", STRING_INT_TUPLE.createSerializer(new ExecutionConfig()));

	WindowOperator<String, Tuple2<String, Integer>, Iterable<Tuple2<String, Integer>>, Tuple2<String, Integer>, TimeWindow> operator =
		new WindowOperator<>(
			EventTimeSessionWindows.withGap(Time.seconds(gapSize)),
			new TimeWindow.Serializer(),
			new TupleKeySelector(),
			BasicTypeInfo.STRING_TYPE_INFO.createSerializer(new ExecutionConfig()),
			windowStateDesc,
			new InternalIterableWindowFunction<>(new PassThroughFunction()),
			EventTimeTrigger.create(),
			lateness,
			null /* late data output tag */);

	OneInputStreamOperatorTestHarness<Tuple2<String, Integer>, Tuple2<String, Integer>> testHarness =
		createTestHarness(operator);

	testHarness.open();

	ConcurrentLinkedQueue<Object> expected = new ConcurrentLinkedQueue<>();

	testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 1000));
	testHarness.processWatermark(new Watermark(4998));

	expected.add(new StreamRecord<>(new Tuple2<>("key2", 1), 3999));
	expected.add(new Watermark(4998));

	testHarness.processWatermark(new Watermark(14600));
	expected.add(new Watermark(14600));

	ConcurrentLinkedQueue<Object> actual = testHarness.getOutput();
	TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expected, actual, new Tuple2ResultSortComparator());
	testHarness.close();
}
 

@Test
public void testFunctionWithTimer() throws Exception {
	final String expectedKey = "6";

	try (
			TwoInputStreamOperatorTestHarness<String, Integer, String> testHarness = getInitializedTestHarness(
					BasicTypeInfo.STRING_TYPE_INFO,
					new IdentityKeySelector<>(),
					new FunctionWithTimerOnKeyed(41L, expectedKey))
	) {
		testHarness.processWatermark1(new Watermark(10L));
		testHarness.processWatermark2(new Watermark(10L));
		testHarness.processElement2(new StreamRecord<>(5, 12L));

		testHarness.processWatermark1(new Watermark(40L));
		testHarness.processWatermark2(new Watermark(40L));
		testHarness.processElement1(new StreamRecord<>(expectedKey, 13L));
		testHarness.processElement1(new StreamRecord<>(expectedKey, 15L));

		testHarness.processWatermark1(new Watermark(50L));
		testHarness.processWatermark2(new Watermark(50L));

		Queue<Object> expectedOutput = new ConcurrentLinkedQueue<>();

		expectedOutput.add(new Watermark(10L));
		expectedOutput.add(new StreamRecord<>("BR:5 WM:10 TS:12", 12L));
		expectedOutput.add(new Watermark(40L));
		expectedOutput.add(new StreamRecord<>("NON-BR:6 WM:40 TS:13", 13L));
		expectedOutput.add(new StreamRecord<>("NON-BR:6 WM:40 TS:15", 15L));
		expectedOutput.add(new StreamRecord<>("TIMER:41", 41L));
		expectedOutput.add(new Watermark(50L));

		TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput());
	}
}
 

@Test
public void testLateElements() throws Exception {
    testHarness.processWatermark(1L);
    assertEquals(1L, operator.lastWatermark);
    testHarness.processElement(record(K1, 0L));
    testHarness.processElement(record(K1, 1L));
    testHarness.processWatermark(2L);
    assertEquals(2L, operator.lastWatermark);

    Queue<Object> actual = testHarness.getOutput();
    Queue<Object> expected = new ConcurrentLinkedQueue<>();
    expected.add(watermark(1L));
    expected.add(watermark(2L));
    TestHarnessUtil.assertOutputEquals("Unexpected output", expected, actual);
}
 

/**
 * Verifies that we don't have leakage between different keys.
 */
@Test
public void testEventTimeTimerWithState() throws Exception {

	KeyedProcessOperator<Integer, Integer, String> operator =
			new KeyedProcessOperator<>(new TriggeringStatefulFlatMapFunction(TimeDomain.EVENT_TIME));

	OneInputStreamOperatorTestHarness<Integer, String> testHarness =
			new KeyedOneInputStreamOperatorTestHarness<>(operator, new IdentityKeySelector<Integer>(), BasicTypeInfo.INT_TYPE_INFO);

	testHarness.setup();
	testHarness.open();

	testHarness.processWatermark(new Watermark(1));
	testHarness.processElement(new StreamRecord<>(17, 0L)); // should set timer for 6
	testHarness.processElement(new StreamRecord<>(13, 0L)); // should set timer for 6

	testHarness.processWatermark(new Watermark(2));
	testHarness.processElement(new StreamRecord<>(42, 1L)); // should set timer for 7
	testHarness.processElement(new StreamRecord<>(13, 1L)); // should delete timer

	testHarness.processWatermark(new Watermark(6));
	testHarness.processWatermark(new Watermark(7));

	ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>();

	expectedOutput.add(new Watermark(1L));
	expectedOutput.add(new StreamRecord<>("INPUT:17", 0L));
	expectedOutput.add(new StreamRecord<>("INPUT:13", 0L));
	expectedOutput.add(new Watermark(2L));
	expectedOutput.add(new StreamRecord<>("INPUT:42", 1L));
	expectedOutput.add(new StreamRecord<>("STATE:17", 6L));
	expectedOutput.add(new Watermark(6L));
	expectedOutput.add(new StreamRecord<>("STATE:42", 7L));
	expectedOutput.add(new Watermark(7L));

	TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput());

	testHarness.close();
}
 

private void processRecords(OneInputStreamTaskTestHarness<String, String> testHarness) throws Exception {
	ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>();

	testHarness.processElement(new StreamRecord<>("10"), 0, 0);
	testHarness.processElement(new StreamRecord<>("20"), 0, 0);
	testHarness.processElement(new StreamRecord<>("30"), 0, 0);

	testHarness.waitForInputProcessing();

	expectedOutput.add(new StreamRecord<>("10"));
	expectedOutput.add(new StreamRecord<>("20"));
	expectedOutput.add(new StreamRecord<>("30"));
	TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput());
}