Java Code Examples for org.apache.flink.streaming.util.OneInputStreamOperatorTestHarness#getOutput()

@Test
public void testWatermarkAssignerWithIdleSource() throws Exception {
	// with timeout 1000 ms
	final WatermarkAssignerOperator operator = new WatermarkAssignerOperator(0, 1, 1000);
	OneInputStreamOperatorTestHarness<BaseRow, BaseRow> testHarness =
		new OneInputStreamOperatorTestHarness<>(operator);
	testHarness.getExecutionConfig().setAutoWatermarkInterval(50);
	testHarness.open();

	testHarness.processElement(new StreamRecord<>(GenericRow.of(1L)));
	testHarness.processElement(new StreamRecord<>(GenericRow.of(2L)));
	testHarness.processWatermark(new Watermark(2)); // this watermark should be ignored
	testHarness.processElement(new StreamRecord<>(GenericRow.of(3L)));
	testHarness.processElement(new StreamRecord<>(GenericRow.of(4L)));

	// trigger watermark emit
	testHarness.setProcessingTime(51);
	ConcurrentLinkedQueue<Object> output = testHarness.getOutput();
	List<Watermark> watermarks = extractWatermarks(output);
	assertEquals(1, watermarks.size());
	assertEquals(new Watermark(3), watermarks.get(0));
	assertEquals(StreamStatus.ACTIVE, testHarness.getStreamStatus());
	output.clear();

	testHarness.setProcessingTime(1001);
	assertEquals(StreamStatus.IDLE, testHarness.getStreamStatus());

	testHarness.processElement(new StreamRecord<>(GenericRow.of(4L)));
	testHarness.processElement(new StreamRecord<>(GenericRow.of(5L)));
	testHarness.processElement(new StreamRecord<>(GenericRow.of(6L)));
	testHarness.processElement(new StreamRecord<>(GenericRow.of(7L)));
	testHarness.processElement(new StreamRecord<>(GenericRow.of(8L)));

	assertEquals(StreamStatus.ACTIVE, testHarness.getStreamStatus());
	testHarness.setProcessingTime(1060);
	output = testHarness.getOutput();
	watermarks = extractWatermarks(output);
	assertEquals(1, watermarks.size());
	assertEquals(new Watermark(7), watermarks.get(0));
}
 

@Test
public void testKeyedCEPOperatorNFAUpdateTimes() throws Exception {
	CepOperator<Event, Integer, Map<String, List<Event>>> operator = CepOperatorTestUtilities.getKeyedCepOpearator(
		true,
		new SimpleNFAFactory());
	OneInputStreamOperatorTestHarness<Event, Map<String, List<Event>>> harness = CepOperatorTestUtilities.getCepTestHarness(operator);

	try {
		harness.open();

		final ValueState nfaOperatorState = (ValueState) Whitebox.<ValueState>getInternalState(operator, "computationStates");
		final ValueState nfaOperatorStateSpy = Mockito.spy(nfaOperatorState);
		Whitebox.setInternalState(operator, "computationStates", nfaOperatorStateSpy);

		Event startEvent = new Event(42, "c", 1.0);
		SubEvent middleEvent = new SubEvent(42, "a", 1.0, 10.0);
		Event endEvent = new Event(42, "b", 1.0);

		harness.processElement(new StreamRecord<>(startEvent, 1L));
		harness.processElement(new StreamRecord<>(new Event(42, "d", 1.0), 4L));
		harness.processElement(new StreamRecord<Event>(middleEvent, 4L));
		harness.processElement(new StreamRecord<>(endEvent, 4L));

		// verify the number of invocations NFA is updated
		Mockito.verify(nfaOperatorStateSpy, Mockito.times(3)).update(Mockito.any());

		// get and verify the output
		Queue<Object> result = harness.getOutput();

		assertEquals(1, result.size());

		verifyPattern(result.poll(), startEvent, middleEvent, endEvent);
	} finally {
		harness.close();
	}
}
 

@Test
public void testCleanupTimerWithEmptyListStateForSessionWindows() throws Exception {
	final int gapSize = 3;
	final long lateness = 10;

	ListStateDescriptor<Tuple2<String, Integer>> windowStateDesc =
		new ListStateDescriptor<>("window-contents", STRING_INT_TUPLE.createSerializer(new ExecutionConfig()));

	WindowOperator<String, Tuple2<String, Integer>, Iterable<Tuple2<String, Integer>>, Tuple2<String, Integer>, TimeWindow> operator =
		new WindowOperator<>(
			EventTimeSessionWindows.withGap(Time.seconds(gapSize)),
			new TimeWindow.Serializer(),
			new TupleKeySelector(),
			BasicTypeInfo.STRING_TYPE_INFO.createSerializer(new ExecutionConfig()),
			windowStateDesc,
			new InternalIterableWindowFunction<>(new PassThroughFunction()),
			EventTimeTrigger.create(),
			lateness,
			null /* late data output tag */);

	OneInputStreamOperatorTestHarness<Tuple2<String, Integer>, Tuple2<String, Integer>> testHarness =
		createTestHarness(operator);

	testHarness.open();

	ConcurrentLinkedQueue<Object> expected = new ConcurrentLinkedQueue<>();

	testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 1000));
	testHarness.processWatermark(new Watermark(4998));

	expected.add(new StreamRecord<>(new Tuple2<>("key2", 1), 3999));
	expected.add(new Watermark(4998));

	testHarness.processWatermark(new Watermark(14600));
	expected.add(new Watermark(14600));

	ConcurrentLinkedQueue<Object> actual = testHarness.getOutput();
	TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expected, actual, new Tuple2ResultSortComparator());
	testHarness.close();
}
 

@Test
public void testNotSideOutputDueToLatenessSessionWithLateness() throws Exception {
	// same as testSideOutputDueToLatenessSessionWithLateness() but with an accumulating trigger, i.e.
	// one that does not return FIRE_AND_PURGE when firing but just FIRE. The expected
	// results are therefore slightly different.

	final int gapSize = 3;
	final long lateness = 10;

	ReducingStateDescriptor<Tuple2<String, Integer>> stateDesc = new ReducingStateDescriptor<>("window-contents",
		new SumReducer(),
		STRING_INT_TUPLE.createSerializer(new ExecutionConfig()));

	WindowOperator<String, Tuple2<String, Integer>, Tuple2<String, Integer>, Tuple3<String, Long, Long>, TimeWindow> operator =
		new WindowOperator<>(
			EventTimeSessionWindows.withGap(Time.seconds(gapSize)),
			new TimeWindow.Serializer(),
			new TupleKeySelector(),
			BasicTypeInfo.STRING_TYPE_INFO.createSerializer(new ExecutionConfig()),
			stateDesc,
			new InternalSingleValueWindowFunction<>(new ReducedSessionWindowFunction()),
			EventTimeTrigger.create(),
			lateness,
			lateOutputTag /* late data output tag */);

	OneInputStreamOperatorTestHarness<Tuple2<String, Integer>, Tuple3<String, Long, Long>> testHarness =
		createTestHarness(operator);

	testHarness.open();

	ConcurrentLinkedQueue<Object> expected = new ConcurrentLinkedQueue<>();

	testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 1000));
	testHarness.processWatermark(new Watermark(1999));

	expected.add(new Watermark(1999));

	testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 2000));
	testHarness.processWatermark(new Watermark(4998));

	expected.add(new Watermark(4998));

	// this will not be sideoutput because the session we're adding two has maxTimestamp
	// after the current watermark
	testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 4500));

	// new session
	testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 8500));
	testHarness.processWatermark(new Watermark(7400));

	expected.add(new Watermark(7400));

	// this will merge the two sessions into one
	testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 7000));
	testHarness.processWatermark(new Watermark(11501));

	expected.add(new StreamRecord<>(new Tuple3<>("key2-5", 1000L, 11500L), 11499));
	expected.add(new Watermark(11501));

	// new session
	testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 11600));
	testHarness.processWatermark(new Watermark(14600));

	expected.add(new StreamRecord<>(new Tuple3<>("key2-1", 11600L, 14600L), 14599));
	expected.add(new Watermark(14600));

	// because of the small allowed lateness and because the trigger is accumulating
	// this will be merged into the session (11600-14600) and therefore will not
	// be sideoutput as late
	testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 10000));

	testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 14500));

	// adding ("key2", 1) extended the session to (10000-146000) for which
	// maxTimestamp <= currentWatermark. Therefore, we immediately get a firing
	// with the current version of EventTimeTrigger/EventTimeTriggerAccum
	expected.add(new StreamRecord<>(new Tuple3<>("key2-2", 10000L, 14600L), 14599));

	ConcurrentLinkedQueue<Object> actual = testHarness.getOutput();
	ConcurrentLinkedQueue<StreamRecord<Tuple2<String, Integer>>> sideActual = testHarness.getSideOutput(
			lateOutputTag);

	TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expected, actual, new Tuple3ResultSortComparator());
	assertEquals(null, sideActual);

	testHarness.processWatermark(new Watermark(20000));

	expected.add(new StreamRecord<>(new Tuple3<>("key2-3", 10000L, 17500L), 17499));
	expected.add(new Watermark(20000));

	testHarness.processWatermark(new Watermark(100000));

	expected.add(new Watermark(100000));

	actual = testHarness.getOutput();
	sideActual = testHarness.getSideOutput(lateOutputTag);
	TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expected, actual, new Tuple3ResultSortComparator());
	assertEquals(null, sideActual);

	testHarness.close();
}
 

@Test
public void testKeyedCEPOperatorNFAUpdate() throws Exception {

	CepOperator<Event, Integer, Map<String, List<Event>>> operator = CepOperatorTestUtilities.getKeyedCepOpearator(
		true,
		new SimpleNFAFactory());
	OneInputStreamOperatorTestHarness<Event, Map<String, List<Event>>> harness = CepOperatorTestUtilities.getCepTestHarness(
		operator);

	try {
		harness.open();

		Event startEvent = new Event(42, "c", 1.0);
		SubEvent middleEvent = new SubEvent(42, "a", 1.0, 10.0);
		Event endEvent = new Event(42, "b", 1.0);

		harness.processElement(new StreamRecord<>(startEvent, 1L));

		// simulate snapshot/restore with some elements in internal sorting queue
		OperatorSubtaskState snapshot = harness.snapshot(0L, 0L);
		harness.close();

		operator = CepOperatorTestUtilities.getKeyedCepOpearator(true, new SimpleNFAFactory());
		harness = CepOperatorTestUtilities.getCepTestHarness(operator);

		harness.setup();
		harness.initializeState(snapshot);
		harness.open();

		harness.processElement(new StreamRecord<>(new Event(42, "d", 1.0), 4L));
		OperatorSubtaskState snapshot2 = harness.snapshot(0L, 0L);
		harness.close();

		operator = CepOperatorTestUtilities.getKeyedCepOpearator(true, new SimpleNFAFactory());
		harness = CepOperatorTestUtilities.getCepTestHarness(operator);

		harness.setup();
		harness.initializeState(snapshot2);
		harness.open();

		harness.processElement(new StreamRecord<Event>(middleEvent, 4L));
		harness.processElement(new StreamRecord<>(endEvent, 4L));

		// get and verify the output

		Queue<Object> result = harness.getOutput();

		assertEquals(1, result.size());

		verifyPattern(result.poll(), startEvent, middleEvent, endEvent);
	} finally {
		harness.close();
	}
}
 

@Test
public void testSinglePatternAfterMigration() throws Exception {

	KeySelector<Event, Integer> keySelector = new KeySelector<Event, Integer>() {
		private static final long serialVersionUID = -4873366487571254798L;

		@Override
		public Integer getKey(Event value) throws Exception {
			return value.getId();
		}
	};

	final Event startEvent1 = new Event(42, "start", 1.0);

	OneInputStreamOperatorTestHarness<Event, Map<String, List<Event>>> harness =
			new KeyedOneInputStreamOperatorTestHarness<>(
					getKeyedCepOpearator(false, new SinglePatternNFAFactory()),
					keySelector,
					BasicTypeInfo.INT_TYPE_INFO);

	try {
		harness.setup();

		harness.initializeState(
			OperatorSnapshotUtil.getResourceFilename(
				"cep-migration-single-pattern-afterwards-flink" + migrateVersion + "-snapshot"));

		harness.open();

		harness.processElement(new StreamRecord<>(startEvent1, 5));

		harness.processWatermark(new Watermark(20));

		ConcurrentLinkedQueue<Object> result = harness.getOutput();

		// watermark and the result
		assertEquals(2, result.size());

		Object resultObject = result.poll();
		assertTrue(resultObject instanceof StreamRecord);
		StreamRecord<?> resultRecord = (StreamRecord<?>) resultObject;
		assertTrue(resultRecord.getValue() instanceof Map);

		@SuppressWarnings("unchecked")
		Map<String, List<Event>> patternMap =
			(Map<String, List<Event>>) resultRecord.getValue();

		assertEquals(startEvent1, patternMap.get("start").get(0));
	} finally {
		harness.close();
	}
}
 

@Test
public void testSideOutputDueToLatenessSessionZeroLateness() throws Exception {
	final int gapSize = 3;
	final long lateness = 0;

	ReducingStateDescriptor<Tuple2<String, Integer>> stateDesc = new ReducingStateDescriptor<>("window-contents",
		new SumReducer(),
		STRING_INT_TUPLE.createSerializer(new ExecutionConfig()));

	WindowOperator<String, Tuple2<String, Integer>, Tuple2<String, Integer>, Tuple3<String, Long, Long>, TimeWindow> operator =
		new WindowOperator<>(
			EventTimeSessionWindows.withGap(Time.seconds(gapSize)),
			new TimeWindow.Serializer(),
			new TupleKeySelector(),
			BasicTypeInfo.STRING_TYPE_INFO.createSerializer(new ExecutionConfig()),
			stateDesc,
			new InternalSingleValueWindowFunction<>(new ReducedSessionWindowFunction()),
			EventTimeTrigger.create(),
			lateness,
			lateOutputTag);

	OneInputStreamOperatorTestHarness<Tuple2<String, Integer>, Tuple3<String, Long, Long>> testHarness =
		createTestHarness(operator);

	testHarness.open();

	ConcurrentLinkedQueue<Object> expected = new ConcurrentLinkedQueue<>();
	ConcurrentLinkedQueue<Object> sideExpected = new ConcurrentLinkedQueue<>();

	testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 1000));
	testHarness.processWatermark(new Watermark(1999));

	expected.add(new Watermark(1999));

	testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 2000));
	testHarness.processWatermark(new Watermark(4998));

	expected.add(new Watermark(4998));

	// this will not be dropped because the session we're adding two has maxTimestamp
	// after the current watermark
	testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 4500));

	// new session
	testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 8500));
	testHarness.processWatermark(new Watermark(7400));

	expected.add(new Watermark(7400));

	// this will merge the two sessions into one
	testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 7000));
	testHarness.processWatermark(new Watermark(11501));

	expected.add(new StreamRecord<>(new Tuple3<>("key2-5", 1000L, 11500L), 11499));
	expected.add(new Watermark(11501));

	// new session
	testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 11600));
	testHarness.processWatermark(new Watermark(14600));

	expected.add(new StreamRecord<>(new Tuple3<>("key2-1", 11600L, 14600L), 14599));
	expected.add(new Watermark(14600));

	// this is sideoutput as late, reuse last timestamp
	testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 10000));
	sideExpected.add(new StreamRecord<>(new Tuple2<>("key2", 1), 10000));

	testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 14500));
	testHarness.processWatermark(new Watermark(20000));

	expected.add(new StreamRecord<>(new Tuple3<>("key2-1", 14500L, 17500L), 17499));
	expected.add(new Watermark(20000));

	testHarness.processWatermark(new Watermark(100000));
	expected.add(new Watermark(100000));

	ConcurrentLinkedQueue<Object> actual = testHarness.getOutput();
	ConcurrentLinkedQueue<StreamRecord<Tuple2<String, Integer>>> sideActual = testHarness.getSideOutput(
			lateOutputTag);
	TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expected, actual, new Tuple2ResultSortComparator());
	TestHarnessUtil.assertOutputEqualsSorted("SideOutput was not correct.", sideExpected, (Iterable) sideActual, new Tuple2ResultSortComparator());
	testHarness.close();
}
 

@Test
public void testNotSideOutputDueToLatenessSessionWithHugeLateness() throws Exception {
	final int gapSize = 3;
	final long lateness = 10000;

	ReducingStateDescriptor<Tuple2<String, Integer>> stateDesc = new ReducingStateDescriptor<>("window-contents",
		new SumReducer(),
		STRING_INT_TUPLE.createSerializer(new ExecutionConfig()));

	WindowOperator<String, Tuple2<String, Integer>, Tuple2<String, Integer>, Tuple3<String, Long, Long>, TimeWindow> operator =
		new WindowOperator<>(
			EventTimeSessionWindows.withGap(Time.seconds(gapSize)),
			new TimeWindow.Serializer(),
			new TupleKeySelector(),
			BasicTypeInfo.STRING_TYPE_INFO.createSerializer(new ExecutionConfig()),
			stateDesc,
			new InternalSingleValueWindowFunction<>(new ReducedSessionWindowFunction()),
			EventTimeTrigger.create(),
			lateness,
			lateOutputTag /* late data output tag */);

	OneInputStreamOperatorTestHarness<Tuple2<String, Integer>, Tuple3<String, Long, Long>> testHarness =
		createTestHarness(operator);

	testHarness.open();

	ConcurrentLinkedQueue<Object> expected = new ConcurrentLinkedQueue<>();

	testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 1000));
	testHarness.processWatermark(new Watermark(1999));

	expected.add(new Watermark(1999));

	testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 2000));
	testHarness.processWatermark(new Watermark(4998));

	expected.add(new Watermark(4998));

	// this will not be sideoutput because the session we're adding two has maxTimestamp
	// after the current watermark
	testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 4500));

	// new session
	testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 8500));
	testHarness.processWatermark(new Watermark(7400));

	expected.add(new Watermark(7400));

	// this will merge the two sessions into one
	testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 7000));
	testHarness.processWatermark(new Watermark(11501));

	expected.add(new StreamRecord<>(new Tuple3<>("key2-5", 1000L, 11500L), 11499));
	expected.add(new Watermark(11501));

	// new session
	testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 11600));
	testHarness.processWatermark(new Watermark(14600));

	expected.add(new StreamRecord<>(new Tuple3<>("key2-1", 11600L, 14600L), 14599));
	expected.add(new Watermark(14600));

	testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 10000));

	// the maxTimestamp of the merged session is already late,
	// so we get an immediate firing
	expected.add(new StreamRecord<>(new Tuple3<>("key2-7", 1000L, 14600L), 14599));

	ConcurrentLinkedQueue<Object> actual = testHarness.getOutput();
	ConcurrentLinkedQueue<StreamRecord<Tuple2<String, Integer>>> sideActual = testHarness.getSideOutput(lateOutputTag);
	TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expected, actual, new Tuple3ResultSortComparator());
	assertEquals(null, sideActual);

	testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 14500));
	testHarness.processWatermark(new Watermark(20000));

	expected.add(new StreamRecord<>(new Tuple3<>("key2-8", 1000L, 17500L), 17499));
	expected.add(new Watermark(20000));

	testHarness.processWatermark(new Watermark(100000));
	expected.add(new Watermark(100000));

	actual = testHarness.getOutput();
	sideActual = testHarness.getSideOutput(lateOutputTag);

	TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expected, actual, new Tuple3ResultSortComparator());
	assertEquals(null, sideActual);

	testHarness.close();
}
 

@Test
public void testMiniBatchedWatermarkAssignerWithIdleSource() throws Exception {
	// with timeout 1000 ms
	final MiniBatchedWatermarkAssignerOperator operator = new MiniBatchedWatermarkAssignerOperator(
			0, 1, 0, 1000, 50);
	OneInputStreamOperatorTestHarness<BaseRow, BaseRow> testHarness =
			new OneInputStreamOperatorTestHarness<>(operator);
	testHarness.open();

	testHarness.processElement(new StreamRecord<>(GenericRow.of(1L)));
	testHarness.processElement(new StreamRecord<>(GenericRow.of(2L)));
	testHarness.processWatermark(new Watermark(2)); // this watermark should be ignored
	testHarness.processElement(new StreamRecord<>(GenericRow.of(3L)));
	testHarness.processElement(new StreamRecord<>(GenericRow.of(4L)));
	// this watermark excess expected watermark, should emit a watermark of 49
	testHarness.processElement(new StreamRecord<>(GenericRow.of(50L)));

	ConcurrentLinkedQueue<Object> output = testHarness.getOutput();
	List<Watermark> watermarks = extractWatermarks(output);
	assertEquals(1, watermarks.size());
	assertEquals(new Watermark(49), watermarks.get(0));
	assertEquals(StreamStatus.ACTIVE, testHarness.getStreamStatus());
	output.clear();

	testHarness.setProcessingTime(1001);
	assertEquals(StreamStatus.IDLE, testHarness.getStreamStatus());

	testHarness.processElement(new StreamRecord<>(GenericRow.of(51L)));
	assertEquals(StreamStatus.ACTIVE, testHarness.getStreamStatus());

	// process time will not trigger to emit watermark
	testHarness.setProcessingTime(1060);
	output = testHarness.getOutput();
	watermarks = extractWatermarks(output);
	assertTrue(watermarks.isEmpty());
	output.clear();

	testHarness.processElement(new StreamRecord<>(GenericRow.of(100L)));
	output = testHarness.getOutput();
	watermarks = extractWatermarks(output);
	assertEquals(1, watermarks.size());
	assertEquals(new Watermark(99), watermarks.get(0));
}
 

@Test
public void testNotSideOutputDueToLatenessSessionWithLateness() throws Exception {
	// same as testSideOutputDueToLatenessSessionWithLateness() but with an accumulating trigger, i.e.
	// one that does not return FIRE_AND_PURGE when firing but just FIRE. The expected
	// results are therefore slightly different.

	final int gapSize = 3;
	final long lateness = 10;

	ReducingStateDescriptor<Tuple2<String, Integer>> stateDesc = new ReducingStateDescriptor<>("window-contents",
		new SumReducer(),
		STRING_INT_TUPLE.createSerializer(new ExecutionConfig()));

	WindowOperator<String, Tuple2<String, Integer>, Tuple2<String, Integer>, Tuple3<String, Long, Long>, TimeWindow> operator =
		new WindowOperator<>(
			EventTimeSessionWindows.withGap(Time.seconds(gapSize)),
			new TimeWindow.Serializer(),
			new TupleKeySelector(),
			BasicTypeInfo.STRING_TYPE_INFO.createSerializer(new ExecutionConfig()),
			stateDesc,
			new InternalSingleValueWindowFunction<>(new ReducedSessionWindowFunction()),
			EventTimeTrigger.create(),
			lateness,
			lateOutputTag /* late data output tag */);

	OneInputStreamOperatorTestHarness<Tuple2<String, Integer>, Tuple3<String, Long, Long>> testHarness =
		createTestHarness(operator);

	testHarness.open();

	ConcurrentLinkedQueue<Object> expected = new ConcurrentLinkedQueue<>();

	testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 1000));
	testHarness.processWatermark(new Watermark(1999));

	expected.add(new Watermark(1999));

	testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 2000));
	testHarness.processWatermark(new Watermark(4998));

	expected.add(new Watermark(4998));

	// this will not be sideoutput because the session we're adding two has maxTimestamp
	// after the current watermark
	testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 4500));

	// new session
	testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 8500));
	testHarness.processWatermark(new Watermark(7400));

	expected.add(new Watermark(7400));

	// this will merge the two sessions into one
	testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 7000));
	testHarness.processWatermark(new Watermark(11501));

	expected.add(new StreamRecord<>(new Tuple3<>("key2-5", 1000L, 11500L), 11499));
	expected.add(new Watermark(11501));

	// new session
	testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 11600));
	testHarness.processWatermark(new Watermark(14600));

	expected.add(new StreamRecord<>(new Tuple3<>("key2-1", 11600L, 14600L), 14599));
	expected.add(new Watermark(14600));

	// because of the small allowed lateness and because the trigger is accumulating
	// this will be merged into the session (11600-14600) and therefore will not
	// be sideoutput as late
	testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 10000));

	testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 14500));

	// adding ("key2", 1) extended the session to (10000-146000) for which
	// maxTimestamp <= currentWatermark. Therefore, we immediately get a firing
	// with the current version of EventTimeTrigger/EventTimeTriggerAccum
	expected.add(new StreamRecord<>(new Tuple3<>("key2-2", 10000L, 14600L), 14599));

	ConcurrentLinkedQueue<Object> actual = testHarness.getOutput();
	ConcurrentLinkedQueue<StreamRecord<Tuple2<String, Integer>>> sideActual = testHarness.getSideOutput(
			lateOutputTag);

	TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expected, actual, new Tuple3ResultSortComparator());
	assertEquals(null, sideActual);

	testHarness.processWatermark(new Watermark(20000));

	expected.add(new StreamRecord<>(new Tuple3<>("key2-3", 10000L, 17500L), 17499));
	expected.add(new Watermark(20000));

	testHarness.processWatermark(new Watermark(100000));

	expected.add(new Watermark(100000));

	actual = testHarness.getOutput();
	sideActual = testHarness.getSideOutput(lateOutputTag);
	TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expected, actual, new Tuple3ResultSortComparator());
	assertEquals(null, sideActual);

	testHarness.close();
}
 

@Test
public void testDropDueToLatenessSessionWithLatenessPurgingTrigger() throws Exception {

	// this has the same output as testSideOutputDueToLatenessSessionZeroLateness() because
	// the allowed lateness is too small to make a difference

	final int gapSize = 3;
	final long lateness = 10;

	ReducingStateDescriptor<Tuple2<String, Integer>> stateDesc = new ReducingStateDescriptor<>("window-contents",
		new SumReducer(),
		STRING_INT_TUPLE.createSerializer(new ExecutionConfig()));

	WindowOperator<String, Tuple2<String, Integer>, Tuple2<String, Integer>, Tuple3<String, Long, Long>, TimeWindow> operator =
		new WindowOperator<>(
			EventTimeSessionWindows.withGap(Time.seconds(gapSize)),
			new TimeWindow.Serializer(),
			new TupleKeySelector(),
			BasicTypeInfo.STRING_TYPE_INFO.createSerializer(new ExecutionConfig()),
			stateDesc,
			new InternalSingleValueWindowFunction<>(new ReducedSessionWindowFunction()),
			PurgingTrigger.of(EventTimeTrigger.create()),
			lateness,
			lateOutputTag);

	OneInputStreamOperatorTestHarness<Tuple2<String, Integer>, Tuple3<String, Long, Long>> testHarness =
		createTestHarness(operator);

	testHarness.open();

	ConcurrentLinkedQueue<Object> expected = new ConcurrentLinkedQueue<>();

	testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 1000));
	testHarness.processWatermark(new Watermark(1999));

	expected.add(new Watermark(1999));

	testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 2000));
	testHarness.processWatermark(new Watermark(4998));

	expected.add(new Watermark(4998));

	// this will not be dropped because the session we're adding two has maxTimestamp
	// after the current watermark
	testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 4500));

	// new session
	testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 8500));
	testHarness.processWatermark(new Watermark(7400));

	expected.add(new Watermark(7400));

	// this will merge the two sessions into one
	testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 7000));
	testHarness.processWatermark(new Watermark(11501));

	expected.add(new StreamRecord<>(new Tuple3<>("key2-5", 1000L, 11500L), 11499));
	expected.add(new Watermark(11501));

	// new session
	testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 11600));
	testHarness.processWatermark(new Watermark(14600));

	expected.add(new StreamRecord<>(new Tuple3<>("key2-1", 11600L, 14600L), 14599));
	expected.add(new Watermark(14600));

	testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 10000));

	expected.add(new StreamRecord<>(new Tuple3<>("key2-1", 10000L, 14600L), 14599));

	testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 14500));
	testHarness.processWatermark(new Watermark(20000));

	expected.add(new StreamRecord<>(new Tuple3<>("key2-1", 10000L, 17500L), 17499));
	expected.add(new Watermark(20000));

	testHarness.processWatermark(new Watermark(100000));
	expected.add(new Watermark(100000));

	ConcurrentLinkedQueue<Object> actual = testHarness.getOutput();

	TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expected, actual, new Tuple3ResultSortComparator());
	testHarness.close();
}
 

@Test
public void testMiniBatchedWatermarkAssignerOperator() throws Exception {
	final MiniBatchedWatermarkAssignerOperator operator = new MiniBatchedWatermarkAssignerOperator(0, 1, 0, -1, 50);

	OneInputStreamOperatorTestHarness<BaseRow, BaseRow> testHarness =
			new OneInputStreamOperatorTestHarness<>(operator);

	testHarness.open();

	testHarness.processElement(new StreamRecord<>(GenericRow.of(1L)));
	testHarness.processElement(new StreamRecord<>(GenericRow.of(2L)));
	testHarness.processWatermark(new Watermark(2)); // this watermark should be ignored
	testHarness.processElement(new StreamRecord<>(GenericRow.of(3L)));
	testHarness.processElement(new StreamRecord<>(GenericRow.of(4L)));
	// this watermark excess expected watermark, should emit a watermark of 49
	testHarness.processElement(new StreamRecord<>(GenericRow.of(50L)));

	ConcurrentLinkedQueue<Object> output = testHarness.getOutput();
	List<Watermark> watermarks = extractWatermarks(output);
	assertEquals(1, watermarks.size());
	assertEquals(new Watermark(49), watermarks.get(0));
	output.clear();

	testHarness.setProcessingTime(1001);
	output = testHarness.getOutput();
	watermarks = extractWatermarks(output);
	assertTrue(watermarks.isEmpty());
	output.clear();

	testHarness.processElement(new StreamRecord<>(GenericRow.of(99L)));
	output = testHarness.getOutput();
	watermarks = extractWatermarks(output);
	assertTrue(watermarks.isEmpty());
	output.clear();

	testHarness.processElement(new StreamRecord<>(GenericRow.of(100L)));
	output = testHarness.getOutput();
	watermarks = extractWatermarks(output);
	assertEquals(1, watermarks.size());
	assertEquals(new Watermark(99), watermarks.get(0));
	output.clear();
}
 

@Test
public void testAndOrSubtypeConditionsAfterMigration() throws Exception {

	KeySelector<Event, Integer> keySelector = new KeySelector<Event, Integer>() {
		private static final long serialVersionUID = -4873366487571254798L;

		@Override
		public Integer getKey(Event value) throws Exception {
			return value.getId();
		}
	};

	final Event startEvent1 = new SubEvent(42, "start", 1.0, 6.0);

	OneInputStreamOperatorTestHarness<Event, Map<String, List<Event>>> harness =
		new KeyedOneInputStreamOperatorTestHarness<>(
			getKeyedCepOpearator(false, new NFAComplexConditionsFactory()),
			keySelector,
			BasicTypeInfo.INT_TYPE_INFO);

	try {
		harness.setup();

		harness.initializeState(
			OperatorSnapshotUtil.getResourceFilename(
				"cep-migration-conditions-flink" + migrateVersion + "-snapshot"));

		harness.open();

		final Event endEvent = new SubEvent(42, "end", 1.0, 2.0);
		harness.processElement(new StreamRecord<>(endEvent, 9));
		harness.processWatermark(new Watermark(20));

		ConcurrentLinkedQueue<Object> result = harness.getOutput();

		// watermark and the result
		assertEquals(2, result.size());

		Object resultObject = result.poll();
		assertTrue(resultObject instanceof StreamRecord);
		StreamRecord<?> resultRecord = (StreamRecord<?>) resultObject;
		assertTrue(resultRecord.getValue() instanceof Map);

		@SuppressWarnings("unchecked")
		Map<String, List<Event>> patternMap =
			(Map<String, List<Event>>) resultRecord.getValue();

		assertEquals(startEvent1, patternMap.get("start").get(0));
		assertEquals(endEvent, patternMap.get("start").get(1));
	} finally {
		harness.close();
	}
}
 

@Test
public void testNotSideOutputDueToLatenessSessionWithHugeLateness() throws Exception {
	final int gapSize = 3;
	final long lateness = 10000;

	ReducingStateDescriptor<Tuple2<String, Integer>> stateDesc = new ReducingStateDescriptor<>("window-contents",
		new SumReducer(),
		STRING_INT_TUPLE.createSerializer(new ExecutionConfig()));

	WindowOperator<String, Tuple2<String, Integer>, Tuple2<String, Integer>, Tuple3<String, Long, Long>, TimeWindow> operator =
		new WindowOperator<>(
			EventTimeSessionWindows.withGap(Time.seconds(gapSize)),
			new TimeWindow.Serializer(),
			new TupleKeySelector(),
			BasicTypeInfo.STRING_TYPE_INFO.createSerializer(new ExecutionConfig()),
			stateDesc,
			new InternalSingleValueWindowFunction<>(new ReducedSessionWindowFunction()),
			EventTimeTrigger.create(),
			lateness,
			lateOutputTag /* late data output tag */);

	OneInputStreamOperatorTestHarness<Tuple2<String, Integer>, Tuple3<String, Long, Long>> testHarness =
		createTestHarness(operator);

	testHarness.open();

	ConcurrentLinkedQueue<Object> expected = new ConcurrentLinkedQueue<>();

	testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 1000));
	testHarness.processWatermark(new Watermark(1999));

	expected.add(new Watermark(1999));

	testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 2000));
	testHarness.processWatermark(new Watermark(4998));

	expected.add(new Watermark(4998));

	// this will not be sideoutput because the session we're adding two has maxTimestamp
	// after the current watermark
	testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 4500));

	// new session
	testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 8500));
	testHarness.processWatermark(new Watermark(7400));

	expected.add(new Watermark(7400));

	// this will merge the two sessions into one
	testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 7000));
	testHarness.processWatermark(new Watermark(11501));

	expected.add(new StreamRecord<>(new Tuple3<>("key2-5", 1000L, 11500L), 11499));
	expected.add(new Watermark(11501));

	// new session
	testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 11600));
	testHarness.processWatermark(new Watermark(14600));

	expected.add(new StreamRecord<>(new Tuple3<>("key2-1", 11600L, 14600L), 14599));
	expected.add(new Watermark(14600));

	testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 10000));

	// the maxTimestamp of the merged session is already late,
	// so we get an immediate firing
	expected.add(new StreamRecord<>(new Tuple3<>("key2-7", 1000L, 14600L), 14599));

	ConcurrentLinkedQueue<Object> actual = testHarness.getOutput();
	ConcurrentLinkedQueue<StreamRecord<Tuple2<String, Integer>>> sideActual = testHarness.getSideOutput(lateOutputTag);
	TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expected, actual, new Tuple3ResultSortComparator());
	assertEquals(null, sideActual);

	testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 14500));
	testHarness.processWatermark(new Watermark(20000));

	expected.add(new StreamRecord<>(new Tuple3<>("key2-8", 1000L, 17500L), 17499));
	expected.add(new Watermark(20000));

	testHarness.processWatermark(new Watermark(100000));
	expected.add(new Watermark(100000));

	actual = testHarness.getOutput();
	sideActual = testHarness.getSideOutput(lateOutputTag);

	TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expected, actual, new Tuple3ResultSortComparator());
	assertEquals(null, sideActual);

	testHarness.close();
}
 

/**
 * Tests that the AysncWaitOperator can restart if checkpointed queue was full.
 *
 * <p>See FLINK-7949
 */
@Test(timeout = 10000)
public void testRestartWithFullQueue() throws Exception {
	int capacity = 10;

	// 1. create the snapshot which contains capacity + 1 elements
	final CompletableFuture<Void> trigger = new CompletableFuture<>();
	final ControllableAsyncFunction<Integer> controllableAsyncFunction = new ControllableAsyncFunction<>(trigger);

	final OneInputStreamOperatorTestHarness<Integer, Integer> snapshotHarness = new OneInputStreamOperatorTestHarness<>(
		new AsyncWaitOperator<>(
			controllableAsyncFunction, // the NoOpAsyncFunction is like a blocking function
			1000L,
			capacity,
			AsyncDataStream.OutputMode.ORDERED),
		IntSerializer.INSTANCE);

	snapshotHarness.open();

	final OperatorSubtaskState snapshot;

	final ArrayList<Integer> expectedOutput = new ArrayList<>(capacity + 1);

	try {
		synchronized (snapshotHarness.getCheckpointLock()) {
			for (int i = 0; i < capacity; i++) {
				snapshotHarness.processElement(i, 0L);
				expectedOutput.add(i);
			}
		}

		expectedOutput.add(capacity);

		final OneShotLatch lastElement = new OneShotLatch();

		final CheckedThread lastElementWriter = new CheckedThread() {
			@Override
			public void go() throws Exception {
				synchronized (snapshotHarness.getCheckpointLock()) {
					lastElement.trigger();
					snapshotHarness.processElement(capacity, 0L);
				}
			}
		};

		lastElementWriter.start();

		lastElement.await();

		synchronized (snapshotHarness.getCheckpointLock()) {
			// execute the snapshot within the checkpoint lock, because then it is guaranteed
			// that the lastElementWriter has written the exceeding element
			snapshot = snapshotHarness.snapshot(0L, 0L);
		}

		// trigger the computation to make the close call finish
		trigger.complete(null);
	} finally {
		synchronized (snapshotHarness.getCheckpointLock()) {
			snapshotHarness.close();
		}
	}

	// 2. restore the snapshot and check that we complete
	final OneInputStreamOperatorTestHarness<Integer, Integer> recoverHarness = new OneInputStreamOperatorTestHarness<>(
		new AsyncWaitOperator<>(
			new ControllableAsyncFunction<>(CompletableFuture.completedFuture(null)),
			1000L,
			capacity,
			AsyncDataStream.OutputMode.ORDERED),
		IntSerializer.INSTANCE);

	recoverHarness.initializeState(snapshot);

	synchronized (recoverHarness.getCheckpointLock()) {
		recoverHarness.open();
	}

	synchronized (recoverHarness.getCheckpointLock()) {
		recoverHarness.close();
	}

	final ConcurrentLinkedQueue<Object> output = recoverHarness.getOutput();

	assertThat(output.size(), Matchers.equalTo(capacity + 1));

	final ArrayList<Integer> outputElements = new ArrayList<>(capacity + 1);

	for (int i = 0; i < capacity + 1; i++) {
		StreamRecord<Integer> streamRecord = ((StreamRecord<Integer>) output.poll());
		outputElements.add(streamRecord.getValue());
	}

	assertThat(outputElements, Matchers.equalTo(expectedOutput));
}
 

@Test
public void testAndOrSubtypeConditionsAfterMigration() throws Exception {

	KeySelector<Event, Integer> keySelector = new KeySelector<Event, Integer>() {
		private static final long serialVersionUID = -4873366487571254798L;

		@Override
		public Integer getKey(Event value) throws Exception {
			return value.getId();
		}
	};

	final Event startEvent1 = new SubEvent(42, "start", 1.0, 6.0);

	OneInputStreamOperatorTestHarness<Event, Map<String, List<Event>>> harness =
		new KeyedOneInputStreamOperatorTestHarness<>(
			getKeyedCepOpearator(false, new NFAComplexConditionsFactory()),
			keySelector,
			BasicTypeInfo.INT_TYPE_INFO);

	try {
		harness.setup();

		harness.initializeState(
			OperatorSnapshotUtil.getResourceFilename(
				"cep-migration-conditions-flink" + migrateVersion + "-snapshot"));

		harness.open();

		final Event endEvent = new SubEvent(42, "end", 1.0, 2.0);
		harness.processElement(new StreamRecord<>(endEvent, 9));
		harness.processWatermark(new Watermark(20));

		ConcurrentLinkedQueue<Object> result = harness.getOutput();

		// watermark and the result
		assertEquals(2, result.size());

		Object resultObject = result.poll();
		assertTrue(resultObject instanceof StreamRecord);
		StreamRecord<?> resultRecord = (StreamRecord<?>) resultObject;
		assertTrue(resultRecord.getValue() instanceof Map);

		@SuppressWarnings("unchecked")
		Map<String, List<Event>> patternMap =
			(Map<String, List<Event>>) resultRecord.getValue();

		assertEquals(startEvent1, patternMap.get("start").get(0));
		assertEquals(endEvent, patternMap.get("start").get(1));
	} finally {
		harness.close();
	}
}
 

@Test
public void testKeyedCEPOperatorNFAUpdateTimesWithRocksDB() throws Exception {

	String rocksDbPath = tempFolder.newFolder().getAbsolutePath();
	RocksDBStateBackend rocksDBStateBackend = new RocksDBStateBackend(new MemoryStateBackend());
	rocksDBStateBackend.setDbStoragePath(rocksDbPath);

	CepOperator<Event, Integer, Map<String, List<Event>>> operator = CepOperatorTestUtilities.getKeyedCepOpearator(
		true,
		new SimpleNFAFactory());
	OneInputStreamOperatorTestHarness<Event, Map<String, List<Event>>> harness = CepOperatorTestUtilities.getCepTestHarness(
		operator);

	try {
		harness.setStateBackend(rocksDBStateBackend);

		harness.open();

		final ValueState nfaOperatorState = (ValueState) Whitebox.<ValueState>getInternalState(operator, "computationStates");
		final ValueState nfaOperatorStateSpy = Mockito.spy(nfaOperatorState);
		Whitebox.setInternalState(operator, "computationStates", nfaOperatorStateSpy);

		Event startEvent = new Event(42, "c", 1.0);
		SubEvent middleEvent = new SubEvent(42, "a", 1.0, 10.0);
		Event endEvent = new Event(42, "b", 1.0);

		harness.processElement(new StreamRecord<>(startEvent, 1L));
		harness.processElement(new StreamRecord<>(new Event(42, "d", 1.0), 4L));
		harness.processElement(new StreamRecord<Event>(middleEvent, 4L));
		harness.processElement(new StreamRecord<>(endEvent, 4L));

		// verify the number of invocations NFA is updated
		Mockito.verify(nfaOperatorStateSpy, Mockito.times(3)).update(Mockito.any());

		// get and verify the output
		Queue<Object> result = harness.getOutput();

		assertEquals(1, result.size());

		verifyPattern(result.poll(), startEvent, middleEvent, endEvent);
	} finally {
		harness.close();
	}
}
 

@Test
public void testProcessingTimeSessionWindows() throws Throwable {
	final int windowGap = 3;

	ReducingStateDescriptor<Tuple2<String, Integer>> stateDesc = new ReducingStateDescriptor<>("window-contents",
			new SumReducer(),
			STRING_INT_TUPLE.createSerializer(new ExecutionConfig()));

	WindowOperator<String, Tuple2<String, Integer>, Tuple2<String, Integer>, Tuple2<String, Integer>, TimeWindow> operator = new WindowOperator<>(
			ProcessingTimeSessionWindows.withGap(Time.of(windowGap, TimeUnit.SECONDS)),
			new TimeWindow.Serializer(),
			new TupleKeySelector(),
			BasicTypeInfo.STRING_TYPE_INFO.createSerializer(new ExecutionConfig()),
			stateDesc,
			new InternalSingleValueWindowFunction<>(new PassThroughWindowFunction<String, TimeWindow, Tuple2<String, Integer>>()),
			ProcessingTimeTrigger.create(),
			0,
			null /* late data output tag */);

	OneInputStreamOperatorTestHarness<Tuple2<String, Integer>, Tuple2<String, Integer>> testHarness =
			createTestHarness(operator);

	ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>();

	testHarness.open();

	// timestamp is ignored in processing time
	testHarness.setProcessingTime(3);
	testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 1)); //Long.MAX_VALUE));

	testHarness.setProcessingTime(1000);
	testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 1002)); //Long.MAX_VALUE));

	testHarness.setProcessingTime(5000);

	expectedOutput.add(new StreamRecord<>(new Tuple2<>("key2", 2), 3999));

	TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expectedOutput, testHarness.getOutput(), new Tuple2ResultSortComparator());

	testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 5000));
	testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 5000));
	testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 1), 5000));
	testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 1), 5000));
	testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 1), 5000));

	testHarness.setProcessingTime(10000);

	expectedOutput.add(new StreamRecord<>(new Tuple2<>("key2", 2), 7999));
	expectedOutput.add(new StreamRecord<>(new Tuple2<>("key1", 3), 7999));

	TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expectedOutput, testHarness.getOutput(), new Tuple2ResultSortComparator());

	assertEquals(expectedOutput.size(), testHarness.getOutput().size());
	for (Object elem : testHarness.getOutput()) {
		if (elem instanceof StreamRecord) {
			StreamRecord<Tuple2<String, Integer>> el = (StreamRecord<Tuple2<String, Integer>>) elem;
			assertTrue(expectedOutput.contains(el));
		}
	}
	testHarness.close();
}
 

@Test
public void testNotSideOutputDueToLatenessSessionWithHugeLatenessPurgingTrigger() throws Exception {

	final int gapSize = 3;
	final long lateness = 10000;

	ReducingStateDescriptor<Tuple2<String, Integer>> stateDesc = new ReducingStateDescriptor<>("window-contents",
		new SumReducer(),
		STRING_INT_TUPLE.createSerializer(new ExecutionConfig()));

	WindowOperator<String, Tuple2<String, Integer>, Tuple2<String, Integer>, Tuple3<String, Long, Long>, TimeWindow> operator =
		new WindowOperator<>(
			EventTimeSessionWindows.withGap(Time.seconds(gapSize)),
			new TimeWindow.Serializer(),
			new TupleKeySelector(),
			BasicTypeInfo.STRING_TYPE_INFO.createSerializer(new ExecutionConfig()),
			stateDesc,
			new InternalSingleValueWindowFunction<>(new ReducedSessionWindowFunction()),
			PurgingTrigger.of(EventTimeTrigger.create()),
			lateness,
			lateOutputTag /* late data output tag */);

	OneInputStreamOperatorTestHarness<Tuple2<String, Integer>, Tuple3<String, Long, Long>> testHarness =
		createTestHarness(operator);

	testHarness.open();

	ConcurrentLinkedQueue<Object> expected = new ConcurrentLinkedQueue<>();

	testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 1000));
	testHarness.processWatermark(new Watermark(1999));

	expected.add(new Watermark(1999));

	testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 2000));
	testHarness.processWatermark(new Watermark(4998));

	expected.add(new Watermark(4998));

	// this will not be sideoutput because the session we're adding two has maxTimestamp
	// after the current watermark
	testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 4500));

	// new session
	testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 8500));
	testHarness.processWatermark(new Watermark(7400));

	expected.add(new Watermark(7400));

	// this will merge the two sessions into one
	testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 7000));
	testHarness.processWatermark(new Watermark(11501));

	expected.add(new StreamRecord<>(new Tuple3<>("key2-5", 1000L, 11500L), 11499));
	expected.add(new Watermark(11501));

	// new session
	testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 11600));
	testHarness.processWatermark(new Watermark(14600));

	expected.add(new StreamRecord<>(new Tuple3<>("key2-1", 11600L, 14600L), 14599));
	expected.add(new Watermark(14600));

	testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 10000));

	expected.add(new StreamRecord<>(new Tuple3<>("key2-1", 1000L, 14600L), 14599));

	ConcurrentLinkedQueue<Object> actual = testHarness.getOutput();
	ConcurrentLinkedQueue<StreamRecord<Tuple2<String, Integer>>> sideActual = testHarness.getSideOutput(lateOutputTag);
	TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expected, actual, new Tuple3ResultSortComparator());
	assertEquals(null, sideActual);

	testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 14500));
	testHarness.processWatermark(new Watermark(20000));

	expected.add(new StreamRecord<>(new Tuple3<>("key2-1", 1000L, 17500L), 17499));
	expected.add(new Watermark(20000));

	testHarness.processWatermark(new Watermark(100000));

	expected.add(new Watermark(100000));

	actual = testHarness.getOutput();
	sideActual = testHarness.getSideOutput(lateOutputTag);
	TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expected, actual, new Tuple3ResultSortComparator());
	assertEquals(null, sideActual);

	testHarness.close();
}
 

@Test
public void testKeyedCEPOperatorNFAUpdateTimesWithRocksDB() throws Exception {

	String rocksDbPath = tempFolder.newFolder().getAbsolutePath();
	RocksDBStateBackend rocksDBStateBackend = new RocksDBStateBackend(new MemoryStateBackend());
	rocksDBStateBackend.setDbStoragePath(rocksDbPath);

	CepOperator<Event, Integer, Map<String, List<Event>>> operator = CepOperatorTestUtilities.getKeyedCepOpearator(
		true,
		new SimpleNFAFactory());
	OneInputStreamOperatorTestHarness<Event, Map<String, List<Event>>> harness = CepOperatorTestUtilities.getCepTestHarness(
		operator);

	try {
		harness.setStateBackend(rocksDBStateBackend);

		harness.open();

		final ValueState nfaOperatorState = (ValueState) Whitebox.<ValueState>getInternalState(operator, "computationStates");
		final ValueState nfaOperatorStateSpy = Mockito.spy(nfaOperatorState);
		Whitebox.setInternalState(operator, "computationStates", nfaOperatorStateSpy);

		Event startEvent = new Event(42, "c", 1.0);
		SubEvent middleEvent = new SubEvent(42, "a", 1.0, 10.0);
		Event endEvent = new Event(42, "b", 1.0);

		harness.processElement(new StreamRecord<>(startEvent, 1L));
		harness.processElement(new StreamRecord<>(new Event(42, "d", 1.0), 4L));
		harness.processElement(new StreamRecord<Event>(middleEvent, 4L));
		harness.processElement(new StreamRecord<>(endEvent, 4L));

		// verify the number of invocations NFA is updated
		Mockito.verify(nfaOperatorStateSpy, Mockito.times(3)).update(Mockito.any());

		// get and verify the output
		Queue<Object> result = harness.getOutput();

		assertEquals(1, result.size());

		verifyPattern(result.poll(), startEvent, middleEvent, endEvent);
	} finally {
		harness.close();
	}
}