Java Code Examples for org.apache.flink.streaming.util.KeyedOneInputStreamOperatorTestHarness#processWatermark()

@Test
public void testProcessingElementsWithinAllowedLateness() throws Exception {
	WindowAssigner<Integer, TimeWindow> mockAssigner = mockTimeWindowAssigner();
	Trigger<Integer, TimeWindow> mockTrigger = mockTrigger();
	InternalWindowFunction<Iterable<Integer>, Void, Integer, TimeWindow> mockWindowFunction = mockWindowFunction();

	KeyedOneInputStreamOperatorTestHarness<Integer, Integer, Void> testHarness =
			createWindowOperator(mockAssigner, mockTrigger, 20L, mockWindowFunction);

	testHarness.open();

	when(mockAssigner.assignWindows(anyInt(), anyLong(), anyAssignerContext()))
			.thenReturn(Arrays.asList(new TimeWindow(0, 2)));

	assertEquals(0, testHarness.getOutput().size());
	assertEquals(0, testHarness.numKeyedStateEntries());

	shouldFireOnElement(mockTrigger);

	// 20 is just at the limit, window.maxTime() is 1 and allowed lateness is 20
	testHarness.processWatermark(new Watermark(20));

	testHarness.processElement(new StreamRecord<>(0, 0L));

	verify(mockWindowFunction, times(1)).process(eq(0), eq(new TimeWindow(0, 2)), anyInternalWindowContext(), intIterable(0), WindowOperatorContractTest.<Void>anyCollector());

	// clear is only called at cleanup time/GC time
	verify(mockTrigger, never()).clear(anyTimeWindow(), anyTriggerContext());

	// FIRE should not purge contents
	assertEquals(1, testHarness.numKeyedStateEntries()); // window contents plus trigger state
	assertEquals(1, testHarness.numEventTimeTimers()); // just the GC timer
}
 

/**
 * Ensure that we get some output from the given operator when pushing in an element and
 * setting watermark and processing time to {@code Long.MAX_VALUE}.
 */
private static <K, IN, OUT> void processElementAndEnsureOutput(
		OneInputStreamOperator<IN, OUT> operator,
		KeySelector<IN, K> keySelector,
		TypeInformation<K> keyType,
		IN element) throws Exception {

	KeyedOneInputStreamOperatorTestHarness<K, IN, OUT> testHarness =
			new KeyedOneInputStreamOperatorTestHarness<>(
					operator,
					keySelector,
					keyType);

	testHarness.open();

	testHarness.setProcessingTime(0);
	testHarness.processWatermark(Long.MIN_VALUE);

	testHarness.processElement(new StreamRecord<>(element, 0));

	// provoke any processing-time/event-time triggers
	testHarness.setProcessingTime(Long.MAX_VALUE);
	testHarness.processWatermark(Long.MAX_VALUE);

	// we at least get the two watermarks and should also see an output element
	assertTrue(testHarness.getOutput().size() >= 3);

	testHarness.close();
}
 

/**
 * Ensure that we get some output from the given operator when pushing in an element and
 * setting watermark and processing time to {@code Long.MAX_VALUE}.
 */
private static <K, IN, OUT> void processElementAndEnsureOutput(
		OneInputStreamOperator<IN, OUT> operator,
		KeySelector<IN, K> keySelector,
		TypeInformation<K> keyType,
		IN element) throws Exception {

	KeyedOneInputStreamOperatorTestHarness<K, IN, OUT> testHarness =
			new KeyedOneInputStreamOperatorTestHarness<>(
					operator,
					keySelector,
					keyType);

	if (operator instanceof OutputTypeConfigurable) {
		// use a dummy type since window functions just need the ExecutionConfig
		// this is also only needed for Fold, which we're getting rid off soon.
		((OutputTypeConfigurable) operator).setOutputType(BasicTypeInfo.STRING_TYPE_INFO, new ExecutionConfig());
	}

	testHarness.open();

	testHarness.setProcessingTime(0);
	testHarness.processWatermark(Long.MIN_VALUE);

	testHarness.processElement(new StreamRecord<>(element, 0));

	// provoke any processing-time/event-time triggers
	testHarness.setProcessingTime(Long.MAX_VALUE);
	testHarness.processWatermark(Long.MAX_VALUE);

	// we at least get the two watermarks and should also see an output element
	assertTrue(testHarness.getOutput().size() >= 3);

	testHarness.close();
}
 

@Test
public void testLateWindowDropping() throws Exception {
	WindowAssigner<Integer, TimeWindow> mockAssigner = mockTimeWindowAssigner();
	Trigger<Integer, TimeWindow> mockTrigger = mockTrigger();
	InternalWindowFunction<Iterable<Integer>, Void, Integer, TimeWindow> mockWindowFunction = mockWindowFunction();

	KeyedOneInputStreamOperatorTestHarness<Integer, Integer, Void> testHarness =
			createWindowOperator(mockAssigner, mockTrigger, 20L, mockWindowFunction);

	testHarness.open();

	when(mockAssigner.assignWindows(anyInt(), anyLong(), anyAssignerContext()))
			.thenReturn(Arrays.asList(new TimeWindow(0, 2)));

	assertEquals(0, testHarness.getOutput().size());
	assertEquals(0, testHarness.numKeyedStateEntries());

	shouldFireOnElement(mockTrigger);

	// window.maxTime() == 1 plus 20L of allowed lateness
	testHarness.processWatermark(new Watermark(21));

	testHarness.processElement(new StreamRecord<>(0, 0L));

	// there should be nothing
	assertEquals(0, testHarness.numKeyedStateEntries());
	assertEquals(0, testHarness.numEventTimeTimers());
	assertEquals(0, testHarness.numProcessingTimeTimers());

	// there should be two elements now
	assertEquals(0, testHarness.extractOutputStreamRecords().size());
}
 

@Test
public void testProcessingElementsWithinAllowedLateness() throws Exception {
	WindowAssigner<Integer, TimeWindow> mockAssigner = mockTimeWindowAssigner();
	Trigger<Integer, TimeWindow> mockTrigger = mockTrigger();
	InternalWindowFunction<Iterable<Integer>, Void, Integer, TimeWindow> mockWindowFunction = mockWindowFunction();

	KeyedOneInputStreamOperatorTestHarness<Integer, Integer, Void> testHarness =
			createWindowOperator(mockAssigner, mockTrigger, 20L, mockWindowFunction);

	testHarness.open();

	when(mockAssigner.assignWindows(anyInt(), anyLong(), anyAssignerContext()))
			.thenReturn(Arrays.asList(new TimeWindow(0, 2)));

	assertEquals(0, testHarness.getOutput().size());
	assertEquals(0, testHarness.numKeyedStateEntries());

	shouldFireOnElement(mockTrigger);

	// 20 is just at the limit, window.maxTime() is 1 and allowed lateness is 20
	testHarness.processWatermark(new Watermark(20));

	testHarness.processElement(new StreamRecord<>(0, 0L));

	verify(mockWindowFunction, times(1)).process(eq(0), eq(new TimeWindow(0, 2)), anyInternalWindowContext(), intIterable(0), WindowOperatorContractTest.<Void>anyCollector());

	// clear is only called at cleanup time/GC time
	verify(mockTrigger, never()).clear(anyTimeWindow(), anyTriggerContext());

	// FIRE should not purge contents
	assertEquals(1, testHarness.numKeyedStateEntries()); // window contents plus trigger state
	assertEquals(1, testHarness.numEventTimeTimers()); // just the GC timer
}
 

/**
 * Ensure that we get some output from the given operator when pushing in an element and
 * setting watermark and processing time to {@code Long.MAX_VALUE}.
 */
private static <K, IN, OUT> void processElementAndEnsureOutput(
		OneInputStreamOperator<IN, OUT> operator,
		KeySelector<IN, K> keySelector,
		TypeInformation<K> keyType,
		IN element) throws Exception {

	KeyedOneInputStreamOperatorTestHarness<K, IN, OUT> testHarness =
			new KeyedOneInputStreamOperatorTestHarness<>(
					operator,
					keySelector,
					keyType);

	testHarness.open();

	testHarness.setProcessingTime(0);
	testHarness.processWatermark(Long.MIN_VALUE);

	testHarness.processElement(new StreamRecord<>(element, 0));

	// provoke any processing-time/event-time triggers
	testHarness.setProcessingTime(Long.MAX_VALUE);
	testHarness.processWatermark(Long.MAX_VALUE);

	// we at least get the two watermarks and should also see an output element
	assertTrue(testHarness.getOutput().size() >= 3);

	testHarness.close();
}
 

/**
 * Ensure that we get some output from the given operator when pushing in an element and
 * setting watermark and processing time to {@code Long.MAX_VALUE}.
 */
private static <K, IN, OUT> void processElementAndEnsureOutput(
		OneInputStreamOperator<IN, OUT> operator,
		KeySelector<IN, K> keySelector,
		TypeInformation<K> keyType,
		IN element) throws Exception {

	KeyedOneInputStreamOperatorTestHarness<K, IN, OUT> testHarness =
			new KeyedOneInputStreamOperatorTestHarness<>(
					operator,
					keySelector,
					keyType);

	if (operator instanceof OutputTypeConfigurable) {
		// use a dummy type since window functions just need the ExecutionConfig
		// this is also only needed for Fold, which we're getting rid off soon.
		((OutputTypeConfigurable) operator).setOutputType(BasicTypeInfo.STRING_TYPE_INFO, new ExecutionConfig());
	}

	testHarness.open();

	testHarness.setProcessingTime(0);
	testHarness.processWatermark(Long.MIN_VALUE);

	testHarness.processElement(new StreamRecord<>(element, 0));

	// provoke any processing-time/event-time triggers
	testHarness.setProcessingTime(Long.MAX_VALUE);
	testHarness.processWatermark(Long.MAX_VALUE);

	// we at least get the two watermarks and should also see an output element
	assertTrue(testHarness.getOutput().size() >= 3);

	testHarness.close();
}
 

@Test
public void testLateWindowDropping() throws Exception {
	WindowAssigner<Integer, TimeWindow> mockAssigner = mockTimeWindowAssigner();
	Trigger<Integer, TimeWindow> mockTrigger = mockTrigger();
	InternalWindowFunction<Iterable<Integer>, Void, Integer, TimeWindow> mockWindowFunction = mockWindowFunction();

	KeyedOneInputStreamOperatorTestHarness<Integer, Integer, Void> testHarness =
			createWindowOperator(mockAssigner, mockTrigger, 20L, mockWindowFunction);

	testHarness.open();

	when(mockAssigner.assignWindows(anyInt(), anyLong(), anyAssignerContext()))
			.thenReturn(Arrays.asList(new TimeWindow(0, 2)));

	assertEquals(0, testHarness.getOutput().size());
	assertEquals(0, testHarness.numKeyedStateEntries());

	shouldFireOnElement(mockTrigger);

	// window.maxTime() == 1 plus 20L of allowed lateness
	testHarness.processWatermark(new Watermark(21));

	testHarness.processElement(new StreamRecord<>(0, 0L));

	// there should be nothing
	assertEquals(0, testHarness.numKeyedStateEntries());
	assertEquals(0, testHarness.numEventTimeTimers());
	assertEquals(0, testHarness.numProcessingTimeTimers());

	// there should be two elements now
	assertEquals(0, testHarness.extractOutputStreamRecords().size());
}
 

/**
 * Verify that timers for the different time domains don't clash.
 */
@Test
public void testProcessingTimeAndEventTimeDontInterfere() throws Exception {
	TestOperator testOperator = new TestOperator();

	KeyedOneInputStreamOperatorTestHarness<Integer, Tuple2<Integer, String>, String> testHarness =
			new KeyedOneInputStreamOperatorTestHarness<>(testOperator, new TestKeySelector(), BasicTypeInfo.INT_TYPE_INFO);

	testHarness.open();

	testHarness.setProcessingTime(0L);
	testHarness.processWatermark(0L);

	testHarness.processElement(new Tuple2<>(0, "SET_PROC_TIME_TIMER:10"), 0);
	testHarness.processElement(new Tuple2<>(0, "SET_EVENT_TIME_TIMER:20"), 0);

	testHarness.processElement(new Tuple2<>(0, "SET_STATE:HELLO"), 0);

	testHarness.processWatermark(20L);

	assertThat(
			extractResult(testHarness),
			contains("ON_EVENT_TIME:HELLO"));

	testHarness.setProcessingTime(10L);

	assertThat(
			extractResult(testHarness),
			contains("ON_PROC_TIME:HELLO"));
}
 

/**
 * Ensure that we get some output from the given operator when pushing in an element and
 * setting watermark and processing time to {@code Long.MAX_VALUE}.
 */
private static <K, IN, OUT> void processElementAndEnsureOutput(
		OneInputStreamOperator<IN, OUT> operator,
		KeySelector<IN, K> keySelector,
		TypeInformation<K> keyType,
		IN element) throws Exception {

	KeyedOneInputStreamOperatorTestHarness<K, IN, OUT> testHarness =
			new KeyedOneInputStreamOperatorTestHarness<>(
					operator,
					keySelector,
					keyType);

	testHarness.open();

	testHarness.setProcessingTime(0);
	testHarness.processWatermark(Long.MIN_VALUE);

	testHarness.processElement(new StreamRecord<>(element, 0));

	// provoke any processing-time/event-time triggers
	testHarness.setProcessingTime(Long.MAX_VALUE);
	testHarness.processWatermark(Long.MAX_VALUE);

	// we at least get the two watermarks and should also see an output element
	assertTrue(testHarness.getOutput().size() >= 3);

	testHarness.close();
}
 

/**
 * Ensure that we get some output from the given operator when pushing in an element and
 * setting watermark and processing time to {@code Long.MAX_VALUE}.
 */
private static <K, IN, OUT> void processElementAndEnsureOutput(
		OneInputStreamOperator<IN, OUT> operator,
		KeySelector<IN, K> keySelector,
		TypeInformation<K> keyType,
		IN element) throws Exception {

	KeyedOneInputStreamOperatorTestHarness<K, IN, OUT> testHarness =
			new KeyedOneInputStreamOperatorTestHarness<>(
					operator,
					keySelector,
					keyType);

	if (operator instanceof OutputTypeConfigurable) {
		// use a dummy type since window functions just need the ExecutionConfig
		// this is also only needed for Fold, which we're getting rid off soon.
		((OutputTypeConfigurable) operator).setOutputType(BasicTypeInfo.STRING_TYPE_INFO, new ExecutionConfig());
	}

	testHarness.open();

	testHarness.setProcessingTime(0);
	testHarness.processWatermark(Long.MIN_VALUE);

	testHarness.processElement(new StreamRecord<>(element, 0));

	// provoke any processing-time/event-time triggers
	testHarness.setProcessingTime(Long.MAX_VALUE);
	testHarness.processWatermark(Long.MAX_VALUE);

	// we at least get the two watermarks and should also see an output element
	assertTrue(testHarness.getOutput().size() >= 3);

	testHarness.close();
}
 

@Test
public void testLateWindowDropping() throws Exception {
	WindowAssigner<Integer, TimeWindow> mockAssigner = mockTimeWindowAssigner();
	Trigger<Integer, TimeWindow> mockTrigger = mockTrigger();
	InternalWindowFunction<Iterable<Integer>, Void, Integer, TimeWindow> mockWindowFunction = mockWindowFunction();

	KeyedOneInputStreamOperatorTestHarness<Integer, Integer, Void> testHarness =
			createWindowOperator(mockAssigner, mockTrigger, 20L, mockWindowFunction);

	testHarness.open();

	when(mockAssigner.assignWindows(anyInt(), anyLong(), anyAssignerContext()))
			.thenReturn(Arrays.asList(new TimeWindow(0, 2)));

	assertEquals(0, testHarness.getOutput().size());
	assertEquals(0, testHarness.numKeyedStateEntries());

	shouldFireOnElement(mockTrigger);

	// window.maxTime() == 1 plus 20L of allowed lateness
	testHarness.processWatermark(new Watermark(21));

	testHarness.processElement(new StreamRecord<>(0, 0L));

	// there should be nothing
	assertEquals(0, testHarness.numKeyedStateEntries());
	assertEquals(0, testHarness.numEventTimeTimers());
	assertEquals(0, testHarness.numProcessingTimeTimers());

	// there should be two elements now
	assertEquals(0, testHarness.extractOutputStreamRecords().size());
}
 

@Test
public void testProcessingElementsWithinAllowedLateness() throws Exception {
	WindowAssigner<Integer, TimeWindow> mockAssigner = mockTimeWindowAssigner();
	Trigger<Integer, TimeWindow> mockTrigger = mockTrigger();
	InternalWindowFunction<Iterable<Integer>, Void, Integer, TimeWindow> mockWindowFunction = mockWindowFunction();

	KeyedOneInputStreamOperatorTestHarness<Integer, Integer, Void> testHarness =
			createWindowOperator(mockAssigner, mockTrigger, 20L, mockWindowFunction);

	testHarness.open();

	when(mockAssigner.assignWindows(anyInt(), anyLong(), anyAssignerContext()))
			.thenReturn(Arrays.asList(new TimeWindow(0, 2)));

	assertEquals(0, testHarness.getOutput().size());
	assertEquals(0, testHarness.numKeyedStateEntries());

	shouldFireOnElement(mockTrigger);

	// 20 is just at the limit, window.maxTime() is 1 and allowed lateness is 20
	testHarness.processWatermark(new Watermark(20));

	testHarness.processElement(new StreamRecord<>(0, 0L));

	verify(mockWindowFunction, times(1)).process(eq(0), eq(new TimeWindow(0, 2)), anyInternalWindowContext(), intIterable(0), WindowOperatorContractTest.<Void>anyCollector());

	// clear is only called at cleanup time/GC time
	verify(mockTrigger, never()).clear(anyTimeWindow(), anyTriggerContext());

	// FIRE should not purge contents
	assertEquals(1, testHarness.numKeyedStateEntries()); // window contents plus trigger state
	assertEquals(1, testHarness.numEventTimeTimers()); // just the GC timer
}
 

/**
 * Verify that timers for the different time domains don't clash.
 */
@Test
public void testProcessingTimeAndEventTimeDontInterfere() throws Exception {
	TestOperator testOperator = new TestOperator();

	KeyedOneInputStreamOperatorTestHarness<Integer, Tuple2<Integer, String>, String> testHarness =
			new KeyedOneInputStreamOperatorTestHarness<>(testOperator, new TestKeySelector(), BasicTypeInfo.INT_TYPE_INFO);

	testHarness.open();

	testHarness.setProcessingTime(0L);
	testHarness.processWatermark(0L);

	testHarness.processElement(new Tuple2<>(0, "SET_PROC_TIME_TIMER:10"), 0);
	testHarness.processElement(new Tuple2<>(0, "SET_EVENT_TIME_TIMER:20"), 0);

	testHarness.processElement(new Tuple2<>(0, "SET_STATE:HELLO"), 0);

	testHarness.processWatermark(20L);

	assertThat(
			extractResult(testHarness),
			contains("ON_EVENT_TIME:HELLO"));

	testHarness.setProcessingTime(10L);

	assertThat(
			extractResult(testHarness),
			contains("ON_PROC_TIME:HELLO"));
}
 

/**
 * Tests that the internal time of a CEP operator advances only given watermarks. See FLINK-5033
 */
@Test
public void testKeyedAdvancingTimeWithoutElements() throws Exception {
	final Event startEvent = new Event(42, "start", 1.0);
	final long watermarkTimestamp1 = 5L;
	final long watermarkTimestamp2 = 13L;

	final Map<String, List<Event>> expectedSequence = new HashMap<>(2);
	expectedSequence.put("start", Collections.<Event>singletonList(startEvent));

	final OutputTag<Tuple2<Map<String, List<Event>>, Long>> timedOut =
		new OutputTag<Tuple2<Map<String, List<Event>>, Long>>("timedOut") {};
	final KeyedOneInputStreamOperatorTestHarness<Integer, Event, Map<String, List<Event>>> harness =
		new KeyedOneInputStreamOperatorTestHarness<>(
			new CepOperator<>(
				Event.createTypeSerializer(),
				false,
				new NFAFactory(true),
				null,
				null,
				new TimedOutProcessFunction(timedOut),
				null), new KeySelector<Event, Integer>() {
			private static final long serialVersionUID = 7219185117566268366L;

			@Override
			public Integer getKey(Event value) throws Exception {
				return value.getId();
			}
		}, BasicTypeInfo.INT_TYPE_INFO);

	try {
		String rocksDbPath = tempFolder.newFolder().getAbsolutePath();
		RocksDBStateBackend rocksDBStateBackend = new RocksDBStateBackend(new MemoryStateBackend());
		rocksDBStateBackend.setDbStoragePath(rocksDbPath);

		harness.setStateBackend(rocksDBStateBackend);
		harness.setup(
			new KryoSerializer<>(
				(Class<Map<String, List<Event>>>) (Object) Map.class,
				new ExecutionConfig()));
		harness.open();

		harness.processElement(new StreamRecord<>(startEvent, 3L));
		harness.processWatermark(new Watermark(watermarkTimestamp1));
		harness.processWatermark(new Watermark(watermarkTimestamp2));

		Queue<Object> result = harness.getOutput();
		Queue<StreamRecord<Tuple2<Map<String, List<Event>>, Long>>> sideOutput = harness.getSideOutput(timedOut);

		assertEquals(2L, result.size());
		assertEquals(1L, sideOutput.size());

		Object watermark1 = result.poll();

		assertTrue(watermark1 instanceof Watermark);

		assertEquals(watermarkTimestamp1, ((Watermark) watermark1).getTimestamp());

		Tuple2<Map<String, List<Event>>, Long> leftResult = sideOutput.poll().getValue();

		assertEquals(watermarkTimestamp2, (long) leftResult.f1);
		assertEquals(expectedSequence, leftResult.f0);

		Object watermark2 = result.poll();

		assertTrue(watermark2 instanceof Watermark);

		assertEquals(watermarkTimestamp2, ((Watermark) watermark2).getTimestamp());
	} finally {
		harness.close();
	}
}
 

/**
 * Tests that the internal time of a CEP operator advances only given watermarks. See FLINK-5033
 */
@Test
public void testKeyedAdvancingTimeWithoutElements() throws Exception {
	final Event startEvent = new Event(42, "start", 1.0);
	final long watermarkTimestamp1 = 5L;
	final long watermarkTimestamp2 = 13L;

	final Map<String, List<Event>> expectedSequence = new HashMap<>(2);
	expectedSequence.put("start", Collections.<Event>singletonList(startEvent));

	final OutputTag<Tuple2<Map<String, List<Event>>, Long>> timedOut =
		new OutputTag<Tuple2<Map<String, List<Event>>, Long>>("timedOut") {};
	final KeyedOneInputStreamOperatorTestHarness<Integer, Event, Map<String, List<Event>>> harness =
		new KeyedOneInputStreamOperatorTestHarness<>(
			new CepOperator<>(
				Event.createTypeSerializer(),
				false,
				new NFAFactory(true),
				null,
				null,
				new TimedOutProcessFunction(timedOut),
				null), new KeySelector<Event, Integer>() {
			private static final long serialVersionUID = 7219185117566268366L;

			@Override
			public Integer getKey(Event value) throws Exception {
				return value.getId();
			}
		}, BasicTypeInfo.INT_TYPE_INFO);

	try {
		String rocksDbPath = tempFolder.newFolder().getAbsolutePath();
		RocksDBStateBackend rocksDBStateBackend = new RocksDBStateBackend(new MemoryStateBackend());
		rocksDBStateBackend.setDbStoragePath(rocksDbPath);

		harness.setStateBackend(rocksDBStateBackend);
		harness.setup(
			new KryoSerializer<>(
				(Class<Map<String, List<Event>>>) (Object) Map.class,
				new ExecutionConfig()));
		harness.open();

		harness.processElement(new StreamRecord<>(startEvent, 3L));
		harness.processWatermark(new Watermark(watermarkTimestamp1));
		harness.processWatermark(new Watermark(watermarkTimestamp2));

		Queue<Object> result = harness.getOutput();
		Queue<StreamRecord<Tuple2<Map<String, List<Event>>, Long>>> sideOutput = harness.getSideOutput(timedOut);

		assertEquals(2L, result.size());
		assertEquals(1L, sideOutput.size());

		Object watermark1 = result.poll();

		assertTrue(watermark1 instanceof Watermark);

		assertEquals(watermarkTimestamp1, ((Watermark) watermark1).getTimestamp());

		Tuple2<Map<String, List<Event>>, Long> leftResult = sideOutput.poll().getValue();

		assertEquals(watermarkTimestamp2, (long) leftResult.f1);
		assertEquals(expectedSequence, leftResult.f0);

		Object watermark2 = result.poll();

		assertTrue(watermark2 instanceof Watermark);

		assertEquals(watermarkTimestamp2, ((Watermark) watermark2).getTimestamp());
	} finally {
		harness.close();
	}
}
 

@Test
public void testTimerCleanupOfPendingTimerList() throws Exception {
  // test harness
  WindowDoFnOperator<Long, Long, Long> windowDoFnOperator = getWindowDoFnOperator();
  KeyedOneInputStreamOperatorTestHarness<
          ByteBuffer, WindowedValue<KeyedWorkItem<Long, Long>>, WindowedValue<KV<Long, Long>>>
      testHarness = createTestHarness(windowDoFnOperator);
  testHarness.open();

  DoFnOperator<KeyedWorkItem<Long, Long>, KV<Long, Long>>.FlinkTimerInternals timerInternals =
      windowDoFnOperator.timerInternals;

  // process elements
  IntervalWindow window = new IntervalWindow(new Instant(0), Duration.millis(100));
  IntervalWindow window2 = new IntervalWindow(new Instant(100), Duration.millis(100));
  testHarness.processWatermark(0L);

  // Use two different keys to check for correct watermark hold calculation
  testHarness.processElement(
      Item.builder().key(1L).timestamp(1L).value(100L).window(window).build().toStreamRecord());
  testHarness.processElement(
      Item.builder()
          .key(2L)
          .timestamp(150L)
          .value(150L)
          .window(window2)
          .build()
          .toStreamRecord());

  testHarness.processWatermark(1);

  // Note that the following is 1 because the state is key-partitioned
  assertThat(Iterables.size(timerInternals.pendingTimersById.keys()), is(1));

  assertThat(testHarness.numKeyedStateEntries(), is(6));
  assertThat(windowDoFnOperator.getCurrentOutputWatermark(), is(1L));
  assertThat(timerInternals.getMinOutputTimestampMs(), is(Long.MAX_VALUE));

  // close window
  testHarness.processWatermark(100L);

  // Note that the following is zero because we only the first key is active
  assertThat(Iterables.size(timerInternals.pendingTimersById.keys()), is(0));

  assertThat(testHarness.numKeyedStateEntries(), is(3));
  assertThat(windowDoFnOperator.getCurrentOutputWatermark(), is(100L));
  assertThat(timerInternals.getMinOutputTimestampMs(), is(Long.MAX_VALUE));

  testHarness.processWatermark(200L);

  // All the state has been cleaned up
  assertThat(testHarness.numKeyedStateEntries(), is(0));

  assertThat(
      stripStreamRecordFromWindowedValue(testHarness.getOutput()),
      containsInAnyOrder(
          WindowedValue.of(
              KV.of(1L, 100L), new Instant(99), window, PaneInfo.createPane(true, true, ON_TIME)),
          WindowedValue.of(
              KV.of(2L, 150L),
              new Instant(199),
              window2,
              PaneInfo.createPane(true, true, ON_TIME))));

  // cleanup
  testHarness.close();
}
 

/**
 * Tests that the internal time of a CEP operator advances only given watermarks. See FLINK-5033
 */
@Test
public void testKeyedAdvancingTimeWithoutElements() throws Exception {
	final Event startEvent = new Event(42, "start", 1.0);
	final long watermarkTimestamp1 = 5L;
	final long watermarkTimestamp2 = 13L;

	final Map<String, List<Event>> expectedSequence = new HashMap<>(2);
	expectedSequence.put("start", Collections.<Event>singletonList(startEvent));

	final OutputTag<Tuple2<Map<String, List<Event>>, Long>> timedOut =
		new OutputTag<Tuple2<Map<String, List<Event>>, Long>>("timedOut") {};
	final KeyedOneInputStreamOperatorTestHarness<Integer, Event, Map<String, List<Event>>> harness =
		new KeyedOneInputStreamOperatorTestHarness<>(
			new CepOperator<>(
				Event.createTypeSerializer(),
				false,
				new NFAFactory(true),
				null,
				null,
				new TimedOutProcessFunction(timedOut),
				null), new KeySelector<Event, Integer>() {
			private static final long serialVersionUID = 7219185117566268366L;

			@Override
			public Integer getKey(Event value) throws Exception {
				return value.getId();
			}
		}, BasicTypeInfo.INT_TYPE_INFO);

	try {
		String rocksDbPath = tempFolder.newFolder().getAbsolutePath();
		RocksDBStateBackend rocksDBStateBackend = new RocksDBStateBackend(new MemoryStateBackend());
		rocksDBStateBackend.setDbStoragePath(rocksDbPath);

		harness.setStateBackend(rocksDBStateBackend);
		harness.setup(
			new KryoSerializer<>(
				(Class<Map<String, List<Event>>>) (Object) Map.class,
				new ExecutionConfig()));
		harness.open();

		harness.processElement(new StreamRecord<>(startEvent, 3L));
		harness.processWatermark(new Watermark(watermarkTimestamp1));
		harness.processWatermark(new Watermark(watermarkTimestamp2));

		Queue<Object> result = harness.getOutput();
		Queue<StreamRecord<Tuple2<Map<String, List<Event>>, Long>>> sideOutput = harness.getSideOutput(timedOut);

		assertEquals(2L, result.size());
		assertEquals(1L, sideOutput.size());

		Object watermark1 = result.poll();

		assertTrue(watermark1 instanceof Watermark);

		assertEquals(watermarkTimestamp1, ((Watermark) watermark1).getTimestamp());

		Tuple2<Map<String, List<Event>>, Long> leftResult = sideOutput.poll().getValue();

		assertEquals(watermarkTimestamp2, (long) leftResult.f1);
		assertEquals(expectedSequence, leftResult.f0);

		Object watermark2 = result.poll();

		assertTrue(watermark2 instanceof Watermark);

		assertEquals(watermarkTimestamp2, ((Watermark) watermark2).getTimestamp());
	} finally {
		harness.close();
	}
}
 

/**
 * Verify that firing event-time timers see the state of the key that was active
 * when the timer was set.
 */
@Test
public void testEventTimeTimersDontInterfere() throws Exception {
	TestOperator testOperator = new TestOperator();

	KeyedOneInputStreamOperatorTestHarness<Integer, Tuple2<Integer, String>, String> testHarness =
			new KeyedOneInputStreamOperatorTestHarness<>(testOperator, new TestKeySelector(), BasicTypeInfo.INT_TYPE_INFO);

	testHarness.open();

	testHarness.processWatermark(0L);

	testHarness.processElement(new Tuple2<>(1, "SET_EVENT_TIME_TIMER:20"), 0);

	testHarness.processElement(new Tuple2<>(0, "SET_STATE:HELLO"), 0);
	testHarness.processElement(new Tuple2<>(1, "SET_STATE:CIAO"), 0);

	testHarness.processElement(new Tuple2<>(0, "SET_EVENT_TIME_TIMER:10"), 0);

	testHarness.processWatermark(10L);

	assertThat(
			extractResult(testHarness),
			contains("ON_EVENT_TIME:HELLO"));

	testHarness.processWatermark(20L);

	assertThat(
			extractResult(testHarness),
			contains("ON_EVENT_TIME:CIAO"));
}
 

/**
 * Verify that firing event-time timers see the state of the key that was active
 * when the timer was set.
 */
@Test
public void testEventTimeTimersDontInterfere() throws Exception {
	TestOperator testOperator = new TestOperator();

	KeyedOneInputStreamOperatorTestHarness<Integer, Tuple2<Integer, String>, String> testHarness =
			new KeyedOneInputStreamOperatorTestHarness<>(testOperator, new TestKeySelector(), BasicTypeInfo.INT_TYPE_INFO);

	testHarness.open();

	testHarness.processWatermark(0L);

	testHarness.processElement(new Tuple2<>(1, "SET_EVENT_TIME_TIMER:20"), 0);

	testHarness.processElement(new Tuple2<>(0, "SET_STATE:HELLO"), 0);
	testHarness.processElement(new Tuple2<>(1, "SET_STATE:CIAO"), 0);

	testHarness.processElement(new Tuple2<>(0, "SET_EVENT_TIME_TIMER:10"), 0);

	testHarness.processWatermark(10L);

	assertThat(
			extractResult(testHarness),
			contains("ON_EVENT_TIME:HELLO"));

	testHarness.processWatermark(20L);

	assertThat(
			extractResult(testHarness),
			contains("ON_EVENT_TIME:CIAO"));
}