org.apache.flink.streaming.util.KeyedOneInputStreamOperatorTestHarness Java Examples

@Test
public void testStateDoesNotInterfere() throws Exception {
	try (KeyedOneInputStreamOperatorTestHarness<Integer, Tuple2<Integer, String>, String> testHarness = createTestHarness()) {
		testHarness.open();

		testHarness.processElement(new Tuple2<>(0, "SET_STATE:HELLO"), 0);
		testHarness.processElement(new Tuple2<>(1, "SET_STATE:CIAO"), 0);

		testHarness.processElement(new Tuple2<>(1, "EMIT_STATE"), 0);
		testHarness.processElement(new Tuple2<>(0, "EMIT_STATE"), 0);

		assertThat(
			extractResult(testHarness),
			contains("ON_ELEMENT:1:CIAO", "ON_ELEMENT:0:HELLO"));
	}
}
 

@Test
public void testNullOutputTagRefusal() throws Exception {
	LegacyKeyedProcessOperator<Integer, Integer, String> operator =
		new LegacyKeyedProcessOperator<>(new NullOutputTagEmittingProcessFunction());

	OneInputStreamOperatorTestHarness<Integer, String> testHarness =
		new KeyedOneInputStreamOperatorTestHarness<>(
			operator, new IdentityKeySelector<>(), BasicTypeInfo.INT_TYPE_INFO);

	testHarness.setup();
	testHarness.open();

	testHarness.setProcessingTime(17);
	try {
		expectedException.expect(IllegalArgumentException.class);
		testHarness.processElement(new StreamRecord<>(5));
	} finally {
		testHarness.close();
	}
}
 

@Test
public void testTimestampAndWatermarkQuerying() throws Exception {

	LegacyKeyedProcessOperator<Integer, Integer, String> operator =
			new LegacyKeyedProcessOperator<>(new QueryingFlatMapFunction(TimeDomain.EVENT_TIME));

	OneInputStreamOperatorTestHarness<Integer, String> testHarness =
			new KeyedOneInputStreamOperatorTestHarness<>(operator, new IdentityKeySelector<Integer>(), BasicTypeInfo.INT_TYPE_INFO);

	testHarness.setup();
	testHarness.open();

	testHarness.processWatermark(new Watermark(17));
	testHarness.processElement(new StreamRecord<>(5, 12L));

	testHarness.processWatermark(new Watermark(42));
	testHarness.processElement(new StreamRecord<>(6, 13L));

	ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>();

	expectedOutput.add(new Watermark(17L));
	expectedOutput.add(new StreamRecord<>("5TIME:17 TS:12", 12L));
	expectedOutput.add(new Watermark(42L));
	expectedOutput.add(new StreamRecord<>("6TIME:42 TS:13", 13L));

	TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput());

	testHarness.close();
}
 

@Test
public void testWindowStateNotAvailableToMergingWindows() throws Exception {
	WindowAssigner<Integer, TimeWindow> mockAssigner = mockMergingAssigner();
	Trigger<Integer, TimeWindow> mockTrigger = mockTrigger();
	InternalWindowFunction<Iterable<Integer>, Void, Integer, TimeWindow> mockWindowFunction = mockWindowFunction();

	KeyedOneInputStreamOperatorTestHarness<Integer, Integer, Void> testHarness =
		createWindowOperator(mockAssigner, mockTrigger, 20L, mockWindowFunction);

	testHarness.open();

	when(mockTrigger.onElement(anyInt(), anyLong(), anyTimeWindow(), anyTriggerContext()))
		.thenReturn(TriggerResult.FIRE);

	when(mockAssigner.assignWindows(anyInt(), anyLong(), anyAssignerContext()))
		.thenReturn(Arrays.asList(new TimeWindow(0, 20)));

	doAnswer(new Answer<Object>() {
		@Override
		public Object answer(InvocationOnMock invocationOnMock) throws Throwable {
			InternalWindowFunction.InternalWindowContext context = (InternalWindowFunction.InternalWindowContext) invocationOnMock.getArguments()[2];
			context.windowState().getState(valueStateDescriptor).update("hello");
			return null;
		}
	}).when(mockWindowFunction).process(anyInt(), anyTimeWindow(), anyInternalWindowContext(), anyIntIterable(), WindowOperatorContractTest.<Void>anyCollector());

	expectedException.expect(UnsupportedOperationException.class);
	expectedException.expectMessage("Per-window state is not allowed when using merging windows.");
	testHarness.processElement(new StreamRecord<>(0, 0L));
}
 

/**
 * Special method for creating a {@link WindowOperator} with a custom {@link StateDescriptor}
 * for the window contents state.
 */
private <W extends Window, ACC, OUT> KeyedOneInputStreamOperatorTestHarness<Integer, Integer, OUT> createWindowOperator(
		WindowAssigner<Integer, W> assigner,
		Trigger<Integer, W> trigger,
		long allowedLatenss,
		StateDescriptor<? extends AppendingState<Integer, ACC>, ?> stateDescriptor,
		InternalWindowFunction<ACC, OUT, Integer, W> windowFunction) throws Exception {

	KeySelector<Integer, Integer> keySelector = new KeySelector<Integer, Integer>() {
		private static final long serialVersionUID = 1L;

		@Override
		public Integer getKey(Integer value) throws Exception {
			return value;
		}
	};

	@SuppressWarnings("unchecked")
	WindowOperator<Integer, Integer, ACC, OUT, W> operator = new WindowOperator<>(
			assigner,
			assigner.getWindowSerializer(new ExecutionConfig()),
			keySelector,
			IntSerializer.INSTANCE,
			stateDescriptor,
			windowFunction,
			trigger,
			allowedLatenss,
			null /* late output tag */);

	return new KeyedOneInputStreamOperatorTestHarness<>(
			operator,
			keySelector,
			BasicTypeInfo.INT_TYPE_INFO);
}
 

@Test
public void testGroupedFold() throws Exception {

	KeySelector<Integer, String> keySelector = new KeySelector<Integer, String>() {

		@Override
		public String getKey(Integer value) {
			return value.toString();
		}
	};

	StreamGroupedFold<Integer, String, String> operator = new StreamGroupedFold<>(new MyFolder(), "100");
	operator.setOutputType(BasicTypeInfo.STRING_TYPE_INFO, new ExecutionConfig());

	OneInputStreamOperatorTestHarness<Integer, String> testHarness =
			new KeyedOneInputStreamOperatorTestHarness<>(operator, keySelector, BasicTypeInfo.STRING_TYPE_INFO);

	long initialTime = 0L;
	ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>();

	testHarness.open();

	testHarness.processElement(new StreamRecord<>(1, initialTime + 1));
	testHarness.processElement(new StreamRecord<>(1, initialTime + 2));
	testHarness.processWatermark(new Watermark(initialTime + 2));
	testHarness.processElement(new StreamRecord<>(2, initialTime + 3));
	testHarness.processElement(new StreamRecord<>(2, initialTime + 4));
	testHarness.processElement(new StreamRecord<>(3, initialTime + 5));

	expectedOutput.add(new StreamRecord<>("1001", initialTime + 1));
	expectedOutput.add(new StreamRecord<>("10011", initialTime + 2));
	expectedOutput.add(new Watermark(initialTime + 2));
	expectedOutput.add(new StreamRecord<>("1002", initialTime + 3));
	expectedOutput.add(new StreamRecord<>("10022", initialTime + 4));
	expectedOutput.add(new StreamRecord<>("1003", initialTime + 5));

	TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput());
}
 

protected <W extends Window, OUT> KeyedOneInputStreamOperatorTestHarness<Integer, Integer, OUT> createWindowOperator(
		WindowAssigner<Integer, W> assigner,
		Trigger<Integer, W> trigger,
		long allowedLatenss,
		InternalWindowFunction<Iterable<Integer>, OUT, Integer, W> windowFunction) throws Exception {

	return createWindowOperator(
			assigner,
			trigger,
			allowedLatenss,
			windowFunction,
			null /* late output tag */);
}
 

@Test
public void testOnElementPurgeDoesNotCleanupMergingSet() throws Exception {

	MergingWindowAssigner<Integer, TimeWindow> mockAssigner = mockMergingAssigner();
	Trigger<Integer, TimeWindow> mockTrigger = mockTrigger();
	InternalWindowFunction<Iterable<Integer>, Void, Integer, TimeWindow> mockWindowFunction = mockWindowFunction();

	KeyedOneInputStreamOperatorTestHarness<Integer, Integer, Void> testHarness =
			createWindowOperator(mockAssigner, mockTrigger, 0L, mockWindowFunction);

	testHarness.open();

	when(mockAssigner.assignWindows(anyInt(), anyLong(), anyAssignerContext()))
			.thenReturn(Arrays.asList(new TimeWindow(0, 2)));

	assertEquals(0, testHarness.getOutput().size());
	assertEquals(0, testHarness.numKeyedStateEntries());

	doAnswer(new Answer<TriggerResult>() {
		@Override
		public TriggerResult answer(InvocationOnMock invocation) throws Exception {
			return TriggerResult.PURGE;
		}
	}).when(mockTrigger).onElement(Matchers.<Integer>anyObject(), anyLong(), anyTimeWindow(), anyTriggerContext());

	testHarness.processElement(new StreamRecord<>(0, 0L));

	assertEquals(1, testHarness.numKeyedStateEntries()); // the merging window set

	assertEquals(1, testHarness.numEventTimeTimers()); // one cleanup timer

	assertEquals(0, testHarness.getOutput().size());
}
 

/**
 * Verify that a low-level timer is set for processing-time timers in case of restore.
 */
@Test
public void testEnsureProcessingTimeTimerRegisteredOnRestore() throws Exception {
	OperatorSubtaskState snapshot;
	try (KeyedOneInputStreamOperatorTestHarness<Integer, Tuple2<Integer, String>, String> testHarness = createTestHarness()) {
		testHarness.open();

		testHarness.setProcessingTime(0L);

		testHarness.processElement(new Tuple2<>(1, "SET_PROC_TIME_TIMER:20"), 0);

		testHarness.processElement(new Tuple2<>(0, "SET_STATE:HELLO"), 0);
		testHarness.processElement(new Tuple2<>(1, "SET_STATE:CIAO"), 0);

		testHarness.processElement(new Tuple2<>(0, "SET_PROC_TIME_TIMER:10"), 0);

		snapshot = testHarness.snapshot(0, 0);
	}

	try (KeyedOneInputStreamOperatorTestHarness<Integer, Tuple2<Integer, String>, String> testHarness1 = createTestHarness()) {
		testHarness1.setProcessingTime(0L);

		testHarness1.setup();
		testHarness1.initializeState(snapshot);
		testHarness1.open();

		testHarness1.setProcessingTime(10L);

		assertThat(
			extractResult(testHarness1),
			contains("ON_PROC_TIME:HELLO"));

		testHarness1.setProcessingTime(20L);

		assertThat(
			extractResult(testHarness1),
			contains("ON_PROC_TIME:CIAO"));
	}
}
 

@Override
protected <W extends Window, OUT> KeyedOneInputStreamOperatorTestHarness<Integer, Integer, OUT> createWindowOperator(
		WindowAssigner<Integer, W> assigner,
		Trigger<Integer, W> trigger,
		long allowedLatenss,
		InternalWindowFunction<Iterable<Integer>, OUT, Integer, W> windowFunction,
		OutputTag<Integer> lateOutputTag) throws Exception {

	KeySelector<Integer, Integer> keySelector = new KeySelector<Integer, Integer>() {
		private static final long serialVersionUID = 1L;

		@Override
		public Integer getKey(Integer value) throws Exception {
			return value;
		}
	};

	ListStateDescriptor<Integer> intListDescriptor =
			new ListStateDescriptor<>("int-list", IntSerializer.INSTANCE);

	@SuppressWarnings("unchecked")
	WindowOperator<Integer, Integer, Iterable<Integer>, OUT, W> operator = new WindowOperator<>(
			assigner,
			assigner.getWindowSerializer(new ExecutionConfig()),
			keySelector,
			IntSerializer.INSTANCE,
			intListDescriptor,
			windowFunction,
			trigger,
			allowedLatenss,
			lateOutputTag);

	return new KeyedOneInputStreamOperatorTestHarness<>(
			operator,
			keySelector,
			BasicTypeInfo.INT_TYPE_INFO);
}
 

/**
 * Verify that firing processing-time timers see the state of the key that was active
 * when the timer was set.
 */
@Test
public void testProcessingTimeTimersDontInterfere() throws Exception {
	try (KeyedOneInputStreamOperatorTestHarness<Integer, Tuple2<Integer, String>, String> testHarness = createTestHarness()) {
		testHarness.open();

		testHarness.setProcessingTime(0L);

		testHarness.processElement(new Tuple2<>(1, "SET_PROC_TIME_TIMER:20"), 0);

		testHarness.processElement(new Tuple2<>(0, "SET_STATE:HELLO"), 0);
		testHarness.processElement(new Tuple2<>(1, "SET_STATE:CIAO"), 0);

		testHarness.processElement(new Tuple2<>(0, "SET_PROC_TIME_TIMER:10"), 0);

		testHarness.setProcessingTime(10L);

		assertThat(
			extractResult(testHarness),
			contains("ON_PROC_TIME:HELLO"));

		testHarness.setProcessingTime(20L);

		assertThat(
			extractResult(testHarness),
			contains("ON_PROC_TIME:CIAO"));
	}
}
 

/**
 * This also verifies that the timestamps ouf side-emitted records is correct.
 */
@Test
public void testSideOutput() throws Exception {
	KeyedProcessOperator<Integer, Integer, String> operator = new KeyedProcessOperator<>(new SideOutputProcessFunction());

	OneInputStreamOperatorTestHarness<Integer, String> testHarness =
		new KeyedOneInputStreamOperatorTestHarness<>(
			operator, new IdentityKeySelector<>(), BasicTypeInfo.INT_TYPE_INFO);

	testHarness.setup();
	testHarness.open();

	testHarness.processElement(new StreamRecord<>(42, 17L /* timestamp */));

	ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>();

	expectedOutput.add(new StreamRecord<>("IN:42", 17L /* timestamp */));

	TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput());

	ConcurrentLinkedQueue<StreamRecord<Integer>> expectedIntSideOutput = new ConcurrentLinkedQueue<>();
	expectedIntSideOutput.add(new StreamRecord<>(42, 17L /* timestamp */));
	ConcurrentLinkedQueue<StreamRecord<Integer>> intSideOutput =
		testHarness.getSideOutput(SideOutputProcessFunction.INTEGER_OUTPUT_TAG);
	TestHarnessUtil.assertOutputEquals(
		"Side output was not correct.",
		expectedIntSideOutput,
		intSideOutput);

	ConcurrentLinkedQueue<StreamRecord<Long>> expectedLongSideOutput = new ConcurrentLinkedQueue<>();
	expectedLongSideOutput.add(new StreamRecord<>(42L, 17L /* timestamp */));
	ConcurrentLinkedQueue<StreamRecord<Long>> longSideOutput =
		testHarness.getSideOutput(SideOutputProcessFunction.LONG_OUTPUT_TAG);
	TestHarnessUtil.assertOutputEquals(
		"Side output was not correct.",
		expectedLongSideOutput,
		longSideOutput);

	testHarness.close();
}
 

protected <W extends Window, OUT> KeyedOneInputStreamOperatorTestHarness<Integer, Integer, OUT> createWindowOperator(
		WindowAssigner<Integer, W> assigner,
		Trigger<Integer, W> trigger,
		long allowedLatenss,
		InternalWindowFunction<Iterable<Integer>, OUT, Integer, W> windowFunction,
		OutputTag<Integer> lateOutputTag) throws Exception {

	KeySelector<Integer, Integer> keySelector = new KeySelector<Integer, Integer>() {
		private static final long serialVersionUID = 1L;

		@Override
		public Integer getKey(Integer value) throws Exception {
			return value;
		}
	};

	ListStateDescriptor<StreamRecord<Integer>> intListDescriptor =
			new ListStateDescriptor<>(
					"int-list",
					(TypeSerializer<StreamRecord<Integer>>) new StreamElementSerializer(IntSerializer.INSTANCE));

	@SuppressWarnings("unchecked")
	EvictingWindowOperator<Integer, Integer, OUT, W> operator = new EvictingWindowOperator<>(
			assigner,
			assigner.getWindowSerializer(new ExecutionConfig()),
			keySelector,
			IntSerializer.INSTANCE,
			intListDescriptor,
			windowFunction,
			trigger,
			CountEvictor.<W>of(100),
			allowedLatenss,
			lateOutputTag);

	return new KeyedOneInputStreamOperatorTestHarness<>(
			operator,
			keySelector,
			BasicTypeInfo.INT_TYPE_INFO);
}
 

@Test
public void testWindowStateNotAvailableToMergingWindows() throws Exception {
	WindowAssigner<Integer, TimeWindow> mockAssigner = mockMergingAssigner();
	Trigger<Integer, TimeWindow> mockTrigger = mockTrigger();
	InternalWindowFunction<Iterable<Integer>, Void, Integer, TimeWindow> mockWindowFunction = mockWindowFunction();

	KeyedOneInputStreamOperatorTestHarness<Integer, Integer, Void> testHarness =
		createWindowOperator(mockAssigner, mockTrigger, 20L, mockWindowFunction);

	testHarness.open();

	when(mockTrigger.onElement(anyInt(), anyLong(), anyTimeWindow(), anyTriggerContext()))
		.thenReturn(TriggerResult.FIRE);

	when(mockAssigner.assignWindows(anyInt(), anyLong(), anyAssignerContext()))
		.thenReturn(Arrays.asList(new TimeWindow(0, 20)));

	doAnswer(new Answer<Object>() {
		@Override
		public Object answer(InvocationOnMock invocationOnMock) throws Throwable {
			InternalWindowFunction.InternalWindowContext context = (InternalWindowFunction.InternalWindowContext) invocationOnMock.getArguments()[2];
			context.windowState().getState(valueStateDescriptor).update("hello");
			return null;
		}
	}).when(mockWindowFunction).process(anyInt(), anyTimeWindow(), anyInternalWindowContext(), anyIntIterable(), WindowOperatorContractTest.<Void>anyCollector());

	expectedException.expect(UnsupportedOperationException.class);
	expectedException.expectMessage("Per-window state is not allowed when using merging windows.");
	testHarness.processElement(new StreamRecord<>(0, 0L));
}
 

/**
 * Verifies that we don't have leakage between different keys.
 */
@Test
public void testProcessingTimeTimerWithState() throws Exception {

	LegacyKeyedProcessOperator<Integer, Integer, String> operator =
			new LegacyKeyedProcessOperator<>(new TriggeringStatefulFlatMapFunction(TimeDomain.PROCESSING_TIME));

	OneInputStreamOperatorTestHarness<Integer, String> testHarness =
			new KeyedOneInputStreamOperatorTestHarness<>(operator, new IdentityKeySelector<Integer>(), BasicTypeInfo.INT_TYPE_INFO);

	testHarness.setup();
	testHarness.open();

	testHarness.setProcessingTime(1);
	testHarness.processElement(new StreamRecord<>(17)); // should set timer for 6

	testHarness.setProcessingTime(2);
	testHarness.processElement(new StreamRecord<>(42)); // should set timer for 7

	testHarness.setProcessingTime(6);
	testHarness.setProcessingTime(7);

	ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>();

	expectedOutput.add(new StreamRecord<>("INPUT:17"));
	expectedOutput.add(new StreamRecord<>("INPUT:42"));
	expectedOutput.add(new StreamRecord<>("STATE:17"));
	expectedOutput.add(new StreamRecord<>("STATE:42"));

	TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput());

	testHarness.close();
}
 

private void testMergingWindowSetClearedAtGarbageCollection(TimeDomainAdaptor timeAdaptor) throws Exception {
	WindowAssigner<Integer, TimeWindow> mockAssigner = mockMergingAssigner();
	timeAdaptor.setIsEventTime(mockAssigner);
	Trigger<Integer, TimeWindow> mockTrigger = mockTrigger();
	InternalWindowFunction<Iterable<Integer>, Void, Integer, TimeWindow> mockWindowFunction = mockWindowFunction();

	KeyedOneInputStreamOperatorTestHarness<Integer, Integer, Void> testHarness =
			createWindowOperator(mockAssigner, mockTrigger, 20L, mockWindowFunction);

	testHarness.open();

	when(mockAssigner.assignWindows(anyInt(), anyLong(), anyAssignerContext()))
			.thenReturn(Arrays.asList(new TimeWindow(0, 20)));

	assertEquals(0, testHarness.getOutput().size());
	assertEquals(0, testHarness.numKeyedStateEntries());

	testHarness.processElement(new StreamRecord<>(0, 0L));

	assertEquals(2, testHarness.numKeyedStateEntries()); // window contents plus merging window set
	assertEquals(1, timeAdaptor.numTimers(testHarness)); // gc timers

	timeAdaptor.advanceTime(testHarness, 19 + 20); // 19 is maxTime of the window

	assertEquals(0, testHarness.numKeyedStateEntries());
	assertEquals(0, timeAdaptor.numTimers(testHarness));
}
 

@Test
public void testOnElementPurgeDoesNotCleanupMergingSet() throws Exception {

	MergingWindowAssigner<Integer, TimeWindow> mockAssigner = mockMergingAssigner();
	Trigger<Integer, TimeWindow> mockTrigger = mockTrigger();
	InternalWindowFunction<Iterable<Integer>, Void, Integer, TimeWindow> mockWindowFunction = mockWindowFunction();

	KeyedOneInputStreamOperatorTestHarness<Integer, Integer, Void> testHarness =
			createWindowOperator(mockAssigner, mockTrigger, 0L, mockWindowFunction);

	testHarness.open();

	when(mockAssigner.assignWindows(anyInt(), anyLong(), anyAssignerContext()))
			.thenReturn(Arrays.asList(new TimeWindow(0, 2)));

	assertEquals(0, testHarness.getOutput().size());
	assertEquals(0, testHarness.numKeyedStateEntries());

	doAnswer(new Answer<TriggerResult>() {
		@Override
		public TriggerResult answer(InvocationOnMock invocation) throws Exception {
			return TriggerResult.PURGE;
		}
	}).when(mockTrigger).onElement(Matchers.<Integer>anyObject(), anyLong(), anyTimeWindow(), anyTriggerContext());

	testHarness.processElement(new StreamRecord<>(0, 0L));

	assertEquals(1, testHarness.numKeyedStateEntries()); // the merging window set

	assertEquals(1, testHarness.numEventTimeTimers()); // one cleanup timer

	assertEquals(0, testHarness.getOutput().size());
}
 

private OperatorSubtaskState createOperatorSubtaskState(OneInputStreamOperator<Integer, Void> operator) throws Exception {
	try (KeyedOneInputStreamOperatorTestHarness<Integer, Integer, Void> testHarness =
			new KeyedOneInputStreamOperatorTestHarness<>(operator, id -> id, Types.INT, 128, 1, 0)) {

		testHarness.setup(VoidSerializer.INSTANCE);
		testHarness.open();

		testHarness.processElement(1, 0);
		testHarness.processElement(2, 0);
		testHarness.processElement(3, 0);

		return testHarness.snapshot(0, 0);
	}
}
 

/**
 * Verifies that we don't have leakage between different keys.
 */
@Test
public void testEventTimeTimerWithState() throws Exception {

	KeyedProcessOperator<Integer, Integer, String> operator =
			new KeyedProcessOperator<>(new TriggeringStatefulFlatMapFunction(TimeDomain.EVENT_TIME));

	OneInputStreamOperatorTestHarness<Integer, String> testHarness =
			new KeyedOneInputStreamOperatorTestHarness<>(operator, new IdentityKeySelector<Integer>(), BasicTypeInfo.INT_TYPE_INFO);

	testHarness.setup();
	testHarness.open();

	testHarness.processWatermark(new Watermark(1));
	testHarness.processElement(new StreamRecord<>(17, 0L)); // should set timer for 6
	testHarness.processElement(new StreamRecord<>(13, 0L)); // should set timer for 6

	testHarness.processWatermark(new Watermark(2));
	testHarness.processElement(new StreamRecord<>(42, 1L)); // should set timer for 7
	testHarness.processElement(new StreamRecord<>(13, 1L)); // should delete timer

	testHarness.processWatermark(new Watermark(6));
	testHarness.processWatermark(new Watermark(7));

	ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>();

	expectedOutput.add(new Watermark(1L));
	expectedOutput.add(new StreamRecord<>("INPUT:17", 0L));
	expectedOutput.add(new StreamRecord<>("INPUT:13", 0L));
	expectedOutput.add(new Watermark(2L));
	expectedOutput.add(new StreamRecord<>("INPUT:42", 1L));
	expectedOutput.add(new StreamRecord<>("STATE:17", 6L));
	expectedOutput.add(new Watermark(6L));
	expectedOutput.add(new StreamRecord<>("STATE:42", 7L));
	expectedOutput.add(new Watermark(7L));

	TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput());

	testHarness.close();
}
 

@Test
public void testProcessingElementsWithinAllowedLateness() throws Exception {
	WindowAssigner<Integer, TimeWindow> mockAssigner = mockTimeWindowAssigner();
	Trigger<Integer, TimeWindow> mockTrigger = mockTrigger();
	InternalWindowFunction<Iterable<Integer>, Void, Integer, TimeWindow> mockWindowFunction = mockWindowFunction();

	KeyedOneInputStreamOperatorTestHarness<Integer, Integer, Void> testHarness =
			createWindowOperator(mockAssigner, mockTrigger, 20L, mockWindowFunction);

	testHarness.open();

	when(mockAssigner.assignWindows(anyInt(), anyLong(), anyAssignerContext()))
			.thenReturn(Arrays.asList(new TimeWindow(0, 2)));

	assertEquals(0, testHarness.getOutput().size());
	assertEquals(0, testHarness.numKeyedStateEntries());

	shouldFireOnElement(mockTrigger);

	// 20 is just at the limit, window.maxTime() is 1 and allowed lateness is 20
	testHarness.processWatermark(new Watermark(20));

	testHarness.processElement(new StreamRecord<>(0, 0L));

	verify(mockWindowFunction, times(1)).process(eq(0), eq(new TimeWindow(0, 2)), anyInternalWindowContext(), intIterable(0), WindowOperatorContractTest.<Void>anyCollector());

	// clear is only called at cleanup time/GC time
	verify(mockTrigger, never()).clear(anyTimeWindow(), anyTriggerContext());

	// FIRE should not purge contents
	assertEquals(1, testHarness.numKeyedStateEntries()); // window contents plus trigger state
	assertEquals(1, testHarness.numEventTimeTimers()); // just the GC timer
}
 

private KeyedOneInputStreamOperatorTestHarness<
        ByteBuffer, WindowedValue<KeyedWorkItem<Long, Long>>, WindowedValue<KV<Long, Long>>>
    createTestHarness(WindowDoFnOperator<Long, Long, Long> windowDoFnOperator) throws Exception {
  return new KeyedOneInputStreamOperatorTestHarness<>(
      windowDoFnOperator,
      (KeySelector<WindowedValue<KeyedWorkItem<Long, Long>>, ByteBuffer>)
          o -> {
            try (ByteArrayOutputStream baos = new ByteArrayOutputStream()) {
              VarLongCoder.of().encode(o.getValue().key(), baos);
              return ByteBuffer.wrap(baos.toByteArray());
            }
          },
      new GenericTypeInfo<>(ByteBuffer.class));
}
 

@Test
public void testGroupedReduce() throws Exception {

	KeySelector<Integer, Integer> keySelector = new IntegerKeySelector();

	StreamGroupedReduce<Integer> operator = new StreamGroupedReduce<>(new MyReducer(), IntSerializer.INSTANCE);

	OneInputStreamOperatorTestHarness<Integer, Integer> testHarness =
			new KeyedOneInputStreamOperatorTestHarness<>(operator, keySelector, BasicTypeInfo.INT_TYPE_INFO);

	long initialTime = 0L;
	ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>();

	testHarness.open();

	testHarness.processElement(new StreamRecord<>(1, initialTime + 1));
	testHarness.processElement(new StreamRecord<>(1, initialTime + 2));
	testHarness.processWatermark(new Watermark(initialTime + 2));
	testHarness.processElement(new StreamRecord<>(2, initialTime + 3));
	testHarness.processElement(new StreamRecord<>(2, initialTime + 4));
	testHarness.processElement(new StreamRecord<>(3, initialTime + 5));

	expectedOutput.add(new StreamRecord<>(1, initialTime + 1));
	expectedOutput.add(new StreamRecord<>(2, initialTime + 2));
	expectedOutput.add(new Watermark(initialTime + 2));
	expectedOutput.add(new StreamRecord<>(2, initialTime + 3));
	expectedOutput.add(new StreamRecord<>(4, initialTime + 4));
	expectedOutput.add(new StreamRecord<>(3, initialTime + 5));

	TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput());
}
 

/**
 * Verifies that we don't have leakage between different keys.
 */
@Test
public void testProcessingTimeTimerWithState() throws Exception {

	KeyedProcessOperator<Integer, Integer, String> operator =
			new KeyedProcessOperator<>(new TriggeringStatefulFlatMapFunction(TimeDomain.PROCESSING_TIME));

	OneInputStreamOperatorTestHarness<Integer, String> testHarness =
			new KeyedOneInputStreamOperatorTestHarness<>(operator, new IdentityKeySelector<Integer>(), BasicTypeInfo.INT_TYPE_INFO);

	testHarness.setup();
	testHarness.open();

	testHarness.setProcessingTime(1);
	testHarness.processElement(new StreamRecord<>(17)); // should set timer for 6
	testHarness.processElement(new StreamRecord<>(13)); // should set timer for 6

	testHarness.setProcessingTime(2);
	testHarness.processElement(new StreamRecord<>(13)); // should delete timer
	testHarness.processElement(new StreamRecord<>(42)); // should set timer for 7

	testHarness.setProcessingTime(6);
	testHarness.setProcessingTime(7);

	ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>();

	expectedOutput.add(new StreamRecord<>("INPUT:17"));
	expectedOutput.add(new StreamRecord<>("INPUT:13"));
	expectedOutput.add(new StreamRecord<>("INPUT:42"));
	expectedOutput.add(new StreamRecord<>("STATE:17"));
	expectedOutput.add(new StreamRecord<>("STATE:42"));

	TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput());

	testHarness.close();
}
 

/**
 * Ensure that we get some output from the given operator when pushing in an element and
 * setting watermark and processing time to {@code Long.MAX_VALUE}.
 */
private static <K, IN, OUT> void processElementAndEnsureOutput(
		OneInputStreamOperator<IN, OUT> operator,
		KeySelector<IN, K> keySelector,
		TypeInformation<K> keyType,
		IN element) throws Exception {

	KeyedOneInputStreamOperatorTestHarness<K, IN, OUT> testHarness =
			new KeyedOneInputStreamOperatorTestHarness<>(
					operator,
					keySelector,
					keyType);

	testHarness.open();

	testHarness.setProcessingTime(0);
	testHarness.processWatermark(Long.MIN_VALUE);

	testHarness.processElement(new StreamRecord<>(element, 0));

	// provoke any processing-time/event-time triggers
	testHarness.setProcessingTime(Long.MAX_VALUE);
	testHarness.processWatermark(Long.MAX_VALUE);

	// we at least get the two watermarks and should also see an output element
	assertTrue(testHarness.getOutput().size() >= 3);

	testHarness.close();
}
 

@SuppressWarnings("unchecked")
private void validHarnessResult(
	KeyedOneInputStreamOperatorTestHarness<?, String, ?> harness,
	Integer expectedValue,
	String... records) throws Exception {
	for (String record : records) {
		harness.processElement(new StreamRecord<>(record, 1));
		StreamRecord<Integer> outputRecord = (StreamRecord<Integer>) harness.getOutput().poll();
		Assert.assertNotNull(outputRecord);
		Assert.assertEquals(expectedValue, outputRecord.getValue());
	}
}
 

private OperatorSubtaskState createOperatorSubtaskState(OneInputStreamOperator<Integer, Void> operator) throws Exception {
	try (KeyedOneInputStreamOperatorTestHarness<Integer, Integer, Void> testHarness =
			new KeyedOneInputStreamOperatorTestHarness<>(operator, id -> id, Types.INT, 128, 1, 0)) {

		testHarness.setup(VoidSerializer.INSTANCE);
		testHarness.open();

		testHarness.processElement(1, 0);
		testHarness.processElement(2, 0);
		testHarness.processElement(3, 0);

		return testHarness.snapshot(0, 0);
	}
}
 

/**
 * Ensure that we get some output from the given operator when pushing in an element and
 * setting watermark and processing time to {@code Long.MAX_VALUE}.
 */
private static <K, IN, OUT> void processElementAndEnsureOutput(
		OneInputStreamOperator<IN, OUT> operator,
		KeySelector<IN, K> keySelector,
		TypeInformation<K> keyType,
		IN element) throws Exception {

	KeyedOneInputStreamOperatorTestHarness<K, IN, OUT> testHarness =
			new KeyedOneInputStreamOperatorTestHarness<>(
					operator,
					keySelector,
					keyType);

	testHarness.open();

	testHarness.setProcessingTime(0);
	testHarness.processWatermark(Long.MIN_VALUE);

	testHarness.processElement(new StreamRecord<>(element, 0));

	// provoke any processing-time/event-time triggers
	testHarness.setProcessingTime(Long.MAX_VALUE);
	testHarness.processWatermark(Long.MAX_VALUE);

	// we at least get the two watermarks and should also see an output element
	assertTrue(testHarness.getOutput().size() >= 3);

	testHarness.close();
}
 

@Test
public void testTimestampAndWatermarkQuerying() throws Exception {

	KeyedProcessOperator<Integer, Integer, String> operator =
			new KeyedProcessOperator<>(new QueryingFlatMapFunction(TimeDomain.EVENT_TIME));

	OneInputStreamOperatorTestHarness<Integer, String> testHarness =
			new KeyedOneInputStreamOperatorTestHarness<>(operator, new IdentityKeySelector<Integer>(), BasicTypeInfo.INT_TYPE_INFO);

	testHarness.setup();
	testHarness.open();

	testHarness.processWatermark(new Watermark(17));
	testHarness.processElement(new StreamRecord<>(5, 12L));

	testHarness.processWatermark(new Watermark(42));
	testHarness.processElement(new StreamRecord<>(6, 13L));

	ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>();

	expectedOutput.add(new Watermark(17L));
	expectedOutput.add(new StreamRecord<>("5TIME:17 TS:12", 12L));
	expectedOutput.add(new Watermark(42L));
	expectedOutput.add(new StreamRecord<>("6TIME:42 TS:13", 13L));

	TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput());

	testHarness.close();
}
 

/**
 * Special method for creating a {@link WindowOperator} with a custom {@link StateDescriptor}
 * for the window contents state.
 */
private <W extends Window, ACC, OUT> KeyedOneInputStreamOperatorTestHarness<Integer, Integer, OUT> createWindowOperator(
		WindowAssigner<Integer, W> assigner,
		Trigger<Integer, W> trigger,
		long allowedLatenss,
		StateDescriptor<? extends AppendingState<Integer, ACC>, ?> stateDescriptor,
		InternalWindowFunction<ACC, OUT, Integer, W> windowFunction) throws Exception {

	KeySelector<Integer, Integer> keySelector = new KeySelector<Integer, Integer>() {
		private static final long serialVersionUID = 1L;

		@Override
		public Integer getKey(Integer value) throws Exception {
			return value;
		}
	};

	@SuppressWarnings("unchecked")
	WindowOperator<Integer, Integer, ACC, OUT, W> operator = new WindowOperator<>(
			assigner,
			assigner.getWindowSerializer(new ExecutionConfig()),
			keySelector,
			IntSerializer.INSTANCE,
			stateDescriptor,
			windowFunction,
			trigger,
			allowedLatenss,
			null /* late output tag */);

	return new KeyedOneInputStreamOperatorTestHarness<>(
			operator,
			keySelector,
			BasicTypeInfo.INT_TYPE_INFO);
}
 

/**
 * Verifies that pipelines including {@link CoGroupedStreams} can be checkpointed properly,
 * which includes snapshotting configurations of any involved serializers.
 *
 * @see <a href="https://issues.apache.org/jira/browse/FLINK-6808">FLINK-6808</a>
 */
@Test
public void testCoGroupOperatorWithCheckpoint() throws Exception {

	// generate an operator for the co-group operation
	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
	env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);
	env.setParallelism(1);

	DataStream<Tuple2<String, Integer>> source1 = env.fromElements(Tuple2.of("a", 0), Tuple2.of("b", 3));
	DataStream<Tuple2<String, Integer>> source2 = env.fromElements(Tuple2.of("a", 1), Tuple2.of("b", 6));

	DataStream<String> coGroupWindow = source1.coGroup(source2)
		.where(new Tuple2KeyExtractor())
		.equalTo(new Tuple2KeyExtractor())
		.window(TumblingEventTimeWindows.of(Time.of(3, TimeUnit.MILLISECONDS)))
		.apply(new CoGroupFunction<Tuple2<String, Integer>, Tuple2<String, Integer>, String>() {
			@Override
			public void coGroup(Iterable<Tuple2<String, Integer>> first,
								Iterable<Tuple2<String, Integer>> second,
								Collector<String> out) throws Exception {
				out.collect(first + ":" + second);
			}
		});

	OneInputTransformation<Tuple2<String, Integer>, String> transform = (OneInputTransformation<Tuple2<String, Integer>, String>) coGroupWindow.getTransformation();
	OneInputStreamOperator<Tuple2<String, Integer>, String> operator = transform.getOperator();

	// wrap the operator in the test harness, and perform a snapshot
	OneInputStreamOperatorTestHarness<Tuple2<String, Integer>, String> testHarness =
		new KeyedOneInputStreamOperatorTestHarness<>(operator, new Tuple2KeyExtractor(), BasicTypeInfo.STRING_TYPE_INFO);

	testHarness.open();
	testHarness.snapshot(0L, 0L);
}