Java Code Examples for org.apache.flink.streaming.api.windowing.assigners.TumblingEventTimeWindows#of()

@Test
@SuppressWarnings("unchecked")
public void testTumblingEventTimeWindowsReduce() throws Exception {
	closeCalled.set(0);

	final int windowSize = 3;

	ReducingStateDescriptor<Tuple2<String, Integer>> stateDesc = new ReducingStateDescriptor<>("window-contents",
			new SumReducer(),
			STRING_INT_TUPLE.createSerializer(new ExecutionConfig()));

	WindowOperator<String, Tuple2<String, Integer>, Tuple2<String, Integer>, Tuple2<String, Integer>, TimeWindow> operator = new WindowOperator<>(
			TumblingEventTimeWindows.of(Time.of(windowSize, TimeUnit.SECONDS)),
			new TimeWindow.Serializer(),
			new TupleKeySelector(),
			BasicTypeInfo.STRING_TYPE_INFO.createSerializer(new ExecutionConfig()),
			stateDesc,
			new InternalSingleValueWindowFunction<>(new PassThroughWindowFunction<String, TimeWindow, Tuple2<String, Integer>>()),
			EventTimeTrigger.create(),
			0,
			null /* late data output tag */);

	testTumblingEventTimeWindows(operator);
}
 

@Test
public void testTimeUnits() {
	// sanity check with one other time unit

	WindowAssigner.WindowAssignerContext mockContext =
			mock(WindowAssigner.WindowAssignerContext.class);

	TumblingEventTimeWindows assigner = TumblingEventTimeWindows.of(Time.seconds(5), Time.seconds(1));

	assertThat(assigner.assignWindows("String", 1000L, mockContext), contains(timeWindow(1000, 6000)));
	assertThat(assigner.assignWindows("String", 5999L, mockContext), contains(timeWindow(1000, 6000)));
	assertThat(assigner.assignWindows("String", 6000L, mockContext), contains(timeWindow(6000, 11000)));
}
 

@Test
public void testWindowAssignmentWithOffset() {
	WindowAssigner.WindowAssignerContext mockContext =
			mock(WindowAssigner.WindowAssignerContext.class);

	TumblingEventTimeWindows assigner = TumblingEventTimeWindows.of(Time.milliseconds(5000), Time.milliseconds(100));

	assertThat(assigner.assignWindows("String", 100L, mockContext), contains(timeWindow(100, 5100)));
	assertThat(assigner.assignWindows("String", 5099L, mockContext), contains(timeWindow(100, 5100)));
	assertThat(assigner.assignWindows("String", 5100L, mockContext), contains(timeWindow(5100, 10100)));
}
 

@Test
public void testWindowAssignmentWithNegativeOffset() {
	WindowAssigner.WindowAssignerContext mockContext =
		mock(WindowAssigner.WindowAssignerContext.class);

	TumblingEventTimeWindows assigner = TumblingEventTimeWindows.of(Time.milliseconds(5000), Time.milliseconds(-100));

	assertThat(assigner.assignWindows("String", 0L, mockContext), contains(timeWindow(-100, 4900)));
	assertThat(assigner.assignWindows("String", 4899L, mockContext), contains(timeWindow(-100, 4900)));
	assertThat(assigner.assignWindows("String", 4900L, mockContext), contains(timeWindow(4900, 9900)));
}
 

@Test
public void testTimeUnits() {
	// sanity check with one other time unit

	WindowAssigner.WindowAssignerContext mockContext =
			mock(WindowAssigner.WindowAssignerContext.class);

	TumblingEventTimeWindows assigner = TumblingEventTimeWindows.of(Time.seconds(5), Time.seconds(1));

	assertThat(assigner.assignWindows("String", 1000L, mockContext), contains(timeWindow(1000, 6000)));
	assertThat(assigner.assignWindows("String", 5999L, mockContext), contains(timeWindow(1000, 6000)));
	assertThat(assigner.assignWindows("String", 6000L, mockContext), contains(timeWindow(6000, 11000)));
}
 

@Before
public void setUp() {
	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
	dataStream1 = env.fromElements("a1", "a2", "a3");
	dataStream2 = env.fromElements("a1", "a2");
	keySelector = element -> element;
	tsAssigner = TumblingEventTimeWindows.of(Time.milliseconds(1L));
	coGroupFunction = (CoGroupFunction<String, String, String>) (first, second, out) -> out.collect("");
}
 

@Test
public void testTimeUnits() {
	// sanity check with one other time unit

	WindowAssigner.WindowAssignerContext mockContext =
			mock(WindowAssigner.WindowAssignerContext.class);

	TumblingEventTimeWindows assigner = TumblingEventTimeWindows.of(Time.seconds(5), Time.seconds(1));

	assertThat(assigner.assignWindows("String", 1000L, mockContext), contains(timeWindow(1000, 6000)));
	assertThat(assigner.assignWindows("String", 5999L, mockContext), contains(timeWindow(1000, 6000)));
	assertThat(assigner.assignWindows("String", 6000L, mockContext), contains(timeWindow(6000, 11000)));
}
 

@Test
public void testProperties() {
	TumblingEventTimeWindows assigner = TumblingEventTimeWindows.of(Time.seconds(5), Time.milliseconds(100));

	assertTrue(assigner.isEventTime());
	assertEquals(new TimeWindow.Serializer(), assigner.getWindowSerializer(new ExecutionConfig()));
	assertThat(assigner.getDefaultTrigger(mock(StreamExecutionEnvironment.class)), instanceOf(EventTimeTrigger.class));
}
 

/**
 * Tests TimeEvictor evictBefore behavior.
 */
@Test
public void testTimeEvictorEvictBefore() throws Exception {
	AtomicInteger closeCalled = new AtomicInteger(0);
	final int triggerCount = 2;
	final int windowSize = 4;

	@SuppressWarnings({"unchecked", "rawtypes"})
	TypeSerializer<StreamRecord<Tuple2<String, Integer>>> streamRecordSerializer =
		(TypeSerializer<StreamRecord<Tuple2<String, Integer>>>) new StreamElementSerializer(STRING_INT_TUPLE.createSerializer(new ExecutionConfig()));

	ListStateDescriptor<StreamRecord<Tuple2<String, Integer>>> stateDesc =
		new ListStateDescriptor<>("window-contents", streamRecordSerializer);

	EvictingWindowOperator<String, Tuple2<String, Integer>, Tuple2<String, Integer>, TimeWindow> operator = new EvictingWindowOperator<>(
		TumblingEventTimeWindows.of(Time.of(windowSize, TimeUnit.SECONDS)),
		new TimeWindow.Serializer(),
		new TupleKeySelector(),
		BasicTypeInfo.STRING_TYPE_INFO.createSerializer(new ExecutionConfig()),
		stateDesc,
		new InternalIterableWindowFunction<>(new RichSumReducer<TimeWindow>(closeCalled)),
		CountTrigger.of(triggerCount),
		TimeEvictor.of(Time.seconds(2)),
		0,
		null /* late data output tag */);

	OneInputStreamOperatorTestHarness<Tuple2<String, Integer>, Tuple2<String, Integer>> testHarness =
		new KeyedOneInputStreamOperatorTestHarness<>(operator, new TupleKeySelector(), BasicTypeInfo.STRING_TYPE_INFO);

	long initialTime = 0L;
	ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>();

	testHarness.open();

	testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 1000));
	testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 3999));

	testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 1), initialTime + 20));
	testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 1), initialTime));
	testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 1), initialTime + 999));
	testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 1), initialTime + 5999));

	testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 3500));
	testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 2001));
	testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 1001));

	expectedOutput.add(new StreamRecord<>(new Tuple2<>("key2", 1), 3999));
	expectedOutput.add(new StreamRecord<>(new Tuple2<>("key1", 2), 3999));
	expectedOutput.add(new StreamRecord<>(new Tuple2<>("key2", 3), 3999));

	TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expectedOutput, testHarness.getOutput(), new ResultSortComparator());

	testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 1), initialTime + 6500));
	testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 1002));

	expectedOutput.add(new StreamRecord<>(new Tuple2<>("key1", 2), 7999));
	expectedOutput.add(new StreamRecord<>(new Tuple2<>("key2", 3), 3999));

	TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expectedOutput, testHarness.getOutput(), new ResultSortComparator());

	testHarness.close();

	Assert.assertEquals("Close was not called.", 1, closeCalled.get());
}
 

@Test
@SuppressWarnings("unchecked")
public void testTumblingWindowWithApply() throws Exception {
	AtomicInteger closeCalled = new AtomicInteger(0);

	final int windowSize = 4;

	@SuppressWarnings({"unchecked", "rawtypes"})
	TypeSerializer<StreamRecord<Tuple2<String, Integer>>> streamRecordSerializer =
			(TypeSerializer<StreamRecord<Tuple2<String, Integer>>>) new StreamElementSerializer(STRING_INT_TUPLE.createSerializer(new ExecutionConfig()));

	ListStateDescriptor<StreamRecord<Tuple2<String, Integer>>> stateDesc =
			new ListStateDescriptor<>("window-contents", streamRecordSerializer);

	EvictingWindowOperator<String, Tuple2<String, Integer>, Tuple2<String, Integer>, TimeWindow> operator = new EvictingWindowOperator<>(
		TumblingEventTimeWindows.of(Time.of(windowSize, TimeUnit.SECONDS)),
		new TimeWindow.Serializer(),
		new TupleKeySelector(),
		BasicTypeInfo.STRING_TYPE_INFO.createSerializer(new ExecutionConfig()),
		stateDesc,
		new InternalIterableWindowFunction<>(new RichSumReducer<TimeWindow>(closeCalled)),
		EventTimeTrigger.create(),
		CountEvictor.of(windowSize),
		0,
		null /* late data output tag */);

	OneInputStreamOperatorTestHarness<Tuple2<String, Integer>, Tuple2<String, Integer>> testHarness =
			new KeyedOneInputStreamOperatorTestHarness<>(operator, new TupleKeySelector(), BasicTypeInfo.STRING_TYPE_INFO);

	long initialTime = 0L;

	testHarness.open();

	testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 1), initialTime + 10));
	testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 1), initialTime + 100));

	testHarness.processWatermark(new Watermark(1999));

	testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 1), initialTime + 1997));
	testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 1), initialTime + 1998));

	testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 1), initialTime + 2310)); // not late but more than 4
	testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 1), initialTime + 2310));

	testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 2310));
	testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 2310));

	testHarness.processWatermark(new Watermark(3999));											 // now is the evictor

	ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>();
	expectedOutput.add(new Watermark(1999));
	expectedOutput.add(new StreamRecord<>(new Tuple2<>("key1", 4), 3999));
	expectedOutput.add(new StreamRecord<>(new Tuple2<>("key2", 2), 3999));
	expectedOutput.add(new Watermark(3999));

	TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expectedOutput, testHarness.getOutput(),
		new EvictingWindowOperatorTest.ResultSortComparator());
	testHarness.close();
}
 

@Test
public void testCleanupTimerWithEmptyFoldingStateForTumblingWindows() throws Exception {
	final int windowSize = 2;
	final long lateness = 1;

	FoldingStateDescriptor<Tuple2<String, Integer>, Tuple2<String, Integer>> windowStateDesc =
		new FoldingStateDescriptor<>(
			"window-contents",
			new Tuple2<>((String) null, 0),
			new FoldFunction<Tuple2<String, Integer>, Tuple2<String, Integer>>() {
				private static final long serialVersionUID = 1L;

				@Override
				public Tuple2<String, Integer> fold(Tuple2<String, Integer> accumulator, Tuple2<String, Integer> value) throws Exception {
					return new Tuple2<>(value.f0, accumulator.f1 + value.f1);
				}
			},
			STRING_INT_TUPLE);
	windowStateDesc.initializeSerializerUnlessSet(new ExecutionConfig());

	WindowOperator<String, Tuple2<String, Integer>, Tuple2<String, Integer>, Tuple2<String, Integer>, TimeWindow> operator =
		new WindowOperator<>(
			TumblingEventTimeWindows.of(Time.of(windowSize, TimeUnit.SECONDS)),
			new TimeWindow.Serializer(),
			new TupleKeySelector(),
			BasicTypeInfo.STRING_TYPE_INFO.createSerializer(new ExecutionConfig()),
			windowStateDesc,
			new InternalSingleValueWindowFunction<>(new PassThroughFunction()),
			EventTimeTrigger.create(),
			lateness,
			null /* late data output tag */);

	OneInputStreamOperatorTestHarness<Tuple2<String, Integer>, Tuple2<String, Integer>> testHarness =
		createTestHarness(operator);

	testHarness.open();

	ConcurrentLinkedQueue<Object> expected = new ConcurrentLinkedQueue<>();

	// normal element
	testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 1000));
	testHarness.processWatermark(new Watermark(1599));
	testHarness.processWatermark(new Watermark(1999));
	testHarness.processWatermark(new Watermark(2000));
	testHarness.processWatermark(new Watermark(5000));

	expected.add(new Watermark(1599));
	expected.add(new StreamRecord<>(new Tuple2<>("key2", 1), 1999));
	expected.add(new Watermark(1999)); // here it fires and purges
	expected.add(new Watermark(2000)); // here is the cleanup timer
	expected.add(new Watermark(5000));

	TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expected, testHarness.getOutput(), new Tuple2ResultSortComparator());
	testHarness.close();
}
 

@Test
public void testCleanupTimerWithEmptyListStateForTumblingWindows() throws Exception {
	final int windowSize = 2;
	final long lateness = 1;

	ListStateDescriptor<Tuple2<String, Integer>> windowStateDesc =
		new ListStateDescriptor<>("window-contents", STRING_INT_TUPLE.createSerializer(new ExecutionConfig()));

	WindowOperator<String, Tuple2<String, Integer>, Iterable<Tuple2<String, Integer>>, Tuple2<String, Integer>, TimeWindow> operator =
		new WindowOperator<>(
			TumblingEventTimeWindows.of(Time.of(windowSize, TimeUnit.SECONDS)),
			new TimeWindow.Serializer(),
			new TupleKeySelector(),
			BasicTypeInfo.STRING_TYPE_INFO.createSerializer(new ExecutionConfig()),
			windowStateDesc,
			new InternalIterableWindowFunction<>(new PassThroughFunction()),
			EventTimeTrigger.create(),
			lateness,
			null /* late data output tag */);

	OneInputStreamOperatorTestHarness<Tuple2<String, Integer>, Tuple2<String, Integer>> testHarness =
		createTestHarness(operator);

	testHarness.open();

	ConcurrentLinkedQueue<Object> expected = new ConcurrentLinkedQueue<>();

	// normal element
	testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 1000));
	testHarness.processWatermark(new Watermark(1599));
	testHarness.processWatermark(new Watermark(1999));
	testHarness.processWatermark(new Watermark(2000));
	testHarness.processWatermark(new Watermark(5000));

	expected.add(new Watermark(1599));
	expected.add(new StreamRecord<>(new Tuple2<>("key2", 1), 1999));
	expected.add(new Watermark(1999)); // here it fires and purges
	expected.add(new Watermark(2000)); // here is the cleanup timer
	expected.add(new Watermark(5000));

	TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expected, testHarness.getOutput(), new Tuple2ResultSortComparator());
	testHarness.close();
}
 

@Test
public void testCleanupTimerWithEmptyListStateForTumblingWindows2() throws Exception {
	final int windowSize = 2;
	final long lateness = 100;

	ListStateDescriptor<Tuple2<String, Integer>> windowStateDesc =
		new ListStateDescriptor<>("window-contents", STRING_INT_TUPLE.createSerializer(new ExecutionConfig()));

	WindowOperator<String, Tuple2<String, Integer>, Iterable<Tuple2<String, Integer>>, String, TimeWindow> operator =
		new WindowOperator<>(
			TumblingEventTimeWindows.of(Time.of(windowSize, TimeUnit.SECONDS)),
			new TimeWindow.Serializer(),
			new TupleKeySelector(),
			BasicTypeInfo.STRING_TYPE_INFO.createSerializer(new ExecutionConfig()),
			windowStateDesc,
			new InternalIterableWindowFunction<>(new PassThroughFunction2()),
				new EventTimeTriggerAccumGC(lateness),
			lateness,
			null /* late data output tag */);

	OneInputStreamOperatorTestHarness<Tuple2<String, Integer>, String> testHarness =
		createTestHarness(operator);

	testHarness.open();

	ConcurrentLinkedQueue<Object> expected = new ConcurrentLinkedQueue<>();

	// normal element
	testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 1000));
	testHarness.processWatermark(new Watermark(1599));
	testHarness.processWatermark(new Watermark(1999));
	testHarness.processWatermark(new Watermark(2100));
	testHarness.processWatermark(new Watermark(5000));

	expected.add(new Watermark(1599));
	expected.add(new StreamRecord<>("GOT: (key2,1)", 1999));
	expected.add(new Watermark(1999)); // here it fires and purges
	expected.add(new Watermark(2100)); // here is the cleanup timer
	expected.add(new Watermark(5000));

	TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expected, testHarness.getOutput(), new Tuple2ResultSortComparator());
	testHarness.close();
}
 

@Test
public void testLateness() throws Exception {
	final int windowSize = 2;
	final long lateness = 500;

	ReducingStateDescriptor<Tuple2<String, Integer>> stateDesc = new ReducingStateDescriptor<>("window-contents",
		new SumReducer(),
			STRING_INT_TUPLE.createSerializer(new ExecutionConfig()));

	WindowOperator<String, Tuple2<String, Integer>, Tuple2<String, Integer>, Tuple2<String, Integer>, TimeWindow> operator =
		new WindowOperator<>(
			TumblingEventTimeWindows.of(Time.of(windowSize, TimeUnit.SECONDS)),
			new TimeWindow.Serializer(),
			new TupleKeySelector(),
			BasicTypeInfo.STRING_TYPE_INFO.createSerializer(new ExecutionConfig()),
			stateDesc,
			new InternalSingleValueWindowFunction<>(new PassThroughWindowFunction<String, TimeWindow, Tuple2<String, Integer>>()),
			PurgingTrigger.of(EventTimeTrigger.create()),
			lateness,
			lateOutputTag);

	OneInputStreamOperatorTestHarness<Tuple2<String, Integer>, Tuple2<String, Integer>> testHarness =
		createTestHarness(operator);

	testHarness.open();

	ConcurrentLinkedQueue<Object> expected = new ConcurrentLinkedQueue<>();
	ConcurrentLinkedQueue<Object> lateExpected = new ConcurrentLinkedQueue<>();

	testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 500));
	testHarness.processWatermark(new Watermark(1500));

	expected.add(new Watermark(1500));

	testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 1300));
	testHarness.processWatermark(new Watermark(2300));

	expected.add(new StreamRecord<>(new Tuple2<>("key2", 2), 1999));
	expected.add(new Watermark(2300));

	// this will not be sideoutput because window.maxTimestamp() + allowedLateness > currentWatermark
	testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 1997));
	testHarness.processWatermark(new Watermark(6000));

	// this is 1 and not 3 because the trigger fires and purges
	expected.add(new StreamRecord<>(new Tuple2<>("key2", 1), 1999));
	expected.add(new Watermark(6000));

	// this will be side output because window.maxTimestamp() + allowedLateness < currentWatermark
	testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 1998));
	testHarness.processWatermark(new Watermark(7000));

	lateExpected.add(new StreamRecord<>(new Tuple2<>("key2", 1), 1998));
	expected.add(new Watermark(7000));

	TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expected, testHarness.getOutput(), new Tuple2ResultSortComparator());

	TestHarnessUtil.assertOutputEqualsSorted(
			"SideOutput was not correct.",
			lateExpected,
			(Iterable) testHarness.getSideOutput(lateOutputTag),
			new Tuple2ResultSortComparator());

	testHarness.close();
}
 

@Test
public void testLateness() throws Exception {
	final int windowSize = 2;
	final long lateness = 500;

	ReducingStateDescriptor<Tuple2<String, Integer>> stateDesc = new ReducingStateDescriptor<>("window-contents",
		new SumReducer(),
			STRING_INT_TUPLE.createSerializer(new ExecutionConfig()));

	WindowOperator<String, Tuple2<String, Integer>, Tuple2<String, Integer>, Tuple2<String, Integer>, TimeWindow> operator =
		new WindowOperator<>(
			TumblingEventTimeWindows.of(Time.of(windowSize, TimeUnit.SECONDS)),
			new TimeWindow.Serializer(),
			new TupleKeySelector(),
			BasicTypeInfo.STRING_TYPE_INFO.createSerializer(new ExecutionConfig()),
			stateDesc,
			new InternalSingleValueWindowFunction<>(new PassThroughWindowFunction<String, TimeWindow, Tuple2<String, Integer>>()),
			PurgingTrigger.of(EventTimeTrigger.create()),
			lateness,
			lateOutputTag);

	OneInputStreamOperatorTestHarness<Tuple2<String, Integer>, Tuple2<String, Integer>> testHarness =
		createTestHarness(operator);

	testHarness.open();

	ConcurrentLinkedQueue<Object> expected = new ConcurrentLinkedQueue<>();
	ConcurrentLinkedQueue<Object> lateExpected = new ConcurrentLinkedQueue<>();

	testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 500));
	testHarness.processWatermark(new Watermark(1500));

	expected.add(new Watermark(1500));

	testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 1300));
	testHarness.processWatermark(new Watermark(2300));

	expected.add(new StreamRecord<>(new Tuple2<>("key2", 2), 1999));
	expected.add(new Watermark(2300));

	// this will not be sideoutput because window.maxTimestamp() + allowedLateness > currentWatermark
	testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 1997));
	testHarness.processWatermark(new Watermark(6000));

	// this is 1 and not 3 because the trigger fires and purges
	expected.add(new StreamRecord<>(new Tuple2<>("key2", 1), 1999));
	expected.add(new Watermark(6000));

	// this will be side output because window.maxTimestamp() + allowedLateness < currentWatermark
	testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 1998));
	testHarness.processWatermark(new Watermark(7000));

	lateExpected.add(new StreamRecord<>(new Tuple2<>("key2", 1), 1998));
	expected.add(new Watermark(7000));

	TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expected, testHarness.getOutput(), new Tuple2ResultSortComparator());

	TestHarnessUtil.assertOutputEqualsSorted(
			"SideOutput was not correct.",
			lateExpected,
			(Iterable) testHarness.getSideOutput(lateOutputTag),
			new Tuple2ResultSortComparator());

	testHarness.close();
}
 

@Test
public void testCleanupTimerWithEmptyFoldingStateForTumblingWindows() throws Exception {
	final int windowSize = 2;
	final long lateness = 1;

	FoldingStateDescriptor<Tuple2<String, Integer>, Tuple2<String, Integer>> windowStateDesc =
		new FoldingStateDescriptor<>(
			"window-contents",
			new Tuple2<>((String) null, 0),
			new FoldFunction<Tuple2<String, Integer>, Tuple2<String, Integer>>() {
				private static final long serialVersionUID = 1L;

				@Override
				public Tuple2<String, Integer> fold(Tuple2<String, Integer> accumulator, Tuple2<String, Integer> value) throws Exception {
					return new Tuple2<>(value.f0, accumulator.f1 + value.f1);
				}
			},
			STRING_INT_TUPLE);
	windowStateDesc.initializeSerializerUnlessSet(new ExecutionConfig());

	WindowOperator<String, Tuple2<String, Integer>, Tuple2<String, Integer>, Tuple2<String, Integer>, TimeWindow> operator =
		new WindowOperator<>(
			TumblingEventTimeWindows.of(Time.of(windowSize, TimeUnit.SECONDS)),
			new TimeWindow.Serializer(),
			new TupleKeySelector(),
			BasicTypeInfo.STRING_TYPE_INFO.createSerializer(new ExecutionConfig()),
			windowStateDesc,
			new InternalSingleValueWindowFunction<>(new PassThroughFunction()),
			EventTimeTrigger.create(),
			lateness,
			null /* late data output tag */);

	OneInputStreamOperatorTestHarness<Tuple2<String, Integer>, Tuple2<String, Integer>> testHarness =
		createTestHarness(operator);

	testHarness.open();

	ConcurrentLinkedQueue<Object> expected = new ConcurrentLinkedQueue<>();

	// normal element
	testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 1000));
	testHarness.processWatermark(new Watermark(1599));
	testHarness.processWatermark(new Watermark(1999));
	testHarness.processWatermark(new Watermark(2000));
	testHarness.processWatermark(new Watermark(5000));

	expected.add(new Watermark(1599));
	expected.add(new StreamRecord<>(new Tuple2<>("key2", 1), 1999));
	expected.add(new Watermark(1999)); // here it fires and purges
	expected.add(new Watermark(2000)); // here is the cleanup timer
	expected.add(new Watermark(5000));

	TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expected, testHarness.getOutput(), new Tuple2ResultSortComparator());
	testHarness.close();
}
 

/**
 * Manually run this to write binary snapshot data.
 */
@Ignore
@Test
public void writeApplyEventTimeWindowsSnapshot() throws Exception {
	final int windowSize = 3;

	ListStateDescriptor<Tuple2<String, Integer>> stateDesc = new ListStateDescriptor<>("window-contents",
			STRING_INT_TUPLE.createSerializer(new ExecutionConfig()));

	WindowOperator<String, Tuple2<String, Integer>, Iterable<Tuple2<String, Integer>>, Tuple2<String, Integer>, TimeWindow> operator = new WindowOperator<>(
			TumblingEventTimeWindows.of(Time.of(windowSize, TimeUnit.SECONDS)),
			new TimeWindow.Serializer(),
			new TupleKeySelector<>(),
			BasicTypeInfo.STRING_TYPE_INFO.createSerializer(new ExecutionConfig()),
			stateDesc,
			new InternalIterableWindowFunction<>(new RichSumReducer<TimeWindow>()),
			EventTimeTrigger.create(),
			0,
			null /* late data output tag */);

	ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>();

	OneInputStreamOperatorTestHarness<Tuple2<String, Integer>, Tuple2<String, Integer>> testHarness =
			new KeyedOneInputStreamOperatorTestHarness<>(operator, new TupleKeySelector<>(), BasicTypeInfo.STRING_TYPE_INFO);

	testHarness.setup();
	testHarness.open();

	// add elements out-of-order
	testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 3999));
	testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 3000));

	testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 1), 20));
	testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 1), 0));
	testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 1), 999));

	testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 1998));
	testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 1999));
	testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 1000));

	testHarness.processWatermark(new Watermark(999));
	expectedOutput.add(new Watermark(999));
	TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expectedOutput, testHarness.getOutput(), new Tuple2ResultSortComparator<>());

	testHarness.processWatermark(new Watermark(1999));
	expectedOutput.add(new Watermark(1999));
	TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expectedOutput, testHarness.getOutput(), new Tuple2ResultSortComparator<>());

	// do snapshot and save to file
	OperatorSubtaskState snapshot = testHarness.snapshot(0, 0);
	OperatorSnapshotUtil.writeStateHandle(
		snapshot,
		"src/test/resources/win-op-migration-test-apply-event-time-flink" + flinkGenerateSavepointVersion + "-snapshot");

	testHarness.close();
}
 

@Test
@SuppressWarnings("unchecked")
public void testTumblingWindowWithApply() throws Exception {
	AtomicInteger closeCalled = new AtomicInteger(0);

	final int windowSize = 4;

	@SuppressWarnings({"unchecked", "rawtypes"})
	TypeSerializer<StreamRecord<Tuple2<String, Integer>>> streamRecordSerializer =
			(TypeSerializer<StreamRecord<Tuple2<String, Integer>>>) new StreamElementSerializer(STRING_INT_TUPLE.createSerializer(new ExecutionConfig()));

	ListStateDescriptor<StreamRecord<Tuple2<String, Integer>>> stateDesc =
			new ListStateDescriptor<>("window-contents", streamRecordSerializer);

	EvictingWindowOperator<String, Tuple2<String, Integer>, Tuple2<String, Integer>, TimeWindow> operator = new EvictingWindowOperator<>(
		TumblingEventTimeWindows.of(Time.of(windowSize, TimeUnit.SECONDS)),
		new TimeWindow.Serializer(),
		new TupleKeySelector(),
		BasicTypeInfo.STRING_TYPE_INFO.createSerializer(new ExecutionConfig()),
		stateDesc,
		new InternalIterableWindowFunction<>(new RichSumReducer<TimeWindow>(closeCalled)),
		EventTimeTrigger.create(),
		CountEvictor.of(windowSize),
		0,
		null /* late data output tag */);

	OneInputStreamOperatorTestHarness<Tuple2<String, Integer>, Tuple2<String, Integer>> testHarness =
			new KeyedOneInputStreamOperatorTestHarness<>(operator, new TupleKeySelector(), BasicTypeInfo.STRING_TYPE_INFO);

	long initialTime = 0L;

	testHarness.open();

	testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 1), initialTime + 10));
	testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 1), initialTime + 100));

	testHarness.processWatermark(new Watermark(1999));

	testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 1), initialTime + 1997));
	testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 1), initialTime + 1998));

	testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 1), initialTime + 2310)); // not late but more than 4
	testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 1), initialTime + 2310));

	testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 2310));
	testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 2310));

	testHarness.processWatermark(new Watermark(3999));											 // now is the evictor

	ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>();
	expectedOutput.add(new Watermark(1999));
	expectedOutput.add(new StreamRecord<>(new Tuple2<>("key1", 4), 3999));
	expectedOutput.add(new StreamRecord<>(new Tuple2<>("key2", 2), 3999));
	expectedOutput.add(new Watermark(3999));

	TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expectedOutput, testHarness.getOutput(),
		new EvictingWindowOperatorTest.ResultSortComparator());
	testHarness.close();
}
 

@Test
@SuppressWarnings("unchecked")
public void testTumblingWindowWithApply() throws Exception {
	AtomicInteger closeCalled = new AtomicInteger(0);

	final int windowSize = 4;

	@SuppressWarnings({"unchecked", "rawtypes"})
	TypeSerializer<StreamRecord<Tuple2<String, Integer>>> streamRecordSerializer =
			(TypeSerializer<StreamRecord<Tuple2<String, Integer>>>) new StreamElementSerializer(STRING_INT_TUPLE.createSerializer(new ExecutionConfig()));

	ListStateDescriptor<StreamRecord<Tuple2<String, Integer>>> stateDesc =
			new ListStateDescriptor<>("window-contents", streamRecordSerializer);

	EvictingWindowOperator<String, Tuple2<String, Integer>, Tuple2<String, Integer>, TimeWindow> operator = new EvictingWindowOperator<>(
		TumblingEventTimeWindows.of(Time.of(windowSize, TimeUnit.SECONDS)),
		new TimeWindow.Serializer(),
		new TupleKeySelector(),
		BasicTypeInfo.STRING_TYPE_INFO.createSerializer(new ExecutionConfig()),
		stateDesc,
		new InternalIterableWindowFunction<>(new RichSumReducer<TimeWindow>(closeCalled)),
		EventTimeTrigger.create(),
		CountEvictor.of(windowSize),
		0,
		null /* late data output tag */);

	OneInputStreamOperatorTestHarness<Tuple2<String, Integer>, Tuple2<String, Integer>> testHarness =
			new KeyedOneInputStreamOperatorTestHarness<>(operator, new TupleKeySelector(), BasicTypeInfo.STRING_TYPE_INFO);

	long initialTime = 0L;

	testHarness.open();

	testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 1), initialTime + 10));
	testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 1), initialTime + 100));

	testHarness.processWatermark(new Watermark(1999));

	testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 1), initialTime + 1997));
	testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 1), initialTime + 1998));

	testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 1), initialTime + 2310)); // not late but more than 4
	testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 1), initialTime + 2310));

	testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 2310));
	testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 2310));

	testHarness.processWatermark(new Watermark(3999));											 // now is the evictor

	ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>();
	expectedOutput.add(new Watermark(1999));
	expectedOutput.add(new StreamRecord<>(new Tuple2<>("key1", 4), 3999));
	expectedOutput.add(new StreamRecord<>(new Tuple2<>("key2", 2), 3999));
	expectedOutput.add(new Watermark(3999));

	TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expectedOutput, testHarness.getOutput(),
		new EvictingWindowOperatorTest.ResultSortComparator());
	testHarness.close();
}
 

/**
 * Manually run this to write binary snapshot data.
 */
@Ignore
@Test
public void writeWindowsWithKryoSerializedKeysSnapshot() throws Exception {
	final int windowSize = 3;

	TypeInformation<Tuple2<NonPojoType, Integer>> inputType = new TypeHint<Tuple2<NonPojoType, Integer>>() {}.getTypeInfo();

	ReducingStateDescriptor<Tuple2<NonPojoType, Integer>> stateDesc = new ReducingStateDescriptor<>("window-contents",
		new SumReducer<>(),
		inputType.createSerializer(new ExecutionConfig()));

	TypeSerializer<NonPojoType> keySerializer = TypeInformation.of(NonPojoType.class).createSerializer(new ExecutionConfig());
	assertTrue(keySerializer instanceof KryoSerializer);

	WindowOperator<NonPojoType, Tuple2<NonPojoType, Integer>, Tuple2<NonPojoType, Integer>, Tuple2<NonPojoType, Integer>, TimeWindow> operator = new WindowOperator<>(
		TumblingEventTimeWindows.of(Time.of(windowSize, TimeUnit.SECONDS)),
		new TimeWindow.Serializer(),
		new TupleKeySelector<>(),
		keySerializer,
		stateDesc,
		new InternalSingleValueWindowFunction<>(new PassThroughWindowFunction<>()),
		EventTimeTrigger.create(),
		0,
		null /* late data output tag */);

	ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>();

	OneInputStreamOperatorTestHarness<Tuple2<NonPojoType, Integer>, Tuple2<NonPojoType, Integer>> testHarness =
		new KeyedOneInputStreamOperatorTestHarness<>(operator, new TupleKeySelector<>(), TypeInformation.of(NonPojoType.class));

	testHarness.setup();
	testHarness.open();

	// add elements out-of-order
	testHarness.processElement(new StreamRecord<>(new Tuple2<>(new NonPojoType("key2"), 1), 3999));
	testHarness.processElement(new StreamRecord<>(new Tuple2<>(new NonPojoType("key2"), 1), 3000));

	testHarness.processElement(new StreamRecord<>(new Tuple2<>(new NonPojoType("key1"), 1), 20));
	testHarness.processElement(new StreamRecord<>(new Tuple2<>(new NonPojoType("key1"), 1), 0));
	testHarness.processElement(new StreamRecord<>(new Tuple2<>(new NonPojoType("key1"), 1), 999));

	testHarness.processElement(new StreamRecord<>(new Tuple2<>(new NonPojoType("key2"), 1), 1998));
	testHarness.processElement(new StreamRecord<>(new Tuple2<>(new NonPojoType("key2"), 1), 1999));
	testHarness.processElement(new StreamRecord<>(new Tuple2<>(new NonPojoType("key2"), 1), 1000));

	testHarness.processWatermark(new Watermark(999));
	expectedOutput.add(new Watermark(999));
	TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expectedOutput, testHarness.getOutput(), new Tuple2ResultSortComparator<>());

	testHarness.processWatermark(new Watermark(1999));
	expectedOutput.add(new Watermark(1999));
	TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expectedOutput, testHarness.getOutput(), new Tuple2ResultSortComparator<>());

	// do snapshot and save to file
	OperatorSubtaskState snapshot = testHarness.snapshot(0, 0);
	OperatorSnapshotUtil.writeStateHandle(
		snapshot,
		"src/test/resources/win-op-migration-test-kryo-serialized-key-flink" + flinkGenerateSavepointVersion + "-snapshot");

	testHarness.close();
}