Java Code Examples for org.apache.flink.streaming.util.TestHarnessUtil#assertOutputEqualsSorted()
The following examples show how to use
org.apache.flink.streaming.util.TestHarnessUtil#assertOutputEqualsSorted() .
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: WindowOperatorTest.java From flink with Apache License 2.0 | 5 votes |
@Test public void testCleanupTimerWithEmptyListStateForSessionWindows() throws Exception { final int gapSize = 3; final long lateness = 10; ListStateDescriptor<Tuple2<String, Integer>> windowStateDesc = new ListStateDescriptor<>("window-contents", STRING_INT_TUPLE.createSerializer(new ExecutionConfig())); WindowOperator<String, Tuple2<String, Integer>, Iterable<Tuple2<String, Integer>>, Tuple2<String, Integer>, TimeWindow> operator = new WindowOperator<>( EventTimeSessionWindows.withGap(Time.seconds(gapSize)), new TimeWindow.Serializer(), new TupleKeySelector(), BasicTypeInfo.STRING_TYPE_INFO.createSerializer(new ExecutionConfig()), windowStateDesc, new InternalIterableWindowFunction<>(new PassThroughFunction()), EventTimeTrigger.create(), lateness, null /* late data output tag */); OneInputStreamOperatorTestHarness<Tuple2<String, Integer>, Tuple2<String, Integer>> testHarness = createTestHarness(operator); testHarness.open(); ConcurrentLinkedQueue<Object> expected = new ConcurrentLinkedQueue<>(); testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 1000)); testHarness.processWatermark(new Watermark(4998)); expected.add(new StreamRecord<>(new Tuple2<>("key2", 1), 3999)); expected.add(new Watermark(4998)); testHarness.processWatermark(new Watermark(14600)); expected.add(new Watermark(14600)); ConcurrentLinkedQueue<Object> actual = testHarness.getOutput(); TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expected, actual, new Tuple2ResultSortComparator()); testHarness.close(); }
Example 2
Source File: EvictingWindowOperatorTest.java From flink with Apache License 2.0 | 4 votes |
@Test @SuppressWarnings("unchecked") public void testCountTriggerWithApply() throws Exception { AtomicInteger closeCalled = new AtomicInteger(0); final int windowSize = 4; final int windowSlide = 2; @SuppressWarnings({"unchecked", "rawtypes"}) TypeSerializer<StreamRecord<Tuple2<String, Integer>>> streamRecordSerializer = (TypeSerializer<StreamRecord<Tuple2<String, Integer>>>) new StreamElementSerializer(STRING_INT_TUPLE.createSerializer(new ExecutionConfig())); ListStateDescriptor<StreamRecord<Tuple2<String, Integer>>> stateDesc = new ListStateDescriptor<>("window-contents", streamRecordSerializer); EvictingWindowOperator<String, Tuple2<String, Integer>, Tuple2<String, Integer>, GlobalWindow> operator = new EvictingWindowOperator<>( GlobalWindows.create(), new GlobalWindow.Serializer(), new TupleKeySelector(), BasicTypeInfo.STRING_TYPE_INFO.createSerializer(new ExecutionConfig()), stateDesc, new InternalIterableWindowFunction<>(new RichSumReducer<GlobalWindow>(closeCalled)), CountTrigger.of(windowSlide), CountEvictor.of(windowSize), 0, null /* late data output tag */); OneInputStreamOperatorTestHarness<Tuple2<String, Integer>, Tuple2<String, Integer>> testHarness = new KeyedOneInputStreamOperatorTestHarness<>(operator, new TupleKeySelector(), BasicTypeInfo.STRING_TYPE_INFO); long initialTime = 0L; ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>(); testHarness.open(); // The global window actually ignores these timestamps... // add elements out-of-order testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 3000)); testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 3999)); testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 1), initialTime + 20)); testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 1), initialTime)); testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 1), initialTime + 999)); testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 1998)); testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 1999)); testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 1000)); expectedOutput.add(new StreamRecord<>(new Tuple2<>("key2", 2), Long.MAX_VALUE)); expectedOutput.add(new StreamRecord<>(new Tuple2<>("key2", 4), Long.MAX_VALUE)); expectedOutput.add(new StreamRecord<>(new Tuple2<>("key1", 2), Long.MAX_VALUE)); TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expectedOutput, testHarness.getOutput(), new ResultSortComparator()); testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 1), initialTime + 10999)); testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 1000)); expectedOutput.add(new StreamRecord<>(new Tuple2<>("key1", 4), Long.MAX_VALUE)); expectedOutput.add(new StreamRecord<>(new Tuple2<>("key2", 4), Long.MAX_VALUE)); TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expectedOutput, testHarness.getOutput(), new ResultSortComparator()); testHarness.close(); Assert.assertEquals("Close was not called.", 1, closeCalled.get()); }
Example 3
Source File: WindowOperatorMigrationTest.java From flink with Apache License 2.0 | 4 votes |
@Test public void testRestoreReducingProcessingTimeWindows() throws Exception { final int windowSize = 3; ReducingStateDescriptor<Tuple2<String, Integer>> stateDesc = new ReducingStateDescriptor<>("window-contents", new SumReducer<>(), STRING_INT_TUPLE.createSerializer(new ExecutionConfig())); WindowOperator<String, Tuple2<String, Integer>, Tuple2<String, Integer>, Tuple2<String, Integer>, TimeWindow> operator = new WindowOperator<>( TumblingProcessingTimeWindows.of(Time.of(windowSize, TimeUnit.SECONDS)), new TimeWindow.Serializer(), new TupleKeySelector<>(), BasicTypeInfo.STRING_TYPE_INFO.createSerializer(new ExecutionConfig()), stateDesc, new InternalSingleValueWindowFunction<>(new PassThroughWindowFunction<String, TimeWindow, Tuple2<String, Integer>>()), ProcessingTimeTrigger.create(), 0, null /* late data output tag */); ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>(); OneInputStreamOperatorTestHarness<Tuple2<String, Integer>, Tuple2<String, Integer>> testHarness = new KeyedOneInputStreamOperatorTestHarness<>(operator, new TupleKeySelector<>(), BasicTypeInfo.STRING_TYPE_INFO); testHarness.setup(); testHarness.initializeState( OperatorSnapshotUtil.getResourceFilename( "win-op-migration-test-reduce-processing-time-flink" + testMigrateVersion + "-snapshot")); testHarness.open(); testHarness.setProcessingTime(3020); testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 3))); testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 3))); testHarness.setProcessingTime(6000); expectedOutput.add(new StreamRecord<>(new Tuple2<>("key1", 3), 5999)); expectedOutput.add(new StreamRecord<>(new Tuple2<>("key2", 4), 5999)); expectedOutput.add(new StreamRecord<>(new Tuple2<>("key3", 1), 5999)); TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expectedOutput, testHarness.getOutput(), new Tuple2ResultSortComparator<>()); testHarness.close(); }
Example 4
Source File: WindowOperatorTest.java From flink with Apache License 2.0 | 4 votes |
@Test @SuppressWarnings("unchecked") public void testReduceSessionWindows() throws Exception { closeCalled.set(0); final int sessionSize = 3; ReducingStateDescriptor<Tuple2<String, Integer>> stateDesc = new ReducingStateDescriptor<>( "window-contents", new SumReducer(), STRING_INT_TUPLE.createSerializer(new ExecutionConfig())); WindowOperator<String, Tuple2<String, Integer>, Tuple2<String, Integer>, Tuple3<String, Long, Long>, TimeWindow> operator = new WindowOperator<>( EventTimeSessionWindows.withGap(Time.seconds(sessionSize)), new TimeWindow.Serializer(), new TupleKeySelector(), BasicTypeInfo.STRING_TYPE_INFO.createSerializer(new ExecutionConfig()), stateDesc, new InternalSingleValueWindowFunction<>(new ReducedSessionWindowFunction()), EventTimeTrigger.create(), 0, null /* late data output tag */); OneInputStreamOperatorTestHarness<Tuple2<String, Integer>, Tuple3<String, Long, Long>> testHarness = createTestHarness(operator); ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>(); testHarness.open(); // add elements out-of-order testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 0)); testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 2), 1000)); testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 3), 2500)); // do a snapshot, close and restore again OperatorSubtaskState snapshot = testHarness.snapshot(0L, 0L); testHarness.close(); testHarness = createTestHarness(operator); testHarness.setup(); testHarness.initializeState(snapshot); testHarness.open(); testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 1), 10)); testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 2), 1000)); testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 3), 2500)); testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 4), 5501)); testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 5), 6000)); testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 5), 6000)); testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 6), 6050)); testHarness.processWatermark(new Watermark(12000)); expectedOutput.add(new StreamRecord<>(new Tuple3<>("key1-6", 10L, 5500L), 5499)); expectedOutput.add(new StreamRecord<>(new Tuple3<>("key2-6", 0L, 5500L), 5499)); expectedOutput.add(new StreamRecord<>(new Tuple3<>("key2-20", 5501L, 9050L), 9049)); expectedOutput.add(new Watermark(12000)); testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 10), 15000)); testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 20), 15000)); testHarness.processWatermark(new Watermark(17999)); expectedOutput.add(new StreamRecord<>(new Tuple3<>("key2-30", 15000L, 18000L), 17999)); expectedOutput.add(new Watermark(17999)); TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expectedOutput, testHarness.getOutput(), new Tuple3ResultSortComparator()); testHarness.close(); }
Example 5
Source File: WindowOperatorTest.java From Flink-CEPplus with Apache License 2.0 | 4 votes |
@Test @SuppressWarnings("unchecked") public void testCountTrigger() throws Exception { closeCalled.set(0); final int windowSize = 4; ReducingStateDescriptor<Tuple2<String, Integer>> stateDesc = new ReducingStateDescriptor<>("window-contents", new SumReducer(), STRING_INT_TUPLE.createSerializer(new ExecutionConfig())); WindowOperator<String, Tuple2<String, Integer>, Tuple2<String, Integer>, Tuple2<String, Integer>, GlobalWindow> operator = new WindowOperator<>( GlobalWindows.create(), new GlobalWindow.Serializer(), new TupleKeySelector(), BasicTypeInfo.STRING_TYPE_INFO.createSerializer(new ExecutionConfig()), stateDesc, new InternalSingleValueWindowFunction<>(new PassThroughWindowFunction<String, GlobalWindow, Tuple2<String, Integer>>()), PurgingTrigger.of(CountTrigger.of(windowSize)), 0, null /* late data output tag */); OneInputStreamOperatorTestHarness<Tuple2<String, Integer>, Tuple2<String, Integer>> testHarness = createTestHarness(operator); ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>(); testHarness.open(); // The global window actually ignores these timestamps... // add elements out-of-order testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 3000)); testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 3999)); testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 1), 20)); testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 1), 0)); testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 1), 999)); testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 1998)); testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 1999)); // do a snapshot, close and restore again OperatorSubtaskState snapshot = testHarness.snapshot(0L, 0L); testHarness.close(); ConcurrentLinkedQueue<Object> outputBeforeClose = testHarness.getOutput(); stateDesc = new ReducingStateDescriptor<>("window-contents", new SumReducer(), STRING_INT_TUPLE.createSerializer(new ExecutionConfig())); operator = new WindowOperator<>( GlobalWindows.create(), new GlobalWindow.Serializer(), new TupleKeySelector(), BasicTypeInfo.STRING_TYPE_INFO.createSerializer(new ExecutionConfig()), stateDesc, new InternalSingleValueWindowFunction<>(new PassThroughWindowFunction<String, GlobalWindow, Tuple2<String, Integer>>()), PurgingTrigger.of(CountTrigger.of(windowSize)), 0, null /* late data output tag */); testHarness = createTestHarness(operator); testHarness.setup(); testHarness.initializeState(snapshot); testHarness.open(); testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 1000)); expectedOutput.add(new StreamRecord<>(new Tuple2<>("key2", 4), Long.MAX_VALUE)); TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expectedOutput, Iterables.concat(outputBeforeClose, testHarness.getOutput()), new Tuple2ResultSortComparator()); testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 1), 10999)); testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 1000)); testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 1000)); testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 1000)); expectedOutput.add(new StreamRecord<>(new Tuple2<>("key1", 4), Long.MAX_VALUE)); expectedOutput.add(new StreamRecord<>(new Tuple2<>("key2", 4), Long.MAX_VALUE)); TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expectedOutput, Iterables.concat(outputBeforeClose, testHarness.getOutput()), new Tuple2ResultSortComparator()); testHarness.close(); }
Example 6
Source File: GroupAlsoByWindowTest.java From flink-dataflow with Apache License 2.0 | 4 votes |
@Test public void testSlidingWindows() throws Exception { WindowingStrategy strategy = slidingWindowWithAfterWatermarkTriggerStrategy; long initialTime = 0L; OneInputStreamOperatorTestHarness<WindowedValue<KV<String, Integer>>, WindowedValue<KV<String, Integer>>> testHarness = createTestingOperatorAndState(strategy, initialTime); ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>(); testHarness.processWatermark(new Watermark(initialTime + 25000)); expectedOutput.add(new StreamRecord<>( WindowedValue.of(KV.of("key1", 6), new Instant(initialTime + 5000), new IntervalWindow(new Instant(0), new Instant(10000)), PaneInfo.createPane(true, true, PaneInfo.Timing.ON_TIME)) , initialTime + 5000)); expectedOutput.add(new StreamRecord<>( WindowedValue.of(KV.of("key1", 6), new Instant(initialTime + 1), new IntervalWindow(new Instant(-5000), new Instant(5000)), PaneInfo.createPane(true, true, PaneInfo.Timing.ON_TIME)) , initialTime + 1)); expectedOutput.add(new Watermark(initialTime + 10000)); expectedOutput.add(new StreamRecord<>( WindowedValue.of(KV.of("key1", 11), new Instant(initialTime + 15000), new IntervalWindow(new Instant(10000), new Instant(20000)), PaneInfo.createPane(true, true, PaneInfo.Timing.ON_TIME)) , initialTime + 15000)); expectedOutput.add(new StreamRecord<>( WindowedValue.of(KV.of("key1", 3), new Instant(initialTime + 10000), new IntervalWindow(new Instant(5000), new Instant(15000)), PaneInfo.createPane(true, true, PaneInfo.Timing.ON_TIME)) , initialTime + 10000)); expectedOutput.add(new StreamRecord<>( WindowedValue.of(KV.of("key2", 1), new Instant(initialTime + 19500), new IntervalWindow(new Instant(10000), new Instant(20000)), PaneInfo.createPane(true, true, PaneInfo.Timing.ON_TIME)) , initialTime + 19500)); expectedOutput.add(new Watermark(initialTime + 20000)); expectedOutput.add(new StreamRecord<>( WindowedValue.of(KV.of("key2", 1), new Instant(initialTime + 20000), /** * this is 20000 and not 19500 because of a convention in dataflow where * timestamps of windowed values in a window cannot be smaller than the * end of a previous window. Checkout the documentation of the * {@link WindowFn#getOutputTime(Instant, BoundedWindow)} */ new IntervalWindow(new Instant(15000), new Instant(25000)), PaneInfo.createPane(true, true, PaneInfo.Timing.ON_TIME)) , initialTime + 20000)); expectedOutput.add(new StreamRecord<>( WindowedValue.of(KV.of("key1", 8), new Instant(initialTime + 20000), new IntervalWindow(new Instant(15000), new Instant(25000)), PaneInfo.createPane(true, true, PaneInfo.Timing.ON_TIME)) , initialTime + 20000)); expectedOutput.add(new Watermark(initialTime + 25000)); TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expectedOutput, testHarness.getOutput(), new ResultSortComparator()); testHarness.close(); }
Example 7
Source File: WindowOperatorMigrationTest.java From Flink-CEPplus with Apache License 2.0 | 4 votes |
@Test public void testRestoreReducingEventTimeWindows() throws Exception { final int windowSize = 3; ReducingStateDescriptor<Tuple2<String, Integer>> stateDesc = new ReducingStateDescriptor<>("window-contents", new SumReducer<>(), STRING_INT_TUPLE.createSerializer(new ExecutionConfig())); WindowOperator<String, Tuple2<String, Integer>, Tuple2<String, Integer>, Tuple2<String, Integer>, TimeWindow> operator = new WindowOperator<>( TumblingEventTimeWindows.of(Time.of(windowSize, TimeUnit.SECONDS)), new TimeWindow.Serializer(), new TupleKeySelector<>(), BasicTypeInfo.STRING_TYPE_INFO.createSerializer(new ExecutionConfig()), stateDesc, new InternalSingleValueWindowFunction<>(new PassThroughWindowFunction<String, TimeWindow, Tuple2<String, Integer>>()), EventTimeTrigger.create(), 0, null /* late data output tag */); ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>(); OneInputStreamOperatorTestHarness<Tuple2<String, Integer>, Tuple2<String, Integer>> testHarness = new KeyedOneInputStreamOperatorTestHarness<>(operator, new TupleKeySelector<>(), BasicTypeInfo.STRING_TYPE_INFO); testHarness.setup(); testHarness.initializeState( OperatorSnapshotUtil.getResourceFilename( "win-op-migration-test-reduce-event-time-flink" + testMigrateVersion + "-snapshot")); testHarness.open(); testHarness.processWatermark(new Watermark(2999)); expectedOutput.add(new StreamRecord<>(new Tuple2<>("key1", 3), 2999)); expectedOutput.add(new StreamRecord<>(new Tuple2<>("key2", 3), 2999)); expectedOutput.add(new Watermark(2999)); testHarness.processWatermark(new Watermark(3999)); expectedOutput.add(new Watermark(3999)); testHarness.processWatermark(new Watermark(4999)); expectedOutput.add(new Watermark(4999)); testHarness.processWatermark(new Watermark(5999)); expectedOutput.add(new StreamRecord<>(new Tuple2<>("key2", 2), 5999)); expectedOutput.add(new Watermark(5999)); TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expectedOutput, testHarness.getOutput(), new Tuple2ResultSortComparator<>()); testHarness.close(); }
Example 8
Source File: AsyncWaitOperatorTest.java From flink with Apache License 2.0 | 4 votes |
private void testEventTime(AsyncDataStream.OutputMode mode) throws Exception { final OneInputStreamOperatorTestHarness<Integer, Integer> testHarness = createTestHarness(new MyAsyncFunction(), TIMEOUT, 2, mode); final long initialTime = 0L; final ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>(); testHarness.open(); synchronized (testHarness.getCheckpointLock()) { testHarness.processElement(new StreamRecord<>(1, initialTime + 1)); testHarness.processElement(new StreamRecord<>(2, initialTime + 2)); testHarness.processWatermark(new Watermark(initialTime + 2)); testHarness.processElement(new StreamRecord<>(3, initialTime + 3)); } // wait until all async collectors in the buffer have been emitted out. synchronized (testHarness.getCheckpointLock()) { testHarness.endInput(); testHarness.close(); } expectedOutput.add(new StreamRecord<>(2, initialTime + 1)); expectedOutput.add(new StreamRecord<>(4, initialTime + 2)); expectedOutput.add(new Watermark(initialTime + 2)); expectedOutput.add(new StreamRecord<>(6, initialTime + 3)); if (AsyncDataStream.OutputMode.ORDERED == mode) { TestHarnessUtil.assertOutputEquals("Output with watermark was not correct.", expectedOutput, testHarness.getOutput()); } else { Object[] jobOutputQueue = testHarness.getOutput().toArray(); Assert.assertEquals("Watermark should be at index 2", new Watermark(initialTime + 2), jobOutputQueue[2]); Assert.assertEquals("StreamRecord 3 should be at the end", new StreamRecord<>(6, initialTime + 3), jobOutputQueue[3]); TestHarnessUtil.assertOutputEqualsSorted( "Output for StreamRecords does not match", expectedOutput, testHarness.getOutput(), new StreamRecordComparator()); } }
Example 9
Source File: WindowOperatorTest.java From Flink-CEPplus with Apache License 2.0 | 4 votes |
@Test public void testCleanupTimerWithEmptyFoldingStateForTumblingWindows() throws Exception { final int windowSize = 2; final long lateness = 1; FoldingStateDescriptor<Tuple2<String, Integer>, Tuple2<String, Integer>> windowStateDesc = new FoldingStateDescriptor<>( "window-contents", new Tuple2<>((String) null, 0), new FoldFunction<Tuple2<String, Integer>, Tuple2<String, Integer>>() { private static final long serialVersionUID = 1L; @Override public Tuple2<String, Integer> fold(Tuple2<String, Integer> accumulator, Tuple2<String, Integer> value) throws Exception { return new Tuple2<>(value.f0, accumulator.f1 + value.f1); } }, STRING_INT_TUPLE); windowStateDesc.initializeSerializerUnlessSet(new ExecutionConfig()); WindowOperator<String, Tuple2<String, Integer>, Tuple2<String, Integer>, Tuple2<String, Integer>, TimeWindow> operator = new WindowOperator<>( TumblingEventTimeWindows.of(Time.of(windowSize, TimeUnit.SECONDS)), new TimeWindow.Serializer(), new TupleKeySelector(), BasicTypeInfo.STRING_TYPE_INFO.createSerializer(new ExecutionConfig()), windowStateDesc, new InternalSingleValueWindowFunction<>(new PassThroughFunction()), EventTimeTrigger.create(), lateness, null /* late data output tag */); OneInputStreamOperatorTestHarness<Tuple2<String, Integer>, Tuple2<String, Integer>> testHarness = createTestHarness(operator); testHarness.open(); ConcurrentLinkedQueue<Object> expected = new ConcurrentLinkedQueue<>(); // normal element testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 1000)); testHarness.processWatermark(new Watermark(1599)); testHarness.processWatermark(new Watermark(1999)); testHarness.processWatermark(new Watermark(2000)); testHarness.processWatermark(new Watermark(5000)); expected.add(new Watermark(1599)); expected.add(new StreamRecord<>(new Tuple2<>("key2", 1), 1999)); expected.add(new Watermark(1999)); // here it fires and purges expected.add(new Watermark(2000)); // here is the cleanup timer expected.add(new Watermark(5000)); TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expected, testHarness.getOutput(), new Tuple2ResultSortComparator()); testHarness.close(); }
Example 10
Source File: WindowOperatorTest.java From Flink-CEPplus with Apache License 2.0 | 4 votes |
/** * This tests whether merging works correctly with the ContinuousEventTimeTrigger. */ @Test public void testSessionWindowsWithContinuousEventTimeTrigger() throws Exception { closeCalled.set(0); final int sessionSize = 3; ListStateDescriptor<Tuple2<String, Integer>> stateDesc = new ListStateDescriptor<>("window-contents", STRING_INT_TUPLE.createSerializer(new ExecutionConfig())); WindowOperator<String, Tuple2<String, Integer>, Iterable<Tuple2<String, Integer>>, Tuple3<String, Long, Long>, TimeWindow> operator = new WindowOperator<>( EventTimeSessionWindows.withGap(Time.seconds(sessionSize)), new TimeWindow.Serializer(), new TupleKeySelector(), BasicTypeInfo.STRING_TYPE_INFO.createSerializer(new ExecutionConfig()), stateDesc, new InternalIterableWindowFunction<>(new SessionWindowFunction()), ContinuousEventTimeTrigger.of(Time.seconds(2)), 0, null /* late data output tag */); OneInputStreamOperatorTestHarness<Tuple2<String, Integer>, Tuple3<String, Long, Long>> testHarness = createTestHarness(operator); ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>(); testHarness.open(); // add elements out-of-order and first trigger time is 2000 testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 1), 1500)); testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 0)); testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 3), 2500)); testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 2), 1000)); // triggers emit and next trigger time is 4000 testHarness.processWatermark(new Watermark(2500)); expectedOutput.add(new StreamRecord<>(new Tuple3<>("key1-1", 1500L, 4500L), 4499)); expectedOutput.add(new StreamRecord<>(new Tuple3<>("key2-6", 0L, 5500L), 5499)); expectedOutput.add(new Watermark(2500)); testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 5), 4000)); testHarness.processWatermark(new Watermark(3000)); expectedOutput.add(new Watermark(3000)); // do a snapshot, close and restore again OperatorSubtaskState snapshot = testHarness.snapshot(0L, 0L); TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expectedOutput, testHarness.getOutput(), new Tuple3ResultSortComparator()); testHarness.close(); expectedOutput.clear(); testHarness = createTestHarness(operator); testHarness.setup(); testHarness.initializeState(snapshot); testHarness.open(); testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 2), 4000)); testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 4), 3500)); // triggers emit and next trigger time is 6000 testHarness.processWatermark(new Watermark(4000)); expectedOutput.add(new StreamRecord<>(new Tuple3<>("key1-3", 1500L, 7000L), 6999)); expectedOutput.add(new StreamRecord<>(new Tuple3<>("key2-15", 0L, 7000L), 6999)); expectedOutput.add(new Watermark(4000)); TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expectedOutput, testHarness.getOutput(), new Tuple3ResultSortComparator()); testHarness.close(); }
Example 11
Source File: EvictingWindowOperatorTest.java From flink with Apache License 2.0 | 4 votes |
/** * Tests CountEvictor evictAfter behavior. */ @Test public void testCountEvictorEvictAfter() throws Exception { AtomicInteger closeCalled = new AtomicInteger(0); final int windowSize = 4; final int triggerCount = 2; final boolean evictAfter = true; @SuppressWarnings({"unchecked", "rawtypes"}) TypeSerializer<StreamRecord<Tuple2<String, Integer>>> streamRecordSerializer = (TypeSerializer<StreamRecord<Tuple2<String, Integer>>>) new StreamElementSerializer(STRING_INT_TUPLE.createSerializer(new ExecutionConfig())); ListStateDescriptor<StreamRecord<Tuple2<String, Integer>>> stateDesc = new ListStateDescriptor<>("window-contents", streamRecordSerializer); EvictingWindowOperator<String, Tuple2<String, Integer>, Tuple2<String, Integer>, GlobalWindow> operator = new EvictingWindowOperator<>( GlobalWindows.create(), new GlobalWindow.Serializer(), new TupleKeySelector(), BasicTypeInfo.STRING_TYPE_INFO.createSerializer(new ExecutionConfig()), stateDesc, new InternalIterableWindowFunction<>(new RichSumReducer<GlobalWindow>(closeCalled)), CountTrigger.of(triggerCount), CountEvictor.of(windowSize, evictAfter), 0, null /* late data output tag */); OneInputStreamOperatorTestHarness<Tuple2<String, Integer>, Tuple2<String, Integer>> testHarness = new KeyedOneInputStreamOperatorTestHarness<>(operator, new TupleKeySelector(), BasicTypeInfo.STRING_TYPE_INFO); long initialTime = 0L; ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>(); testHarness.open(); testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 3000)); testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 3999)); testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 1), initialTime + 20)); testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 1), initialTime)); testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 1), initialTime + 999)); testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 1998)); testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 1999)); testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 1000)); expectedOutput.add(new StreamRecord<>(new Tuple2<>("key2", 2), Long.MAX_VALUE)); expectedOutput.add(new StreamRecord<>(new Tuple2<>("key2", 4), Long.MAX_VALUE)); expectedOutput.add(new StreamRecord<>(new Tuple2<>("key1", 2), Long.MAX_VALUE)); TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expectedOutput, testHarness.getOutput(), new ResultSortComparator()); testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 1), initialTime + 10999)); testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 1000)); expectedOutput.add(new StreamRecord<>(new Tuple2<>("key1", 4), Long.MAX_VALUE)); expectedOutput.add(new StreamRecord<>(new Tuple2<>("key2", 6), Long.MAX_VALUE)); TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expectedOutput, testHarness.getOutput(), new ResultSortComparator()); testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 1000)); testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 1000)); expectedOutput.add(new StreamRecord<>(new Tuple2<>("key2", 6), Long.MAX_VALUE)); TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expectedOutput, testHarness.getOutput(), new ResultSortComparator()); testHarness.close(); Assert.assertEquals("Close was not called.", 1, closeCalled.get()); }
Example 12
Source File: WindowOperatorTest.java From Flink-CEPplus with Apache License 2.0 | 4 votes |
@Test public void testCleanupTimerWithEmptyListStateForTumblingWindows2() throws Exception { final int windowSize = 2; final long lateness = 100; ListStateDescriptor<Tuple2<String, Integer>> windowStateDesc = new ListStateDescriptor<>("window-contents", STRING_INT_TUPLE.createSerializer(new ExecutionConfig())); WindowOperator<String, Tuple2<String, Integer>, Iterable<Tuple2<String, Integer>>, String, TimeWindow> operator = new WindowOperator<>( TumblingEventTimeWindows.of(Time.of(windowSize, TimeUnit.SECONDS)), new TimeWindow.Serializer(), new TupleKeySelector(), BasicTypeInfo.STRING_TYPE_INFO.createSerializer(new ExecutionConfig()), windowStateDesc, new InternalIterableWindowFunction<>(new PassThroughFunction2()), new EventTimeTriggerAccumGC(lateness), lateness, null /* late data output tag */); OneInputStreamOperatorTestHarness<Tuple2<String, Integer>, String> testHarness = createTestHarness(operator); testHarness.open(); ConcurrentLinkedQueue<Object> expected = new ConcurrentLinkedQueue<>(); // normal element testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 1000)); testHarness.processWatermark(new Watermark(1599)); testHarness.processWatermark(new Watermark(1999)); testHarness.processWatermark(new Watermark(2100)); testHarness.processWatermark(new Watermark(5000)); expected.add(new Watermark(1599)); expected.add(new StreamRecord<>("GOT: (key2,1)", 1999)); expected.add(new Watermark(1999)); // here it fires and purges expected.add(new Watermark(2100)); // here is the cleanup timer expected.add(new Watermark(5000)); TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expected, testHarness.getOutput(), new Tuple2ResultSortComparator()); testHarness.close(); }
Example 13
Source File: WindowOperatorTest.java From flink with Apache License 2.0 | 4 votes |
@Test public void testCleanupTimerWithEmptyListStateForTumblingWindows() throws Exception { final int windowSize = 2; final long lateness = 1; ListStateDescriptor<Tuple2<String, Integer>> windowStateDesc = new ListStateDescriptor<>("window-contents", STRING_INT_TUPLE.createSerializer(new ExecutionConfig())); WindowOperator<String, Tuple2<String, Integer>, Iterable<Tuple2<String, Integer>>, Tuple2<String, Integer>, TimeWindow> operator = new WindowOperator<>( TumblingEventTimeWindows.of(Time.of(windowSize, TimeUnit.SECONDS)), new TimeWindow.Serializer(), new TupleKeySelector(), BasicTypeInfo.STRING_TYPE_INFO.createSerializer(new ExecutionConfig()), windowStateDesc, new InternalIterableWindowFunction<>(new PassThroughFunction()), EventTimeTrigger.create(), lateness, null /* late data output tag */); OneInputStreamOperatorTestHarness<Tuple2<String, Integer>, Tuple2<String, Integer>> testHarness = createTestHarness(operator); testHarness.open(); ConcurrentLinkedQueue<Object> expected = new ConcurrentLinkedQueue<>(); // normal element testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 1000)); testHarness.processWatermark(new Watermark(1599)); testHarness.processWatermark(new Watermark(1999)); testHarness.processWatermark(new Watermark(2000)); testHarness.processWatermark(new Watermark(5000)); expected.add(new Watermark(1599)); expected.add(new StreamRecord<>(new Tuple2<>("key2", 1), 1999)); expected.add(new Watermark(1999)); // here it fires and purges expected.add(new Watermark(2000)); // here is the cleanup timer expected.add(new Watermark(5000)); TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expected, testHarness.getOutput(), new Tuple2ResultSortComparator()); testHarness.close(); }
Example 14
Source File: WindowOperatorMigrationTest.java From Flink-CEPplus with Apache License 2.0 | 4 votes |
@Test public void testRestoreKryoSerializedKeysWindows() throws Exception { final int windowSize = 3; TypeInformation<Tuple2<NonPojoType, Integer>> inputType = new TypeHint<Tuple2<NonPojoType, Integer>>() {}.getTypeInfo(); ReducingStateDescriptor<Tuple2<NonPojoType, Integer>> stateDesc = new ReducingStateDescriptor<>("window-contents", new SumReducer<>(), inputType.createSerializer(new ExecutionConfig())); TypeSerializer<NonPojoType> keySerializer = TypeInformation.of(NonPojoType.class).createSerializer(new ExecutionConfig()); assertTrue(keySerializer instanceof KryoSerializer); WindowOperator<NonPojoType, Tuple2<NonPojoType, Integer>, Tuple2<NonPojoType, Integer>, Tuple2<NonPojoType, Integer>, TimeWindow> operator = new WindowOperator<>( TumblingEventTimeWindows.of(Time.of(windowSize, TimeUnit.SECONDS)), new TimeWindow.Serializer(), new TupleKeySelector<>(), keySerializer, stateDesc, new InternalSingleValueWindowFunction<>(new PassThroughWindowFunction<>()), EventTimeTrigger.create(), 0, null /* late data output tag */); ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>(); OneInputStreamOperatorTestHarness<Tuple2<NonPojoType, Integer>, Tuple2<NonPojoType, Integer>> testHarness = new KeyedOneInputStreamOperatorTestHarness<>(operator, new TupleKeySelector<>(), TypeInformation.of(NonPojoType.class)); testHarness.setup(); testHarness.initializeState( OperatorSnapshotUtil.getResourceFilename( "win-op-migration-test-kryo-serialized-key-flink" + testMigrateVersion + "-snapshot")); testHarness.open(); testHarness.processWatermark(new Watermark(2999)); expectedOutput.add(new StreamRecord<>(new Tuple2<>(new NonPojoType("key1"), 3), 2999)); expectedOutput.add(new StreamRecord<>(new Tuple2<>(new NonPojoType("key2"), 3), 2999)); expectedOutput.add(new Watermark(2999)); testHarness.processWatermark(new Watermark(3999)); expectedOutput.add(new Watermark(3999)); testHarness.processWatermark(new Watermark(4999)); expectedOutput.add(new Watermark(4999)); testHarness.processWatermark(new Watermark(5999)); expectedOutput.add(new StreamRecord<>(new Tuple2<>(new NonPojoType("key2"), 2), 5999)); expectedOutput.add(new Watermark(5999)); TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expectedOutput, testHarness.getOutput(), new Tuple2ResultSortComparator<>()); testHarness.close(); }
Example 15
Source File: WindowOperatorTest.java From flink with Apache License 2.0 | 4 votes |
@Test @SuppressWarnings("unchecked") public void testContinuousWatermarkTrigger() throws Exception { closeCalled.set(0); final int windowSize = 3; ReducingStateDescriptor<Tuple2<String, Integer>> stateDesc = new ReducingStateDescriptor<>("window-contents", new SumReducer(), STRING_INT_TUPLE.createSerializer(new ExecutionConfig())); WindowOperator<String, Tuple2<String, Integer>, Tuple2<String, Integer>, Tuple2<String, Integer>, GlobalWindow> operator = new WindowOperator<>( GlobalWindows.create(), new GlobalWindow.Serializer(), new TupleKeySelector(), BasicTypeInfo.STRING_TYPE_INFO.createSerializer(new ExecutionConfig()), stateDesc, new InternalSingleValueWindowFunction<>(new PassThroughWindowFunction<String, GlobalWindow, Tuple2<String, Integer>>()), ContinuousEventTimeTrigger.of(Time.of(windowSize, TimeUnit.SECONDS)), 0, null /* late data output tag */); OneInputStreamOperatorTestHarness<Tuple2<String, Integer>, Tuple2<String, Integer>> testHarness = createTestHarness(operator); ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>(); testHarness.open(); // The global window actually ignores these timestamps... testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 1), 0)); // add elements out-of-order testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 3000)); testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 3999)); testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 1), 20)); testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 1), 999)); testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 1998)); testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 1999)); testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 1000)); testHarness.processWatermark(new Watermark(1000)); expectedOutput.add(new Watermark(1000)); TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expectedOutput, testHarness.getOutput(), new Tuple2ResultSortComparator()); testHarness.processWatermark(new Watermark(2000)); expectedOutput.add(new Watermark(2000)); TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expectedOutput, testHarness.getOutput(), new Tuple2ResultSortComparator()); testHarness.processWatermark(new Watermark(3000)); expectedOutput.add(new StreamRecord<>(new Tuple2<>("key1", 3), Long.MAX_VALUE)); expectedOutput.add(new Watermark(3000)); TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expectedOutput, testHarness.getOutput(), new Tuple2ResultSortComparator()); testHarness.processWatermark(new Watermark(4000)); expectedOutput.add(new Watermark(4000)); TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expectedOutput, testHarness.getOutput(), new Tuple2ResultSortComparator()); testHarness.processWatermark(new Watermark(5000)); expectedOutput.add(new Watermark(5000)); TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expectedOutput, testHarness.getOutput(), new Tuple2ResultSortComparator()); testHarness.processWatermark(new Watermark(6000)); expectedOutput.add(new StreamRecord<>(new Tuple2<>("key1", 3), Long.MAX_VALUE)); expectedOutput.add(new StreamRecord<>(new Tuple2<>("key2", 5), Long.MAX_VALUE)); expectedOutput.add(new Watermark(6000)); TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expectedOutput, testHarness.getOutput(), new Tuple2ResultSortComparator()); // those don't have any effect... testHarness.processWatermark(new Watermark(7000)); testHarness.processWatermark(new Watermark(8000)); expectedOutput.add(new Watermark(7000)); expectedOutput.add(new Watermark(8000)); TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expectedOutput, testHarness.getOutput(), new Tuple2ResultSortComparator()); testHarness.close(); }
Example 16
Source File: WindowOperatorTest.java From Flink-CEPplus with Apache License 2.0 | 4 votes |
@Test public void testLateness() throws Exception { final int windowSize = 2; final long lateness = 500; ReducingStateDescriptor<Tuple2<String, Integer>> stateDesc = new ReducingStateDescriptor<>("window-contents", new SumReducer(), STRING_INT_TUPLE.createSerializer(new ExecutionConfig())); WindowOperator<String, Tuple2<String, Integer>, Tuple2<String, Integer>, Tuple2<String, Integer>, TimeWindow> operator = new WindowOperator<>( TumblingEventTimeWindows.of(Time.of(windowSize, TimeUnit.SECONDS)), new TimeWindow.Serializer(), new TupleKeySelector(), BasicTypeInfo.STRING_TYPE_INFO.createSerializer(new ExecutionConfig()), stateDesc, new InternalSingleValueWindowFunction<>(new PassThroughWindowFunction<String, TimeWindow, Tuple2<String, Integer>>()), PurgingTrigger.of(EventTimeTrigger.create()), lateness, lateOutputTag); OneInputStreamOperatorTestHarness<Tuple2<String, Integer>, Tuple2<String, Integer>> testHarness = createTestHarness(operator); testHarness.open(); ConcurrentLinkedQueue<Object> expected = new ConcurrentLinkedQueue<>(); ConcurrentLinkedQueue<Object> lateExpected = new ConcurrentLinkedQueue<>(); testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 500)); testHarness.processWatermark(new Watermark(1500)); expected.add(new Watermark(1500)); testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 1300)); testHarness.processWatermark(new Watermark(2300)); expected.add(new StreamRecord<>(new Tuple2<>("key2", 2), 1999)); expected.add(new Watermark(2300)); // this will not be sideoutput because window.maxTimestamp() + allowedLateness > currentWatermark testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 1997)); testHarness.processWatermark(new Watermark(6000)); // this is 1 and not 3 because the trigger fires and purges expected.add(new StreamRecord<>(new Tuple2<>("key2", 1), 1999)); expected.add(new Watermark(6000)); // this will be side output because window.maxTimestamp() + allowedLateness < currentWatermark testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 1998)); testHarness.processWatermark(new Watermark(7000)); lateExpected.add(new StreamRecord<>(new Tuple2<>("key2", 1), 1998)); expected.add(new Watermark(7000)); TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expected, testHarness.getOutput(), new Tuple2ResultSortComparator()); TestHarnessUtil.assertOutputEqualsSorted( "SideOutput was not correct.", lateExpected, (Iterable) testHarness.getSideOutput(lateOutputTag), new Tuple2ResultSortComparator()); testHarness.close(); }
Example 17
Source File: WindowOperatorMigrationTest.java From flink with Apache License 2.0 | 4 votes |
@Test public void testRestoreKryoSerializedKeysWindows() throws Exception { final int windowSize = 3; TypeInformation<Tuple2<NonPojoType, Integer>> inputType = new TypeHint<Tuple2<NonPojoType, Integer>>() {}.getTypeInfo(); ReducingStateDescriptor<Tuple2<NonPojoType, Integer>> stateDesc = new ReducingStateDescriptor<>("window-contents", new SumReducer<>(), inputType.createSerializer(new ExecutionConfig())); TypeSerializer<NonPojoType> keySerializer = TypeInformation.of(NonPojoType.class).createSerializer(new ExecutionConfig()); assertTrue(keySerializer instanceof KryoSerializer); WindowOperator<NonPojoType, Tuple2<NonPojoType, Integer>, Tuple2<NonPojoType, Integer>, Tuple2<NonPojoType, Integer>, TimeWindow> operator = new WindowOperator<>( TumblingEventTimeWindows.of(Time.of(windowSize, TimeUnit.SECONDS)), new TimeWindow.Serializer(), new TupleKeySelector<>(), keySerializer, stateDesc, new InternalSingleValueWindowFunction<>(new PassThroughWindowFunction<>()), EventTimeTrigger.create(), 0, null /* late data output tag */); ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>(); OneInputStreamOperatorTestHarness<Tuple2<NonPojoType, Integer>, Tuple2<NonPojoType, Integer>> testHarness = new KeyedOneInputStreamOperatorTestHarness<>(operator, new TupleKeySelector<>(), TypeInformation.of(NonPojoType.class)); testHarness.setup(); testHarness.initializeState( OperatorSnapshotUtil.getResourceFilename( "win-op-migration-test-kryo-serialized-key-flink" + testMigrateVersion + "-snapshot")); testHarness.open(); testHarness.processWatermark(new Watermark(2999)); expectedOutput.add(new StreamRecord<>(new Tuple2<>(new NonPojoType("key1"), 3), 2999)); expectedOutput.add(new StreamRecord<>(new Tuple2<>(new NonPojoType("key2"), 3), 2999)); expectedOutput.add(new Watermark(2999)); testHarness.processWatermark(new Watermark(3999)); expectedOutput.add(new Watermark(3999)); testHarness.processWatermark(new Watermark(4999)); expectedOutput.add(new Watermark(4999)); testHarness.processWatermark(new Watermark(5999)); expectedOutput.add(new StreamRecord<>(new Tuple2<>(new NonPojoType("key2"), 2), 5999)); expectedOutput.add(new Watermark(5999)); TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expectedOutput, testHarness.getOutput(), new Tuple2ResultSortComparator<>()); testHarness.close(); }
Example 18
Source File: WindowOperatorMigrationTest.java From flink with Apache License 2.0 | 4 votes |
@Test public void testRestoreReducingEventTimeWindows() throws Exception { final int windowSize = 3; ReducingStateDescriptor<Tuple2<String, Integer>> stateDesc = new ReducingStateDescriptor<>("window-contents", new SumReducer<>(), STRING_INT_TUPLE.createSerializer(new ExecutionConfig())); WindowOperator<String, Tuple2<String, Integer>, Tuple2<String, Integer>, Tuple2<String, Integer>, TimeWindow> operator = new WindowOperator<>( TumblingEventTimeWindows.of(Time.of(windowSize, TimeUnit.SECONDS)), new TimeWindow.Serializer(), new TupleKeySelector<>(), BasicTypeInfo.STRING_TYPE_INFO.createSerializer(new ExecutionConfig()), stateDesc, new InternalSingleValueWindowFunction<>(new PassThroughWindowFunction<String, TimeWindow, Tuple2<String, Integer>>()), EventTimeTrigger.create(), 0, null /* late data output tag */); ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>(); OneInputStreamOperatorTestHarness<Tuple2<String, Integer>, Tuple2<String, Integer>> testHarness = new KeyedOneInputStreamOperatorTestHarness<>(operator, new TupleKeySelector<>(), BasicTypeInfo.STRING_TYPE_INFO); testHarness.setup(); testHarness.initializeState( OperatorSnapshotUtil.getResourceFilename( "win-op-migration-test-reduce-event-time-flink" + testMigrateVersion + "-snapshot")); testHarness.open(); testHarness.processWatermark(new Watermark(2999)); expectedOutput.add(new StreamRecord<>(new Tuple2<>("key1", 3), 2999)); expectedOutput.add(new StreamRecord<>(new Tuple2<>("key2", 3), 2999)); expectedOutput.add(new Watermark(2999)); testHarness.processWatermark(new Watermark(3999)); expectedOutput.add(new Watermark(3999)); testHarness.processWatermark(new Watermark(4999)); expectedOutput.add(new Watermark(4999)); testHarness.processWatermark(new Watermark(5999)); expectedOutput.add(new StreamRecord<>(new Tuple2<>("key2", 2), 5999)); expectedOutput.add(new Watermark(5999)); TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expectedOutput, testHarness.getOutput(), new Tuple2ResultSortComparator<>()); testHarness.close(); }
Example 19
Source File: WindowOperatorTest.java From flink with Apache License 2.0 | 4 votes |
/** * This tests a custom Session window assigner that assigns some elements to "point windows", * windows that have the same timestamp for start and end. * * <p>In this test, elements that have 33 as the second tuple field will be put into a point * window. */ @Test @SuppressWarnings("unchecked") public void testPointSessions() throws Exception { closeCalled.set(0); ListStateDescriptor<Tuple2<String, Integer>> stateDesc = new ListStateDescriptor<>("window-contents", STRING_INT_TUPLE.createSerializer(new ExecutionConfig())); WindowOperator<String, Tuple2<String, Integer>, Iterable<Tuple2<String, Integer>>, Tuple3<String, Long, Long>, TimeWindow> operator = new WindowOperator<>( new PointSessionWindows(3000), new TimeWindow.Serializer(), new TupleKeySelector(), BasicTypeInfo.STRING_TYPE_INFO.createSerializer(new ExecutionConfig()), stateDesc, new InternalIterableWindowFunction<>(new SessionWindowFunction()), EventTimeTrigger.create(), 0, null /* late data output tag */); ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>(); OperatorSubtaskState snapshot; try (OneInputStreamOperatorTestHarness<Tuple2<String, Integer>, Tuple3<String, Long, Long>> testHarness = createTestHarness(operator)) { testHarness.open(); // add elements out-of-order testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 0)); testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 33), 1000)); // do a snapshot, close and restore again snapshot = testHarness.snapshot(0L, 0L); } try (OneInputStreamOperatorTestHarness<Tuple2<String, Integer>, Tuple3<String, Long, Long>> testHarness = createTestHarness(operator)) { testHarness.setup(); testHarness.initializeState(snapshot); testHarness.open(); testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 33), 2500)); testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 1), 10)); testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 2), 1000)); testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 33), 2500)); testHarness.processWatermark(new Watermark(12000)); expectedOutput.add(new StreamRecord<>(new Tuple3<>("key1-36", 10L, 4000L), 3999)); expectedOutput.add(new StreamRecord<>(new Tuple3<>("key2-67", 0L, 3000L), 2999)); expectedOutput.add(new Watermark(12000)); TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expectedOutput, testHarness.getOutput(), new Tuple3ResultSortComparator()); } }
Example 20
Source File: WindowOperatorMigrationTest.java From flink with Apache License 2.0 | 4 votes |
@Test public void testRestoreSessionWindowsWithCountTrigger() throws Exception { final int sessionSize = 3; ListStateDescriptor<Tuple2<String, Integer>> stateDesc = new ListStateDescriptor<>("window-contents", STRING_INT_TUPLE.createSerializer(new ExecutionConfig())); WindowOperator<String, Tuple2<String, Integer>, Iterable<Tuple2<String, Integer>>, Tuple3<String, Long, Long>, TimeWindow> operator = new WindowOperator<>( EventTimeSessionWindows.withGap(Time.seconds(sessionSize)), new TimeWindow.Serializer(), new TupleKeySelector<String>(), BasicTypeInfo.STRING_TYPE_INFO.createSerializer(new ExecutionConfig()), stateDesc, new InternalIterableWindowFunction<>(new SessionWindowFunction()), PurgingTrigger.of(CountTrigger.of(4)), 0, null /* late data output tag */); ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>(); OneInputStreamOperatorTestHarness<Tuple2<String, Integer>, Tuple3<String, Long, Long>> testHarness = new KeyedOneInputStreamOperatorTestHarness<>(operator, new TupleKeySelector<>(), BasicTypeInfo.STRING_TYPE_INFO); testHarness.setup(); testHarness.initializeState( OperatorSnapshotUtil.getResourceFilename( "win-op-migration-test-session-with-stateful-trigger-flink" + testMigrateVersion + "-snapshot")); testHarness.open(); testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 3), 2500)); testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 1), 6000)); testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 2), 6500)); testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 3), 7000)); TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expectedOutput, testHarness.getOutput(), new Tuple3ResultSortComparator()); // add an element that merges the two "key1" sessions, they should now have count 6, and therefore fire testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 10), 4500)); expectedOutput.add(new StreamRecord<>(new Tuple3<>("key1-22", 10L, 10000L), 9999L)); TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expectedOutput, testHarness.getOutput(), new Tuple3ResultSortComparator()); testHarness.close(); }