org.apache.flink.streaming.runtime.operators.windowing.functions.InternalIterableProcessWindowFunction Java Examples
The following examples show how to use
org.apache.flink.streaming.runtime.operators.windowing.functions.InternalIterableProcessWindowFunction.
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: WindowOperatorTest.java From Flink-CEPplus with Apache License 2.0 | 4 votes |
@Test @SuppressWarnings("unchecked") public void testSessionWindowsWithProcessFunction() throws Exception { closeCalled.set(0); final int sessionSize = 3; ListStateDescriptor<Tuple2<String, Integer>> stateDesc = new ListStateDescriptor<>("window-contents", STRING_INT_TUPLE.createSerializer(new ExecutionConfig())); WindowOperator<String, Tuple2<String, Integer>, Iterable<Tuple2<String, Integer>>, Tuple3<String, Long, Long>, TimeWindow> operator = new WindowOperator<>( EventTimeSessionWindows.withGap(Time.seconds(sessionSize)), new TimeWindow.Serializer(), new TupleKeySelector(), BasicTypeInfo.STRING_TYPE_INFO.createSerializer(new ExecutionConfig()), stateDesc, new InternalIterableProcessWindowFunction<>(new SessionProcessWindowFunction()), EventTimeTrigger.create(), 0, null /* late data output tag */); OneInputStreamOperatorTestHarness<Tuple2<String, Integer>, Tuple3<String, Long, Long>> testHarness = createTestHarness(operator); ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>(); testHarness.open(); // add elements out-of-order testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 0)); testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 2), 1000)); testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 3), 2500)); testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 1), 10)); testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 2), 1000)); // do a snapshot, close and restore again OperatorSubtaskState snapshot = testHarness.snapshot(0L, 0L); TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expectedOutput, testHarness.getOutput(), new Tuple3ResultSortComparator()); testHarness.close(); testHarness = createTestHarness(operator); testHarness.setup(); testHarness.initializeState(snapshot); testHarness.open(); testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 3), 2500)); testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 4), 5501)); testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 5), 6000)); testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 5), 6000)); testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 6), 6050)); testHarness.processWatermark(new Watermark(12000)); expectedOutput.add(new StreamRecord<>(new Tuple3<>("key1-6", 10L, 5500L), 5499)); expectedOutput.add(new StreamRecord<>(new Tuple3<>("key2-6", 0L, 5500L), 5499)); expectedOutput.add(new StreamRecord<>(new Tuple3<>("key2-20", 5501L, 9050L), 9049)); expectedOutput.add(new Watermark(12000)); testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 10), 15000)); testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 20), 15000)); testHarness.processWatermark(new Watermark(17999)); expectedOutput.add(new StreamRecord<>(new Tuple3<>("key2-30", 15000L, 18000L), 17999)); expectedOutput.add(new Watermark(17999)); TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expectedOutput, testHarness.getOutput(), new Tuple3ResultSortComparator()); testHarness.close(); }
Example #2
Source File: WindowOperatorTest.java From flink with Apache License 2.0 | 4 votes |
@Test @SuppressWarnings("unchecked") public void testSessionWindowsWithProcessFunction() throws Exception { closeCalled.set(0); final int sessionSize = 3; ListStateDescriptor<Tuple2<String, Integer>> stateDesc = new ListStateDescriptor<>("window-contents", STRING_INT_TUPLE.createSerializer(new ExecutionConfig())); WindowOperator<String, Tuple2<String, Integer>, Iterable<Tuple2<String, Integer>>, Tuple3<String, Long, Long>, TimeWindow> operator = new WindowOperator<>( EventTimeSessionWindows.withGap(Time.seconds(sessionSize)), new TimeWindow.Serializer(), new TupleKeySelector(), BasicTypeInfo.STRING_TYPE_INFO.createSerializer(new ExecutionConfig()), stateDesc, new InternalIterableProcessWindowFunction<>(new SessionProcessWindowFunction()), EventTimeTrigger.create(), 0, null /* late data output tag */); OneInputStreamOperatorTestHarness<Tuple2<String, Integer>, Tuple3<String, Long, Long>> testHarness = createTestHarness(operator); ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>(); testHarness.open(); // add elements out-of-order testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 0)); testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 2), 1000)); testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 3), 2500)); testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 1), 10)); testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 2), 1000)); // do a snapshot, close and restore again OperatorSubtaskState snapshot = testHarness.snapshot(0L, 0L); TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expectedOutput, testHarness.getOutput(), new Tuple3ResultSortComparator()); testHarness.close(); testHarness = createTestHarness(operator); testHarness.setup(); testHarness.initializeState(snapshot); testHarness.open(); testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 3), 2500)); testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 4), 5501)); testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 5), 6000)); testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 5), 6000)); testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 6), 6050)); testHarness.processWatermark(new Watermark(12000)); expectedOutput.add(new StreamRecord<>(new Tuple3<>("key1-6", 10L, 5500L), 5499)); expectedOutput.add(new StreamRecord<>(new Tuple3<>("key2-6", 0L, 5500L), 5499)); expectedOutput.add(new StreamRecord<>(new Tuple3<>("key2-20", 5501L, 9050L), 9049)); expectedOutput.add(new Watermark(12000)); testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 10), 15000)); testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 20), 15000)); testHarness.processWatermark(new Watermark(17999)); expectedOutput.add(new StreamRecord<>(new Tuple3<>("key2-30", 15000L, 18000L), 17999)); expectedOutput.add(new Watermark(17999)); TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expectedOutput, testHarness.getOutput(), new Tuple3ResultSortComparator()); testHarness.close(); }
Example #3
Source File: WindowOperatorTest.java From flink with Apache License 2.0 | 4 votes |
@Test @SuppressWarnings("unchecked") public void testSessionWindowsWithProcessFunction() throws Exception { closeCalled.set(0); final int sessionSize = 3; ListStateDescriptor<Tuple2<String, Integer>> stateDesc = new ListStateDescriptor<>("window-contents", STRING_INT_TUPLE.createSerializer(new ExecutionConfig())); WindowOperator<String, Tuple2<String, Integer>, Iterable<Tuple2<String, Integer>>, Tuple3<String, Long, Long>, TimeWindow> operator = new WindowOperator<>( EventTimeSessionWindows.withGap(Time.seconds(sessionSize)), new TimeWindow.Serializer(), new TupleKeySelector(), BasicTypeInfo.STRING_TYPE_INFO.createSerializer(new ExecutionConfig()), stateDesc, new InternalIterableProcessWindowFunction<>(new SessionProcessWindowFunction()), EventTimeTrigger.create(), 0, null /* late data output tag */); OneInputStreamOperatorTestHarness<Tuple2<String, Integer>, Tuple3<String, Long, Long>> testHarness = createTestHarness(operator); ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>(); testHarness.open(); // add elements out-of-order testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 0)); testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 2), 1000)); testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 3), 2500)); testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 1), 10)); testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 2), 1000)); // do a snapshot, close and restore again OperatorSubtaskState snapshot = testHarness.snapshot(0L, 0L); TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expectedOutput, testHarness.getOutput(), new Tuple3ResultSortComparator()); testHarness.close(); testHarness = createTestHarness(operator); testHarness.setup(); testHarness.initializeState(snapshot); testHarness.open(); testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 3), 2500)); testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 4), 5501)); testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 5), 6000)); testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 5), 6000)); testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 6), 6050)); testHarness.processWatermark(new Watermark(12000)); expectedOutput.add(new StreamRecord<>(new Tuple3<>("key1-6", 10L, 5500L), 5499)); expectedOutput.add(new StreamRecord<>(new Tuple3<>("key2-6", 0L, 5500L), 5499)); expectedOutput.add(new StreamRecord<>(new Tuple3<>("key2-20", 5501L, 9050L), 9049)); expectedOutput.add(new Watermark(12000)); testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 10), 15000)); testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 20), 15000)); testHarness.processWatermark(new Watermark(17999)); expectedOutput.add(new StreamRecord<>(new Tuple3<>("key2-30", 15000L, 18000L), 17999)); expectedOutput.add(new Watermark(17999)); TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expectedOutput, testHarness.getOutput(), new Tuple3ResultSortComparator()); testHarness.close(); }
Example #4
Source File: WindowedStream.java From Flink-CEPplus with Apache License 2.0 | 2 votes |
/** * Applies the given window function to each window. The window function is called for each * evaluation of the window for each key individually. The output of the window function is * interpreted as a regular non-windowed stream. * * <p>Note that this function requires that all data in the windows is buffered until the window * is evaluated, as the function provides no means of incremental aggregation. * * @param function The window function. * @param resultType Type information for the result type of the window function * @return The data stream that is the result of applying the window function to the window. */ @Internal public <R> SingleOutputStreamOperator<R> process(ProcessWindowFunction<T, R, K, W> function, TypeInformation<R> resultType) { function = input.getExecutionEnvironment().clean(function); return apply(new InternalIterableProcessWindowFunction<>(function), resultType, function); }
Example #5
Source File: WindowedStream.java From flink with Apache License 2.0 | 2 votes |
/** * Applies the given window function to each window. The window function is called for each * evaluation of the window for each key individually. The output of the window function is * interpreted as a regular non-windowed stream. * * <p>Note that this function requires that all data in the windows is buffered until the window * is evaluated, as the function provides no means of incremental aggregation. * * @param function The window function. * @param resultType Type information for the result type of the window function * @return The data stream that is the result of applying the window function to the window. */ @Internal public <R> SingleOutputStreamOperator<R> process(ProcessWindowFunction<T, R, K, W> function, TypeInformation<R> resultType) { function = input.getExecutionEnvironment().clean(function); return apply(new InternalIterableProcessWindowFunction<>(function), resultType, function); }
Example #6
Source File: WindowedStream.java From flink with Apache License 2.0 | 2 votes |
/** * Applies the given window function to each window. The window function is called for each * evaluation of the window for each key individually. The output of the window function is * interpreted as a regular non-windowed stream. * * <p>Note that this function requires that all data in the windows is buffered until the window * is evaluated, as the function provides no means of incremental aggregation. * * @param function The window function. * @param resultType Type information for the result type of the window function * @return The data stream that is the result of applying the window function to the window. */ @Internal public <R> SingleOutputStreamOperator<R> process(ProcessWindowFunction<T, R, K, W> function, TypeInformation<R> resultType) { function = input.getExecutionEnvironment().clean(function); return apply(new InternalIterableProcessWindowFunction<>(function), resultType, function); }