Java Code Examples for org.apache.flink.streaming.api.windowing.assigners.SlidingEventTimeWindows#of()
The following examples show how to use
org.apache.flink.streaming.api.windowing.assigners.SlidingEventTimeWindows#of() .
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
| Source File: WindowOperatorTest.java From Flink-CEPplus with Apache License 2.0 | 6 votes |
|
@Test
@SuppressWarnings("unchecked")
public void testSlidingEventTimeWindowsReduce() throws Exception {
closeCalled.set(0);
final int windowSize = 3;
final int windowSlide = 1;
ReducingStateDescriptor<Tuple2<String, Integer>> stateDesc = new ReducingStateDescriptor<>("window-contents",
new SumReducer(),
STRING_INT_TUPLE.createSerializer(new ExecutionConfig()));
WindowOperator<String, Tuple2<String, Integer>, Tuple2<String, Integer>, Tuple2<String, Integer>, TimeWindow> operator = new WindowOperator<>(
SlidingEventTimeWindows.of(Time.of(windowSize, TimeUnit.SECONDS), Time.of(windowSlide, TimeUnit.SECONDS)),
new TimeWindow.Serializer(),
new TupleKeySelector(),
BasicTypeInfo.STRING_TYPE_INFO.createSerializer(new ExecutionConfig()),
stateDesc,
new InternalSingleValueWindowFunction<>(new PassThroughWindowFunction<String, TimeWindow, Tuple2<String, Integer>>()),
EventTimeTrigger.create(),
0,
null /* late data output tag */);
testSlidingEventTimeWindows(operator);
}
Example 2
| Source File: WindowOperatorTest.java From flink with Apache License 2.0 | 6 votes |
|
@Test
public void testSlidingEventTimeWindowsApply() throws Exception {
closeCalled.set(0);
final int windowSize = 3;
final int windowSlide = 1;
ListStateDescriptor<Tuple2<String, Integer>> stateDesc = new ListStateDescriptor<>("window-contents",
STRING_INT_TUPLE.createSerializer(new ExecutionConfig()));
WindowOperator<String, Tuple2<String, Integer>, Iterable<Tuple2<String, Integer>>, Tuple2<String, Integer>, TimeWindow> operator = new WindowOperator<>(
SlidingEventTimeWindows.of(Time.of(windowSize, TimeUnit.SECONDS), Time.of(windowSlide, TimeUnit.SECONDS)),
new TimeWindow.Serializer(),
new TupleKeySelector(),
BasicTypeInfo.STRING_TYPE_INFO.createSerializer(new ExecutionConfig()),
stateDesc,
new InternalIterableWindowFunction<>(new RichSumReducer<TimeWindow>()),
EventTimeTrigger.create(),
0,
null /* late data output tag */);
testSlidingEventTimeWindows(operator);
// we close once in the rest...
Assert.assertEquals("Close was not called.", 2, closeCalled.get());
}
Example 3
| Source File: WindowOperatorTest.java From flink with Apache License 2.0 | 6 votes |
|
@Test
public void testSlidingEventTimeWindowsApply() throws Exception {
closeCalled.set(0);
final int windowSize = 3;
final int windowSlide = 1;
ListStateDescriptor<Tuple2<String, Integer>> stateDesc = new ListStateDescriptor<>("window-contents",
STRING_INT_TUPLE.createSerializer(new ExecutionConfig()));
WindowOperator<String, Tuple2<String, Integer>, Iterable<Tuple2<String, Integer>>, Tuple2<String, Integer>, TimeWindow> operator = new WindowOperator<>(
SlidingEventTimeWindows.of(Time.of(windowSize, TimeUnit.SECONDS), Time.of(windowSlide, TimeUnit.SECONDS)),
new TimeWindow.Serializer(),
new TupleKeySelector(),
BasicTypeInfo.STRING_TYPE_INFO.createSerializer(new ExecutionConfig()),
stateDesc,
new InternalIterableWindowFunction<>(new RichSumReducer<TimeWindow>()),
EventTimeTrigger.create(),
0,
null /* late data output tag */);
testSlidingEventTimeWindows(operator);
// we close once in the rest...
Assert.assertEquals("Close was not called.", 2, closeCalled.get());
}
Example 4
| Source File: WindowOperatorTest.java From flink with Apache License 2.0 | 6 votes |
|
@Test
@SuppressWarnings("unchecked")
public void testSlidingEventTimeWindowsReduce() throws Exception {
closeCalled.set(0);
final int windowSize = 3;
final int windowSlide = 1;
ReducingStateDescriptor<Tuple2<String, Integer>> stateDesc = new ReducingStateDescriptor<>("window-contents",
new SumReducer(),
STRING_INT_TUPLE.createSerializer(new ExecutionConfig()));
WindowOperator<String, Tuple2<String, Integer>, Tuple2<String, Integer>, Tuple2<String, Integer>, TimeWindow> operator = new WindowOperator<>(
SlidingEventTimeWindows.of(Time.of(windowSize, TimeUnit.SECONDS), Time.of(windowSlide, TimeUnit.SECONDS)),
new TimeWindow.Serializer(),
new TupleKeySelector(),
BasicTypeInfo.STRING_TYPE_INFO.createSerializer(new ExecutionConfig()),
stateDesc,
new InternalSingleValueWindowFunction<>(new PassThroughWindowFunction<String, TimeWindow, Tuple2<String, Integer>>()),
EventTimeTrigger.create(),
0,
null /* late data output tag */);
testSlidingEventTimeWindows(operator);
}
Example 5
| Source File: WindowOperatorTest.java From Flink-CEPplus with Apache License 2.0 | 6 votes |
|
@Test
public void testSlidingEventTimeWindowsApply() throws Exception {
closeCalled.set(0);
final int windowSize = 3;
final int windowSlide = 1;
ListStateDescriptor<Tuple2<String, Integer>> stateDesc = new ListStateDescriptor<>("window-contents",
STRING_INT_TUPLE.createSerializer(new ExecutionConfig()));
WindowOperator<String, Tuple2<String, Integer>, Iterable<Tuple2<String, Integer>>, Tuple2<String, Integer>, TimeWindow> operator = new WindowOperator<>(
SlidingEventTimeWindows.of(Time.of(windowSize, TimeUnit.SECONDS), Time.of(windowSlide, TimeUnit.SECONDS)),
new TimeWindow.Serializer(),
new TupleKeySelector(),
BasicTypeInfo.STRING_TYPE_INFO.createSerializer(new ExecutionConfig()),
stateDesc,
new InternalIterableWindowFunction<>(new RichSumReducer<TimeWindow>()),
EventTimeTrigger.create(),
0,
null /* late data output tag */);
testSlidingEventTimeWindows(operator);
// we close once in the rest...
Assert.assertEquals("Close was not called.", 2, closeCalled.get());
}
Example 6
| Source File: SlidingEventTimeWindowsTest.java From flink with Apache License 2.0 | 5 votes |
|
@Test
public void testWindowAssignmentWithNegativeOffset() {
WindowAssigner.WindowAssignerContext mockContext =
mock(WindowAssigner.WindowAssignerContext.class);
SlidingEventTimeWindows assigner =
SlidingEventTimeWindows.of(Time.milliseconds(5000), Time.milliseconds(1000), Time.milliseconds(-100));
assertThat(assigner.assignWindows("String", 0L, mockContext), containsInAnyOrder(
timeWindow(-4100, 900),
timeWindow(-3100, 1900),
timeWindow(-2100, 2900),
timeWindow(-1100, 3900),
timeWindow(-100, 4900)));
assertThat(assigner.assignWindows("String", 4899L, mockContext), containsInAnyOrder(
timeWindow(-100, 4900),
timeWindow(900, 5900),
timeWindow(1900, 6900),
timeWindow(2900, 7900),
timeWindow(3900, 8900)));
assertThat(assigner.assignWindows("String", 4900L, mockContext), containsInAnyOrder(
timeWindow(900, 5900),
timeWindow(1900, 6900),
timeWindow(2900, 7900),
timeWindow(3900, 8900),
timeWindow(4900, 9900)));
}
Example 7
| Source File: SlidingEventTimeWindowsTest.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
|
@Test
public void testWindowAssignmentWithOffset() {
WindowAssigner.WindowAssignerContext mockContext =
mock(WindowAssigner.WindowAssignerContext.class);
SlidingEventTimeWindows assigner =
SlidingEventTimeWindows.of(Time.milliseconds(5000), Time.milliseconds(1000), Time.milliseconds(100));
assertThat(assigner.assignWindows("String", 100L, mockContext), containsInAnyOrder(
timeWindow(-3900, 1100),
timeWindow(-2900, 2100),
timeWindow(-1900, 3100),
timeWindow(-900, 4100),
timeWindow(100, 5100)));
assertThat(assigner.assignWindows("String", 5099L, mockContext), containsInAnyOrder(
timeWindow(100, 5100),
timeWindow(1100, 6100),
timeWindow(2100, 7100),
timeWindow(3100, 8100),
timeWindow(4100, 9100)));
assertThat(assigner.assignWindows("String", 5100L, mockContext), containsInAnyOrder(
timeWindow(1100, 6100),
timeWindow(2100, 7100),
timeWindow(3100, 8100),
timeWindow(4100, 9100),
timeWindow(5100, 10100)));
}
Example 8
| Source File: SlidingEventTimeWindowsTest.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
|
@Test
public void testWindowAssignmentWithNegativeOffset() {
WindowAssigner.WindowAssignerContext mockContext =
mock(WindowAssigner.WindowAssignerContext.class);
SlidingEventTimeWindows assigner =
SlidingEventTimeWindows.of(Time.milliseconds(5000), Time.milliseconds(1000), Time.milliseconds(-100));
assertThat(assigner.assignWindows("String", 0L, mockContext), containsInAnyOrder(
timeWindow(-4100, 900),
timeWindow(-3100, 1900),
timeWindow(-2100, 2900),
timeWindow(-1100, 3900),
timeWindow(-100, 4900)));
assertThat(assigner.assignWindows("String", 4899L, mockContext), containsInAnyOrder(
timeWindow(-100, 4900),
timeWindow(900, 5900),
timeWindow(1900, 6900),
timeWindow(2900, 7900),
timeWindow(3900, 8900)));
assertThat(assigner.assignWindows("String", 4900L, mockContext), containsInAnyOrder(
timeWindow(900, 5900),
timeWindow(1900, 6900),
timeWindow(2900, 7900),
timeWindow(3900, 8900),
timeWindow(4900, 9900)));
}
Example 9
| Source File: SlidingEventTimeWindowsTest.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
|
@Test
public void testTimeUnits() {
// sanity check with one other time unit
WindowAssigner.WindowAssignerContext mockContext =
mock(WindowAssigner.WindowAssignerContext.class);
SlidingEventTimeWindows assigner = SlidingEventTimeWindows.of(Time.seconds(5), Time.seconds(1), Time.milliseconds(500));
assertThat(assigner.assignWindows("String", 100L, mockContext), containsInAnyOrder(
timeWindow(-4500, 500),
timeWindow(-3500, 1500),
timeWindow(-2500, 2500),
timeWindow(-1500, 3500),
timeWindow(-500, 4500)));
assertThat(assigner.assignWindows("String", 5499L, mockContext), containsInAnyOrder(
timeWindow(500, 5500),
timeWindow(1500, 6500),
timeWindow(2500, 7500),
timeWindow(3500, 8500),
timeWindow(4500, 9500)));
assertThat(assigner.assignWindows("String", 5100L, mockContext), containsInAnyOrder(
timeWindow(500, 5500),
timeWindow(1500, 6500),
timeWindow(2500, 7500),
timeWindow(3500, 8500),
timeWindow(4500, 9500)));
}
Example 10
| Source File: DataStreamAllroundTestJobFactory.java From flink with Apache License 2.0 | 5 votes |
|
static SlidingEventTimeWindows createSlidingWindow(ParameterTool pt) {
long slideSize = pt.getLong(
TEST_SLIDE_SIZE.key(),
TEST_SLIDE_SIZE.defaultValue());
long slideFactor = pt.getInt(
TEST_SLIDE_FACTOR.key(),
TEST_SLIDE_FACTOR.defaultValue()
);
return SlidingEventTimeWindows.of(Time.milliseconds(slideSize * slideFactor), Time.milliseconds(slideSize));
}
Example 11
| Source File: SlidingEventTimeWindowsTest.java From flink with Apache License 2.0 | 5 votes |
|
@Test
public void testWindowAssignmentWithOffset() {
WindowAssigner.WindowAssignerContext mockContext =
mock(WindowAssigner.WindowAssignerContext.class);
SlidingEventTimeWindows assigner =
SlidingEventTimeWindows.of(Time.milliseconds(5000), Time.milliseconds(1000), Time.milliseconds(100));
assertThat(assigner.assignWindows("String", 100L, mockContext), containsInAnyOrder(
timeWindow(-3900, 1100),
timeWindow(-2900, 2100),
timeWindow(-1900, 3100),
timeWindow(-900, 4100),
timeWindow(100, 5100)));
assertThat(assigner.assignWindows("String", 5099L, mockContext), containsInAnyOrder(
timeWindow(100, 5100),
timeWindow(1100, 6100),
timeWindow(2100, 7100),
timeWindow(3100, 8100),
timeWindow(4100, 9100)));
assertThat(assigner.assignWindows("String", 5100L, mockContext), containsInAnyOrder(
timeWindow(1100, 6100),
timeWindow(2100, 7100),
timeWindow(3100, 8100),
timeWindow(4100, 9100),
timeWindow(5100, 10100)));
}
Example 12
| Source File: SlidingEventTimeWindowsTest.java From flink with Apache License 2.0 | 5 votes |
|
@Test
public void testWindowAssignmentWithOffset() {
WindowAssigner.WindowAssignerContext mockContext =
mock(WindowAssigner.WindowAssignerContext.class);
SlidingEventTimeWindows assigner =
SlidingEventTimeWindows.of(Time.milliseconds(5000), Time.milliseconds(1000), Time.milliseconds(100));
assertThat(assigner.assignWindows("String", 100L, mockContext), containsInAnyOrder(
timeWindow(-3900, 1100),
timeWindow(-2900, 2100),
timeWindow(-1900, 3100),
timeWindow(-900, 4100),
timeWindow(100, 5100)));
assertThat(assigner.assignWindows("String", 5099L, mockContext), containsInAnyOrder(
timeWindow(100, 5100),
timeWindow(1100, 6100),
timeWindow(2100, 7100),
timeWindow(3100, 8100),
timeWindow(4100, 9100)));
assertThat(assigner.assignWindows("String", 5100L, mockContext), containsInAnyOrder(
timeWindow(1100, 6100),
timeWindow(2100, 7100),
timeWindow(3100, 8100),
timeWindow(4100, 9100),
timeWindow(5100, 10100)));
}
Example 13
| Source File: SlidingEventTimeWindowsTest.java From flink with Apache License 2.0 | 5 votes |
|
@Test
public void testWindowAssignmentWithNegativeOffset() {
WindowAssigner.WindowAssignerContext mockContext =
mock(WindowAssigner.WindowAssignerContext.class);
SlidingEventTimeWindows assigner =
SlidingEventTimeWindows.of(Time.milliseconds(5000), Time.milliseconds(1000), Time.milliseconds(-100));
assertThat(assigner.assignWindows("String", 0L, mockContext), containsInAnyOrder(
timeWindow(-4100, 900),
timeWindow(-3100, 1900),
timeWindow(-2100, 2900),
timeWindow(-1100, 3900),
timeWindow(-100, 4900)));
assertThat(assigner.assignWindows("String", 4899L, mockContext), containsInAnyOrder(
timeWindow(-100, 4900),
timeWindow(900, 5900),
timeWindow(1900, 6900),
timeWindow(2900, 7900),
timeWindow(3900, 8900)));
assertThat(assigner.assignWindows("String", 4900L, mockContext), containsInAnyOrder(
timeWindow(900, 5900),
timeWindow(1900, 6900),
timeWindow(2900, 7900),
timeWindow(3900, 8900),
timeWindow(4900, 9900)));
}
Example 14
| Source File: SlidingEventTimeWindowsTest.java From flink with Apache License 2.0 | 5 votes |
|
@Test
public void testTimeUnits() {
// sanity check with one other time unit
WindowAssigner.WindowAssignerContext mockContext =
mock(WindowAssigner.WindowAssignerContext.class);
SlidingEventTimeWindows assigner = SlidingEventTimeWindows.of(Time.seconds(5), Time.seconds(1), Time.milliseconds(500));
assertThat(assigner.assignWindows("String", 100L, mockContext), containsInAnyOrder(
timeWindow(-4500, 500),
timeWindow(-3500, 1500),
timeWindow(-2500, 2500),
timeWindow(-1500, 3500),
timeWindow(-500, 4500)));
assertThat(assigner.assignWindows("String", 5499L, mockContext), containsInAnyOrder(
timeWindow(500, 5500),
timeWindow(1500, 6500),
timeWindow(2500, 7500),
timeWindow(3500, 8500),
timeWindow(4500, 9500)));
assertThat(assigner.assignWindows("String", 5100L, mockContext), containsInAnyOrder(
timeWindow(500, 5500),
timeWindow(1500, 6500),
timeWindow(2500, 7500),
timeWindow(3500, 8500),
timeWindow(4500, 9500)));
}
Example 15
| Source File: SlidingEventTimeWindowsTest.java From flink with Apache License 2.0 | 5 votes |
|
@Test
public void testProperties() {
SlidingEventTimeWindows assigner = SlidingEventTimeWindows.of(Time.seconds(5), Time.milliseconds(100));
assertTrue(assigner.isEventTime());
assertEquals(new TimeWindow.Serializer(), assigner.getWindowSerializer(new ExecutionConfig()));
assertThat(assigner.getDefaultTrigger(mock(StreamExecutionEnvironment.class)), instanceOf(EventTimeTrigger.class));
}
Example 16
| Source File: DataStreamAllroundTestJobFactory.java From flink with Apache License 2.0 | 5 votes |
|
static SlidingEventTimeWindows createSlidingWindow(ParameterTool pt) {
long slideSize = pt.getLong(
TEST_SLIDE_SIZE.key(),
TEST_SLIDE_SIZE.defaultValue());
long slideFactor = pt.getInt(
TEST_SLIDE_FACTOR.key(),
TEST_SLIDE_FACTOR.defaultValue()
);
return SlidingEventTimeWindows.of(Time.milliseconds(slideSize * slideFactor), Time.milliseconds(slideSize));
}
Example 17
| Source File: SlidingEventTimeWindowsTest.java From flink with Apache License 2.0 | 5 votes |
|
@Test
public void testTimeUnits() {
// sanity check with one other time unit
WindowAssigner.WindowAssignerContext mockContext =
mock(WindowAssigner.WindowAssignerContext.class);
SlidingEventTimeWindows assigner = SlidingEventTimeWindows.of(Time.seconds(5), Time.seconds(1), Time.milliseconds(500));
assertThat(assigner.assignWindows("String", 100L, mockContext), containsInAnyOrder(
timeWindow(-4500, 500),
timeWindow(-3500, 1500),
timeWindow(-2500, 2500),
timeWindow(-1500, 3500),
timeWindow(-500, 4500)));
assertThat(assigner.assignWindows("String", 5499L, mockContext), containsInAnyOrder(
timeWindow(500, 5500),
timeWindow(1500, 6500),
timeWindow(2500, 7500),
timeWindow(3500, 8500),
timeWindow(4500, 9500)));
assertThat(assigner.assignWindows("String", 5100L, mockContext), containsInAnyOrder(
timeWindow(500, 5500),
timeWindow(1500, 6500),
timeWindow(2500, 7500),
timeWindow(3500, 8500),
timeWindow(4500, 9500)));
}
Example 18
| Source File: WindowOperatorTest.java From flink with Apache License 2.0 | 4 votes |
|
@Test
public void testSideOutputDueToLatenessSliding() throws Exception {
final int windowSize = 3;
final int windowSlide = 1;
final long lateness = 0;
ReducingStateDescriptor<Tuple2<String, Integer>> stateDesc = new ReducingStateDescriptor<>("window-contents",
new SumReducer(),
STRING_INT_TUPLE.createSerializer(new ExecutionConfig()));
WindowOperator<String, Tuple2<String, Integer>, Tuple2<String, Integer>, Tuple2<String, Integer>, TimeWindow> operator =
new WindowOperator<>(
SlidingEventTimeWindows.of(Time.of(windowSize, TimeUnit.SECONDS), Time.of(windowSlide, TimeUnit.SECONDS)),
new TimeWindow.Serializer(),
new TupleKeySelector(),
BasicTypeInfo.STRING_TYPE_INFO.createSerializer(new ExecutionConfig()),
stateDesc,
new InternalSingleValueWindowFunction<>(new PassThroughWindowFunction<String, TimeWindow, Tuple2<String, Integer>>()),
EventTimeTrigger.create(),
lateness,
lateOutputTag /* late data output tag */);
OneInputStreamOperatorTestHarness<Tuple2<String, Integer>, Tuple2<String, Integer>> testHarness =
createTestHarness(operator);
testHarness.open();
ConcurrentLinkedQueue<Object> expected = new ConcurrentLinkedQueue<>();
ConcurrentLinkedQueue<Object> sideExpected = new ConcurrentLinkedQueue<>();
testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 1000));
testHarness.processWatermark(new Watermark(1999));
expected.add(new StreamRecord<>(new Tuple2<>("key2", 1), 1999));
expected.add(new Watermark(1999));
testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 2000));
testHarness.processWatermark(new Watermark(3000));
expected.add(new StreamRecord<>(new Tuple2<>("key2", 2), 2999));
expected.add(new Watermark(3000));
testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 1), 3001));
// lateness is set to 0 and window size = 3 sec and slide 1, the following 2 elements (2400)
// are assigned to windows ending at 2999, 3999, 4999.
// The 2999 is dropped because it is already late (WM = 2999) but the rest are kept.
testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 2400));
testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 2400));
testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 1), 3001));
testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 3900));
testHarness.processWatermark(new Watermark(6000));
expected.add(new StreamRecord<>(new Tuple2<>("key2", 5), 3999));
expected.add(new StreamRecord<>(new Tuple2<>("key1", 2), 3999));
expected.add(new StreamRecord<>(new Tuple2<>("key2", 4), 4999));
expected.add(new StreamRecord<>(new Tuple2<>("key1", 2), 4999));
expected.add(new StreamRecord<>(new Tuple2<>("key2", 1), 5999));
expected.add(new StreamRecord<>(new Tuple2<>("key1", 2), 5999));
expected.add(new Watermark(6000));
// sideoutput element due to lateness
testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 1), 3001));
sideExpected.add(new StreamRecord<>(new Tuple2<>("key1", 1), 3001));
testHarness.processWatermark(new Watermark(25000));
expected.add(new Watermark(25000));
TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expected, testHarness.getOutput(), new Tuple2ResultSortComparator());
TestHarnessUtil.assertOutputEqualsSorted("SideOutput was not correct.", sideExpected, (Iterable) testHarness.getSideOutput(lateOutputTag), new Tuple2ResultSortComparator());
testHarness.close();
}
Example 19
| Source File: WindowOperatorTest.java From Flink-CEPplus with Apache License 2.0 | 4 votes |
|
@Test
public void testSideOutputDueToLatenessSliding() throws Exception {
final int windowSize = 3;
final int windowSlide = 1;
final long lateness = 0;
ReducingStateDescriptor<Tuple2<String, Integer>> stateDesc = new ReducingStateDescriptor<>("window-contents",
new SumReducer(),
STRING_INT_TUPLE.createSerializer(new ExecutionConfig()));
WindowOperator<String, Tuple2<String, Integer>, Tuple2<String, Integer>, Tuple2<String, Integer>, TimeWindow> operator =
new WindowOperator<>(
SlidingEventTimeWindows.of(Time.of(windowSize, TimeUnit.SECONDS), Time.of(windowSlide, TimeUnit.SECONDS)),
new TimeWindow.Serializer(),
new TupleKeySelector(),
BasicTypeInfo.STRING_TYPE_INFO.createSerializer(new ExecutionConfig()),
stateDesc,
new InternalSingleValueWindowFunction<>(new PassThroughWindowFunction<String, TimeWindow, Tuple2<String, Integer>>()),
EventTimeTrigger.create(),
lateness,
lateOutputTag /* late data output tag */);
OneInputStreamOperatorTestHarness<Tuple2<String, Integer>, Tuple2<String, Integer>> testHarness =
createTestHarness(operator);
testHarness.open();
ConcurrentLinkedQueue<Object> expected = new ConcurrentLinkedQueue<>();
ConcurrentLinkedQueue<Object> sideExpected = new ConcurrentLinkedQueue<>();
testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 1000));
testHarness.processWatermark(new Watermark(1999));
expected.add(new StreamRecord<>(new Tuple2<>("key2", 1), 1999));
expected.add(new Watermark(1999));
testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 2000));
testHarness.processWatermark(new Watermark(3000));
expected.add(new StreamRecord<>(new Tuple2<>("key2", 2), 2999));
expected.add(new Watermark(3000));
testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 1), 3001));
// lateness is set to 0 and window size = 3 sec and slide 1, the following 2 elements (2400)
// are assigned to windows ending at 2999, 3999, 4999.
// The 2999 is dropped because it is already late (WM = 2999) but the rest are kept.
testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 2400));
testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 2400));
testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 1), 3001));
testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 3900));
testHarness.processWatermark(new Watermark(6000));
expected.add(new StreamRecord<>(new Tuple2<>("key2", 5), 3999));
expected.add(new StreamRecord<>(new Tuple2<>("key1", 2), 3999));
expected.add(new StreamRecord<>(new Tuple2<>("key2", 4), 4999));
expected.add(new StreamRecord<>(new Tuple2<>("key1", 2), 4999));
expected.add(new StreamRecord<>(new Tuple2<>("key2", 1), 5999));
expected.add(new StreamRecord<>(new Tuple2<>("key1", 2), 5999));
expected.add(new Watermark(6000));
// sideoutput element due to lateness
testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 1), 3001));
sideExpected.add(new StreamRecord<>(new Tuple2<>("key1", 1), 3001));
testHarness.processWatermark(new Watermark(25000));
expected.add(new Watermark(25000));
TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expected, testHarness.getOutput(), new Tuple2ResultSortComparator());
TestHarnessUtil.assertOutputEqualsSorted("SideOutput was not correct.", sideExpected, (Iterable) testHarness.getSideOutput(lateOutputTag), new Tuple2ResultSortComparator());
testHarness.close();
}
Example 20
| Source File: WindowOperatorTest.java From flink with Apache License 2.0 | 4 votes |
|
@Test
public void testSideOutputDueToLatenessSliding() throws Exception {
final int windowSize = 3;
final int windowSlide = 1;
final long lateness = 0;
ReducingStateDescriptor<Tuple2<String, Integer>> stateDesc = new ReducingStateDescriptor<>("window-contents",
new SumReducer(),
STRING_INT_TUPLE.createSerializer(new ExecutionConfig()));
WindowOperator<String, Tuple2<String, Integer>, Tuple2<String, Integer>, Tuple2<String, Integer>, TimeWindow> operator =
new WindowOperator<>(
SlidingEventTimeWindows.of(Time.of(windowSize, TimeUnit.SECONDS), Time.of(windowSlide, TimeUnit.SECONDS)),
new TimeWindow.Serializer(),
new TupleKeySelector(),
BasicTypeInfo.STRING_TYPE_INFO.createSerializer(new ExecutionConfig()),
stateDesc,
new InternalSingleValueWindowFunction<>(new PassThroughWindowFunction<String, TimeWindow, Tuple2<String, Integer>>()),
EventTimeTrigger.create(),
lateness,
lateOutputTag /* late data output tag */);
OneInputStreamOperatorTestHarness<Tuple2<String, Integer>, Tuple2<String, Integer>> testHarness =
createTestHarness(operator);
testHarness.open();
ConcurrentLinkedQueue<Object> expected = new ConcurrentLinkedQueue<>();
ConcurrentLinkedQueue<Object> sideExpected = new ConcurrentLinkedQueue<>();
testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 1000));
testHarness.processWatermark(new Watermark(1999));
expected.add(new StreamRecord<>(new Tuple2<>("key2", 1), 1999));
expected.add(new Watermark(1999));
testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 2000));
testHarness.processWatermark(new Watermark(3000));
expected.add(new StreamRecord<>(new Tuple2<>("key2", 2), 2999));
expected.add(new Watermark(3000));
testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 1), 3001));
// lateness is set to 0 and window size = 3 sec and slide 1, the following 2 elements (2400)
// are assigned to windows ending at 2999, 3999, 4999.
// The 2999 is dropped because it is already late (WM = 2999) but the rest are kept.
testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 2400));
testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 2400));
testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 1), 3001));
testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 3900));
testHarness.processWatermark(new Watermark(6000));
expected.add(new StreamRecord<>(new Tuple2<>("key2", 5), 3999));
expected.add(new StreamRecord<>(new Tuple2<>("key1", 2), 3999));
expected.add(new StreamRecord<>(new Tuple2<>("key2", 4), 4999));
expected.add(new StreamRecord<>(new Tuple2<>("key1", 2), 4999));
expected.add(new StreamRecord<>(new Tuple2<>("key2", 1), 5999));
expected.add(new StreamRecord<>(new Tuple2<>("key1", 2), 5999));
expected.add(new Watermark(6000));
// sideoutput element due to lateness
testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 1), 3001));
sideExpected.add(new StreamRecord<>(new Tuple2<>("key1", 1), 3001));
testHarness.processWatermark(new Watermark(25000));
expected.add(new Watermark(25000));
TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expected, testHarness.getOutput(), new Tuple2ResultSortComparator());
TestHarnessUtil.assertOutputEqualsSorted("SideOutput was not correct.", sideExpected, (Iterable) testHarness.getSideOutput(lateOutputTag), new Tuple2ResultSortComparator());
testHarness.close();
}
