org.apache.flink.streaming.api.functions.windowing.RichWindowFunction Java Examples
The following examples show how to use
org.apache.flink.streaming.api.functions.windowing.RichWindowFunction.
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
| Source File: EventTimeWindowCheckpointingITCase.java From flink with Apache License 2.0 | 4 votes |
|
@Test
public void testPreAggregatedTumblingTimeWindow() {
final int numElementsPerKey = numElementsPerKey();
final int windowSize = windowSize();
final int numKeys = numKeys();
try {
StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
env.setParallelism(PARALLELISM);
env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);
env.enableCheckpointing(100);
env.setRestartStrategy(RestartStrategies.fixedDelayRestart(1, 0));
env.getConfig().disableSysoutLogging();
env.setStateBackend(this.stateBackend);
env.getConfig().setUseSnapshotCompression(true);
env
.addSource(new FailingSource(new KeyedEventTimeGenerator(numKeys, windowSize), numElementsPerKey))
.rebalance()
.keyBy(0)
.timeWindow(Time.of(windowSize, MILLISECONDS))
.reduce(
new ReduceFunction<Tuple2<Long, IntType>>() {
@Override
public Tuple2<Long, IntType> reduce(
Tuple2<Long, IntType> a,
Tuple2<Long, IntType> b) {
return new Tuple2<>(a.f0, new IntType(a.f1.value + b.f1.value));
}
},
new RichWindowFunction<Tuple2<Long, IntType>, Tuple4<Long, Long, Long, IntType>, Tuple, TimeWindow>() {
private boolean open = false;
@Override
public void open(Configuration parameters) {
assertEquals(PARALLELISM, getRuntimeContext().getNumberOfParallelSubtasks());
open = true;
}
@Override
public void apply(
Tuple tuple,
TimeWindow window,
Iterable<Tuple2<Long, IntType>> input,
Collector<Tuple4<Long, Long, Long, IntType>> out) {
// validate that the function has been opened properly
assertTrue(open);
for (Tuple2<Long, IntType> in: input) {
final Tuple4<Long, Long, Long, IntType> output = new Tuple4<>(in.f0,
window.getStart(),
window.getEnd(),
in.f1);
out.collect(output);
}
}
})
.addSink(new ValidatingSink<>(
new SinkValidatorUpdateFun(numElementsPerKey),
new SinkValidatorCheckFun(numKeys, numElementsPerKey, windowSize))).setParallelism(1);
env.execute("Tumbling Window Test");
}
catch (Exception e) {
e.printStackTrace();
fail(e.getMessage());
}
}
Example #2
| Source File: WindowCheckpointingITCase.java From flink with Apache License 2.0 | 4 votes |
|
@Test
public void testSlidingProcessingTimeWindow() {
final int numElements = 3000;
try {
StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
env.setParallelism(PARALLELISM);
env.setStreamTimeCharacteristic(timeCharacteristic);
env.getConfig().setAutoWatermarkInterval(10);
env.enableCheckpointing(100);
env.setRestartStrategy(RestartStrategies.fixedDelayRestart(1, 0));
SinkValidatorUpdaterAndChecker updaterAndChecker =
new SinkValidatorUpdaterAndChecker(numElements, 3);
env
.addSource(new FailingSource(new Generator(), numElements, timeCharacteristic))
.rebalance()
.keyBy(0)
.timeWindow(Time.of(150, MILLISECONDS), Time.of(50, MILLISECONDS))
.apply(new RichWindowFunction<Tuple2<Long, IntType>, Tuple2<Long, IntType>, Tuple, TimeWindow>() {
private boolean open = false;
@Override
public void open(Configuration parameters) {
assertEquals(PARALLELISM, getRuntimeContext().getNumberOfParallelSubtasks());
open = true;
}
@Override
public void apply(
Tuple tuple,
TimeWindow window,
Iterable<Tuple2<Long, IntType>> values,
Collector<Tuple2<Long, IntType>> out) {
// validate that the function has been opened properly
assertTrue(open);
for (Tuple2<Long, IntType> value : values) {
assertEquals(value.f0.intValue(), value.f1.value);
out.collect(new Tuple2<>(value.f0, new IntType(1)));
}
}
})
.addSink(new ValidatingSink<>(updaterAndChecker, updaterAndChecker, timeCharacteristic))
.setParallelism(1);
tryExecute(env, "Sliding Window Test");
}
catch (Exception e) {
e.printStackTrace();
fail(e.getMessage());
}
}
Example #3
| Source File: WindowCheckpointingITCase.java From flink with Apache License 2.0 | 4 votes |
|
@Test
public void testTumblingProcessingTimeWindow() {
final int numElements = 3000;
try {
StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
env.setParallelism(PARALLELISM);
env.setStreamTimeCharacteristic(timeCharacteristic);
env.getConfig().setAutoWatermarkInterval(10);
env.enableCheckpointing(100);
env.setRestartStrategy(RestartStrategies.fixedDelayRestart(1, 0));
SinkValidatorUpdaterAndChecker updaterAndChecker =
new SinkValidatorUpdaterAndChecker(numElements, 1);
env
.addSource(new FailingSource(new Generator(), numElements, timeCharacteristic))
.rebalance()
.keyBy(0)
.timeWindow(Time.of(100, MILLISECONDS))
.apply(new RichWindowFunction<Tuple2<Long, IntType>, Tuple2<Long, IntType>, Tuple, TimeWindow>() {
private boolean open = false;
@Override
public void open(Configuration parameters) {
assertEquals(PARALLELISM, getRuntimeContext().getNumberOfParallelSubtasks());
open = true;
}
@Override
public void apply(
Tuple tuple,
TimeWindow window,
Iterable<Tuple2<Long, IntType>> values,
Collector<Tuple2<Long, IntType>> out) {
// validate that the function has been opened properly
assertTrue(open);
for (Tuple2<Long, IntType> value : values) {
assertEquals(value.f0.intValue(), value.f1.value);
out.collect(new Tuple2<>(value.f0, new IntType(1)));
}
}
})
.addSink(new ValidatingSink<>(updaterAndChecker, updaterAndChecker, timeCharacteristic))
.setParallelism(1);
tryExecute(env, "Tumbling Window Test");
}
catch (Exception e) {
e.printStackTrace();
fail(e.getMessage());
}
}
Example #4
| Source File: EventTimeWindowCheckpointingITCase.java From flink with Apache License 2.0 | 4 votes |
|
@Test
public void testPreAggregatedSlidingTimeWindow() {
final int numElementsPerKey = numElementsPerKey();
final int windowSize = windowSize();
final int windowSlide = windowSlide();
final int numKeys = numKeys();
try {
StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
env.setParallelism(PARALLELISM);
env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);
env.enableCheckpointing(100);
env.setRestartStrategy(RestartStrategies.fixedDelayRestart(1, 0));
env.setStateBackend(this.stateBackend);
env.getConfig().setUseSnapshotCompression(true);
env
.addSource(new FailingSource(new KeyedEventTimeGenerator(numKeys, windowSlide), numElementsPerKey))
.rebalance()
.keyBy(0)
.timeWindow(Time.of(windowSize, MILLISECONDS), Time.of(windowSlide, MILLISECONDS))
.reduce(
new ReduceFunction<Tuple2<Long, IntType>>() {
@Override
public Tuple2<Long, IntType> reduce(
Tuple2<Long, IntType> a,
Tuple2<Long, IntType> b) {
// validate that the function has been opened properly
return new Tuple2<>(a.f0, new IntType(a.f1.value + b.f1.value));
}
},
new RichWindowFunction<Tuple2<Long, IntType>, Tuple4<Long, Long, Long, IntType>, Tuple, TimeWindow>() {
private boolean open = false;
@Override
public void open(Configuration parameters) {
assertEquals(PARALLELISM, getRuntimeContext().getNumberOfParallelSubtasks());
open = true;
}
@Override
public void apply(
Tuple tuple,
TimeWindow window,
Iterable<Tuple2<Long, IntType>> input,
Collector<Tuple4<Long, Long, Long, IntType>> out) {
// validate that the function has been opened properly
assertTrue(open);
for (Tuple2<Long, IntType> in: input) {
out.collect(new Tuple4<>(in.f0,
window.getStart(),
window.getEnd(),
in.f1));
}
}
})
.addSink(new ValidatingSink<>(
new SinkValidatorUpdateFun(numElementsPerKey),
new SinkValidatorCheckFun(numKeys, numElementsPerKey, windowSlide))).setParallelism(1);
env.execute("Tumbling Window Test");
}
catch (Exception e) {
e.printStackTrace();
fail(e.getMessage());
}
}
Example #5
| Source File: EventTimeWindowCheckpointingITCase.java From flink with Apache License 2.0 | 4 votes |
|
@Test
public void testPreAggregatedTumblingTimeWindow() {
final int numElementsPerKey = numElementsPerKey();
final int windowSize = windowSize();
final int numKeys = numKeys();
try {
StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
env.setParallelism(PARALLELISM);
env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);
env.enableCheckpointing(100);
env.setRestartStrategy(RestartStrategies.fixedDelayRestart(1, 0));
env.setStateBackend(this.stateBackend);
env.getConfig().setUseSnapshotCompression(true);
env
.addSource(new FailingSource(new KeyedEventTimeGenerator(numKeys, windowSize), numElementsPerKey))
.rebalance()
.keyBy(0)
.timeWindow(Time.of(windowSize, MILLISECONDS))
.reduce(
new ReduceFunction<Tuple2<Long, IntType>>() {
@Override
public Tuple2<Long, IntType> reduce(
Tuple2<Long, IntType> a,
Tuple2<Long, IntType> b) {
return new Tuple2<>(a.f0, new IntType(a.f1.value + b.f1.value));
}
},
new RichWindowFunction<Tuple2<Long, IntType>, Tuple4<Long, Long, Long, IntType>, Tuple, TimeWindow>() {
private boolean open = false;
@Override
public void open(Configuration parameters) {
assertEquals(PARALLELISM, getRuntimeContext().getNumberOfParallelSubtasks());
open = true;
}
@Override
public void apply(
Tuple tuple,
TimeWindow window,
Iterable<Tuple2<Long, IntType>> input,
Collector<Tuple4<Long, Long, Long, IntType>> out) {
// validate that the function has been opened properly
assertTrue(open);
for (Tuple2<Long, IntType> in: input) {
final Tuple4<Long, Long, Long, IntType> output = new Tuple4<>(in.f0,
window.getStart(),
window.getEnd(),
in.f1);
out.collect(output);
}
}
})
.addSink(new ValidatingSink<>(
new SinkValidatorUpdateFun(numElementsPerKey),
new SinkValidatorCheckFun(numKeys, numElementsPerKey, windowSize))).setParallelism(1);
env.execute("Tumbling Window Test");
}
catch (Exception e) {
e.printStackTrace();
fail(e.getMessage());
}
}
Example #6
| Source File: EventTimeWindowCheckpointingITCase.java From flink with Apache License 2.0 | 4 votes |
|
@Test
public void testSlidingTimeWindow() {
final int numElementsPerKey = numElementsPerKey();
final int windowSize = windowSize();
final int windowSlide = windowSlide();
final int numKeys = numKeys();
try {
StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
env.setMaxParallelism(2 * PARALLELISM);
env.setParallelism(PARALLELISM);
env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);
env.enableCheckpointing(100);
env.setRestartStrategy(RestartStrategies.fixedDelayRestart(1, 0));
env.setStateBackend(this.stateBackend);
env.getConfig().setUseSnapshotCompression(true);
env
.addSource(new FailingSource(new KeyedEventTimeGenerator(numKeys, windowSlide), numElementsPerKey))
.rebalance()
.keyBy(0)
.timeWindow(Time.of(windowSize, MILLISECONDS), Time.of(windowSlide, MILLISECONDS))
.apply(new RichWindowFunction<Tuple2<Long, IntType>, Tuple4<Long, Long, Long, IntType>, Tuple, TimeWindow>() {
private boolean open = false;
@Override
public void open(Configuration parameters) {
assertEquals(PARALLELISM, getRuntimeContext().getNumberOfParallelSubtasks());
open = true;
}
@Override
public void apply(
Tuple tuple,
TimeWindow window,
Iterable<Tuple2<Long, IntType>> values,
Collector<Tuple4<Long, Long, Long, IntType>> out) {
// validate that the function has been opened properly
assertTrue(open);
int sum = 0;
long key = -1;
for (Tuple2<Long, IntType> value : values) {
sum += value.f1.value;
key = value.f0;
}
final Tuple4<Long, Long, Long, IntType> output =
new Tuple4<>(key, window.getStart(), window.getEnd(), new IntType(sum));
out.collect(output);
}
})
.addSink(new ValidatingSink<>(
new SinkValidatorUpdateFun(numElementsPerKey),
new SinkValidatorCheckFun(numKeys, numElementsPerKey, windowSlide))).setParallelism(1);
env.execute("Tumbling Window Test");
}
catch (Exception e) {
e.printStackTrace();
fail(e.getMessage());
}
}
Example #7
| Source File: EventTimeWindowCheckpointingITCase.java From flink with Apache License 2.0 | 4 votes |
|
public void doTestTumblingTimeWindowWithKVState(int maxParallelism) {
final int numElementsPerKey = numElementsPerKey();
final int windowSize = windowSize();
final int numKeys = numKeys();
try {
StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
env.setParallelism(PARALLELISM);
env.setMaxParallelism(maxParallelism);
env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);
env.enableCheckpointing(100);
env.setRestartStrategy(RestartStrategies.fixedDelayRestart(1, 0));
env.setStateBackend(this.stateBackend);
env.getConfig().setUseSnapshotCompression(true);
env
.addSource(new FailingSource(new KeyedEventTimeGenerator(numKeys, windowSize), numElementsPerKey))
.rebalance()
.keyBy(0)
.timeWindow(Time.of(windowSize, MILLISECONDS))
.apply(new RichWindowFunction<Tuple2<Long, IntType>, Tuple4<Long, Long, Long, IntType>, Tuple, TimeWindow>() {
private boolean open = false;
private ValueState<Integer> count;
@Override
public void open(Configuration parameters) {
assertEquals(PARALLELISM, getRuntimeContext().getNumberOfParallelSubtasks());
open = true;
count = getRuntimeContext().getState(
new ValueStateDescriptor<>("count", Integer.class, 0));
}
@Override
public void apply(
Tuple tuple,
TimeWindow window,
Iterable<Tuple2<Long, IntType>> values,
Collector<Tuple4<Long, Long, Long, IntType>> out) throws Exception {
// the window count state starts with the key, so that we get
// different count results for each key
if (count.value() == 0) {
count.update(tuple.<Long>getField(0).intValue());
}
// validate that the function has been opened properly
assertTrue(open);
count.update(count.value() + 1);
out.collect(new Tuple4<>(tuple.<Long>getField(0), window.getStart(), window.getEnd(), new IntType(count.value())));
}
})
.addSink(new ValidatingSink<>(
new CountingSinkValidatorUpdateFun(),
new SinkValidatorCheckFun(numKeys, numElementsPerKey, windowSize))).setParallelism(1);
env.execute("Tumbling Window Test");
}
catch (Exception e) {
e.printStackTrace();
fail(e.getMessage());
}
}
Example #8
| Source File: EventTimeWindowCheckpointingITCase.java From flink with Apache License 2.0 | 4 votes |
|
@Test
public void testTumblingTimeWindow() {
final int numElementsPerKey = numElementsPerKey();
final int windowSize = windowSize();
final int numKeys = numKeys();
try {
StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
env.setParallelism(PARALLELISM);
env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);
env.enableCheckpointing(100);
env.setRestartStrategy(RestartStrategies.fixedDelayRestart(1, 0));
env.setStateBackend(this.stateBackend);
env.getConfig().setUseSnapshotCompression(true);
env
.addSource(new FailingSource(new KeyedEventTimeGenerator(numKeys, windowSize), numElementsPerKey))
.rebalance()
.keyBy(0)
.timeWindow(Time.of(windowSize, MILLISECONDS))
.apply(new RichWindowFunction<Tuple2<Long, IntType>, Tuple4<Long, Long, Long, IntType>, Tuple, TimeWindow>() {
private boolean open = false;
@Override
public void open(Configuration parameters) {
assertEquals(PARALLELISM, getRuntimeContext().getNumberOfParallelSubtasks());
open = true;
}
@Override
public void apply(
Tuple tuple,
TimeWindow window,
Iterable<Tuple2<Long, IntType>> values,
Collector<Tuple4<Long, Long, Long, IntType>> out) {
// validate that the function has been opened properly
assertTrue(open);
int sum = 0;
long key = -1;
for (Tuple2<Long, IntType> value : values) {
sum += value.f1.value;
key = value.f0;
}
final Tuple4<Long, Long, Long, IntType> result =
new Tuple4<>(key, window.getStart(), window.getEnd(), new IntType(sum));
out.collect(result);
}
})
.addSink(new ValidatingSink<>(
new SinkValidatorUpdateFun(numElementsPerKey),
new SinkValidatorCheckFun(numKeys, numElementsPerKey, windowSize))).setParallelism(1);
env.execute("Tumbling Window Test");
}
catch (Exception e) {
e.printStackTrace();
fail(e.getMessage());
}
}
Example #9
| Source File: WindowCheckpointingITCase.java From flink with Apache License 2.0 | 4 votes |
|
@Test
public void testSlidingProcessingTimeWindow() {
final int numElements = 3000;
try {
StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
env.setParallelism(PARALLELISM);
env.setStreamTimeCharacteristic(timeCharacteristic);
env.getConfig().setAutoWatermarkInterval(10);
env.enableCheckpointing(100);
env.setRestartStrategy(RestartStrategies.fixedDelayRestart(1, 0));
env.getConfig().disableSysoutLogging();
SinkValidatorUpdaterAndChecker updaterAndChecker =
new SinkValidatorUpdaterAndChecker(numElements, 3);
env
.addSource(new FailingSource(new Generator(), numElements, timeCharacteristic))
.rebalance()
.keyBy(0)
.timeWindow(Time.of(150, MILLISECONDS), Time.of(50, MILLISECONDS))
.apply(new RichWindowFunction<Tuple2<Long, IntType>, Tuple2<Long, IntType>, Tuple, TimeWindow>() {
private boolean open = false;
@Override
public void open(Configuration parameters) {
assertEquals(PARALLELISM, getRuntimeContext().getNumberOfParallelSubtasks());
open = true;
}
@Override
public void apply(
Tuple tuple,
TimeWindow window,
Iterable<Tuple2<Long, IntType>> values,
Collector<Tuple2<Long, IntType>> out) {
// validate that the function has been opened properly
assertTrue(open);
for (Tuple2<Long, IntType> value : values) {
assertEquals(value.f0.intValue(), value.f1.value);
out.collect(new Tuple2<>(value.f0, new IntType(1)));
}
}
})
.addSink(new ValidatingSink<>(updaterAndChecker, updaterAndChecker, timeCharacteristic))
.setParallelism(1);
tryExecute(env, "Sliding Window Test");
}
catch (Exception e) {
e.printStackTrace();
fail(e.getMessage());
}
}
Example #10
| Source File: WindowCheckpointingITCase.java From flink with Apache License 2.0 | 4 votes |
|
@Test
public void testTumblingProcessingTimeWindow() {
final int numElements = 3000;
try {
StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
env.setParallelism(PARALLELISM);
env.setStreamTimeCharacteristic(timeCharacteristic);
env.getConfig().setAutoWatermarkInterval(10);
env.enableCheckpointing(100);
env.setRestartStrategy(RestartStrategies.fixedDelayRestart(1, 0));
env.getConfig().disableSysoutLogging();
SinkValidatorUpdaterAndChecker updaterAndChecker =
new SinkValidatorUpdaterAndChecker(numElements, 1);
env
.addSource(new FailingSource(new Generator(), numElements, timeCharacteristic))
.rebalance()
.keyBy(0)
.timeWindow(Time.of(100, MILLISECONDS))
.apply(new RichWindowFunction<Tuple2<Long, IntType>, Tuple2<Long, IntType>, Tuple, TimeWindow>() {
private boolean open = false;
@Override
public void open(Configuration parameters) {
assertEquals(PARALLELISM, getRuntimeContext().getNumberOfParallelSubtasks());
open = true;
}
@Override
public void apply(
Tuple tuple,
TimeWindow window,
Iterable<Tuple2<Long, IntType>> values,
Collector<Tuple2<Long, IntType>> out) {
// validate that the function has been opened properly
assertTrue(open);
for (Tuple2<Long, IntType> value : values) {
assertEquals(value.f0.intValue(), value.f1.value);
out.collect(new Tuple2<>(value.f0, new IntType(1)));
}
}
})
.addSink(new ValidatingSink<>(updaterAndChecker, updaterAndChecker, timeCharacteristic))
.setParallelism(1);
tryExecute(env, "Tumbling Window Test");
}
catch (Exception e) {
e.printStackTrace();
fail(e.getMessage());
}
}
Example #11
| Source File: EventTimeWindowCheckpointingITCase.java From flink with Apache License 2.0 | 4 votes |
|
@Test
public void testPreAggregatedSlidingTimeWindow() {
final int numElementsPerKey = numElementsPerKey();
final int windowSize = windowSize();
final int windowSlide = windowSlide();
final int numKeys = numKeys();
try {
StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
env.setParallelism(PARALLELISM);
env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);
env.enableCheckpointing(100);
env.setRestartStrategy(RestartStrategies.fixedDelayRestart(1, 0));
env.getConfig().disableSysoutLogging();
env.setStateBackend(this.stateBackend);
env.getConfig().setUseSnapshotCompression(true);
env
.addSource(new FailingSource(new KeyedEventTimeGenerator(numKeys, windowSlide), numElementsPerKey))
.rebalance()
.keyBy(0)
.timeWindow(Time.of(windowSize, MILLISECONDS), Time.of(windowSlide, MILLISECONDS))
.reduce(
new ReduceFunction<Tuple2<Long, IntType>>() {
@Override
public Tuple2<Long, IntType> reduce(
Tuple2<Long, IntType> a,
Tuple2<Long, IntType> b) {
// validate that the function has been opened properly
return new Tuple2<>(a.f0, new IntType(a.f1.value + b.f1.value));
}
},
new RichWindowFunction<Tuple2<Long, IntType>, Tuple4<Long, Long, Long, IntType>, Tuple, TimeWindow>() {
private boolean open = false;
@Override
public void open(Configuration parameters) {
assertEquals(PARALLELISM, getRuntimeContext().getNumberOfParallelSubtasks());
open = true;
}
@Override
public void apply(
Tuple tuple,
TimeWindow window,
Iterable<Tuple2<Long, IntType>> input,
Collector<Tuple4<Long, Long, Long, IntType>> out) {
// validate that the function has been opened properly
assertTrue(open);
for (Tuple2<Long, IntType> in: input) {
out.collect(new Tuple4<>(in.f0,
window.getStart(),
window.getEnd(),
in.f1));
}
}
})
.addSink(new ValidatingSink<>(
new SinkValidatorUpdateFun(numElementsPerKey),
new SinkValidatorCheckFun(numKeys, numElementsPerKey, windowSlide))).setParallelism(1);
env.execute("Tumbling Window Test");
}
catch (Exception e) {
e.printStackTrace();
fail(e.getMessage());
}
}
Example #12
| Source File: EventTimeWindowCheckpointingITCase.java From Flink-CEPplus with Apache License 2.0 | 4 votes |
|
@Test
public void testTumblingTimeWindow() {
final int numElementsPerKey = numElementsPerKey();
final int windowSize = windowSize();
final int numKeys = numKeys();
try {
StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
env.setParallelism(PARALLELISM);
env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);
env.enableCheckpointing(100);
env.setRestartStrategy(RestartStrategies.fixedDelayRestart(1, 0));
env.getConfig().disableSysoutLogging();
env.setStateBackend(this.stateBackend);
env.getConfig().setUseSnapshotCompression(true);
env
.addSource(new FailingSource(new KeyedEventTimeGenerator(numKeys, windowSize), numElementsPerKey))
.rebalance()
.keyBy(0)
.timeWindow(Time.of(windowSize, MILLISECONDS))
.apply(new RichWindowFunction<Tuple2<Long, IntType>, Tuple4<Long, Long, Long, IntType>, Tuple, TimeWindow>() {
private boolean open = false;
@Override
public void open(Configuration parameters) {
assertEquals(PARALLELISM, getRuntimeContext().getNumberOfParallelSubtasks());
open = true;
}
@Override
public void apply(
Tuple tuple,
TimeWindow window,
Iterable<Tuple2<Long, IntType>> values,
Collector<Tuple4<Long, Long, Long, IntType>> out) {
// validate that the function has been opened properly
assertTrue(open);
int sum = 0;
long key = -1;
for (Tuple2<Long, IntType> value : values) {
sum += value.f1.value;
key = value.f0;
}
final Tuple4<Long, Long, Long, IntType> result =
new Tuple4<>(key, window.getStart(), window.getEnd(), new IntType(sum));
out.collect(result);
}
})
.addSink(new ValidatingSink<>(
new SinkValidatorUpdateFun(numElementsPerKey),
new SinkValidatorCheckFun(numKeys, numElementsPerKey, windowSize))).setParallelism(1);
env.execute("Tumbling Window Test");
}
catch (Exception e) {
e.printStackTrace();
fail(e.getMessage());
}
}
Example #13
| Source File: EventTimeWindowCheckpointingITCase.java From flink with Apache License 2.0 | 4 votes |
|
@Test
public void testSlidingTimeWindow() {
final int numElementsPerKey = numElementsPerKey();
final int windowSize = windowSize();
final int windowSlide = windowSlide();
final int numKeys = numKeys();
try {
StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
env.setMaxParallelism(2 * PARALLELISM);
env.setParallelism(PARALLELISM);
env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);
env.enableCheckpointing(100);
env.setRestartStrategy(RestartStrategies.fixedDelayRestart(1, 0));
env.getConfig().disableSysoutLogging();
env.setStateBackend(this.stateBackend);
env.getConfig().setUseSnapshotCompression(true);
env
.addSource(new FailingSource(new KeyedEventTimeGenerator(numKeys, windowSlide), numElementsPerKey))
.rebalance()
.keyBy(0)
.timeWindow(Time.of(windowSize, MILLISECONDS), Time.of(windowSlide, MILLISECONDS))
.apply(new RichWindowFunction<Tuple2<Long, IntType>, Tuple4<Long, Long, Long, IntType>, Tuple, TimeWindow>() {
private boolean open = false;
@Override
public void open(Configuration parameters) {
assertEquals(PARALLELISM, getRuntimeContext().getNumberOfParallelSubtasks());
open = true;
}
@Override
public void apply(
Tuple tuple,
TimeWindow window,
Iterable<Tuple2<Long, IntType>> values,
Collector<Tuple4<Long, Long, Long, IntType>> out) {
// validate that the function has been opened properly
assertTrue(open);
int sum = 0;
long key = -1;
for (Tuple2<Long, IntType> value : values) {
sum += value.f1.value;
key = value.f0;
}
final Tuple4<Long, Long, Long, IntType> output =
new Tuple4<>(key, window.getStart(), window.getEnd(), new IntType(sum));
out.collect(output);
}
})
.addSink(new ValidatingSink<>(
new SinkValidatorUpdateFun(numElementsPerKey),
new SinkValidatorCheckFun(numKeys, numElementsPerKey, windowSlide))).setParallelism(1);
env.execute("Tumbling Window Test");
}
catch (Exception e) {
e.printStackTrace();
fail(e.getMessage());
}
}
Example #14
| Source File: EventTimeWindowCheckpointingITCase.java From flink with Apache License 2.0 | 4 votes |
|
public void doTestTumblingTimeWindowWithKVState(int maxParallelism) {
final int numElementsPerKey = numElementsPerKey();
final int windowSize = windowSize();
final int numKeys = numKeys();
try {
StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
env.setParallelism(PARALLELISM);
env.setMaxParallelism(maxParallelism);
env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);
env.enableCheckpointing(100);
env.setRestartStrategy(RestartStrategies.fixedDelayRestart(1, 0));
env.getConfig().disableSysoutLogging();
env.setStateBackend(this.stateBackend);
env.getConfig().setUseSnapshotCompression(true);
env
.addSource(new FailingSource(new KeyedEventTimeGenerator(numKeys, windowSize), numElementsPerKey))
.rebalance()
.keyBy(0)
.timeWindow(Time.of(windowSize, MILLISECONDS))
.apply(new RichWindowFunction<Tuple2<Long, IntType>, Tuple4<Long, Long, Long, IntType>, Tuple, TimeWindow>() {
private boolean open = false;
private ValueState<Integer> count;
@Override
public void open(Configuration parameters) {
assertEquals(PARALLELISM, getRuntimeContext().getNumberOfParallelSubtasks());
open = true;
count = getRuntimeContext().getState(
new ValueStateDescriptor<>("count", Integer.class, 0));
}
@Override
public void apply(
Tuple tuple,
TimeWindow window,
Iterable<Tuple2<Long, IntType>> values,
Collector<Tuple4<Long, Long, Long, IntType>> out) throws Exception {
// the window count state starts with the key, so that we get
// different count results for each key
if (count.value() == 0) {
count.update(tuple.<Long>getField(0).intValue());
}
// validate that the function has been opened properly
assertTrue(open);
count.update(count.value() + 1);
out.collect(new Tuple4<>(tuple.<Long>getField(0), window.getStart(), window.getEnd(), new IntType(count.value())));
}
})
.addSink(new ValidatingSink<>(
new CountingSinkValidatorUpdateFun(),
new SinkValidatorCheckFun(numKeys, numElementsPerKey, windowSize))).setParallelism(1);
env.execute("Tumbling Window Test");
}
catch (Exception e) {
e.printStackTrace();
fail(e.getMessage());
}
}
Example #15
| Source File: EventTimeWindowCheckpointingITCase.java From flink with Apache License 2.0 | 4 votes |
|
@Test
public void testTumblingTimeWindow() {
final int numElementsPerKey = numElementsPerKey();
final int windowSize = windowSize();
final int numKeys = numKeys();
try {
StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
env.setParallelism(PARALLELISM);
env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);
env.enableCheckpointing(100);
env.setRestartStrategy(RestartStrategies.fixedDelayRestart(1, 0));
env.getConfig().disableSysoutLogging();
env.setStateBackend(this.stateBackend);
env.getConfig().setUseSnapshotCompression(true);
env
.addSource(new FailingSource(new KeyedEventTimeGenerator(numKeys, windowSize), numElementsPerKey))
.rebalance()
.keyBy(0)
.timeWindow(Time.of(windowSize, MILLISECONDS))
.apply(new RichWindowFunction<Tuple2<Long, IntType>, Tuple4<Long, Long, Long, IntType>, Tuple, TimeWindow>() {
private boolean open = false;
@Override
public void open(Configuration parameters) {
assertEquals(PARALLELISM, getRuntimeContext().getNumberOfParallelSubtasks());
open = true;
}
@Override
public void apply(
Tuple tuple,
TimeWindow window,
Iterable<Tuple2<Long, IntType>> values,
Collector<Tuple4<Long, Long, Long, IntType>> out) {
// validate that the function has been opened properly
assertTrue(open);
int sum = 0;
long key = -1;
for (Tuple2<Long, IntType> value : values) {
sum += value.f1.value;
key = value.f0;
}
final Tuple4<Long, Long, Long, IntType> result =
new Tuple4<>(key, window.getStart(), window.getEnd(), new IntType(sum));
out.collect(result);
}
})
.addSink(new ValidatingSink<>(
new SinkValidatorUpdateFun(numElementsPerKey),
new SinkValidatorCheckFun(numKeys, numElementsPerKey, windowSize))).setParallelism(1);
env.execute("Tumbling Window Test");
}
catch (Exception e) {
e.printStackTrace();
fail(e.getMessage());
}
}
Example #16
| Source File: WindowCheckpointingITCase.java From Flink-CEPplus with Apache License 2.0 | 4 votes |
|
@Test
public void testSlidingProcessingTimeWindow() {
final int numElements = 3000;
try {
StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
env.setParallelism(PARALLELISM);
env.setStreamTimeCharacteristic(timeCharacteristic);
env.getConfig().setAutoWatermarkInterval(10);
env.enableCheckpointing(100);
env.setRestartStrategy(RestartStrategies.fixedDelayRestart(1, 0));
env.getConfig().disableSysoutLogging();
SinkValidatorUpdaterAndChecker updaterAndChecker =
new SinkValidatorUpdaterAndChecker(numElements, 3);
env
.addSource(new FailingSource(new Generator(), numElements, timeCharacteristic))
.rebalance()
.keyBy(0)
.timeWindow(Time.of(150, MILLISECONDS), Time.of(50, MILLISECONDS))
.apply(new RichWindowFunction<Tuple2<Long, IntType>, Tuple2<Long, IntType>, Tuple, TimeWindow>() {
private boolean open = false;
@Override
public void open(Configuration parameters) {
assertEquals(PARALLELISM, getRuntimeContext().getNumberOfParallelSubtasks());
open = true;
}
@Override
public void apply(
Tuple tuple,
TimeWindow window,
Iterable<Tuple2<Long, IntType>> values,
Collector<Tuple2<Long, IntType>> out) {
// validate that the function has been opened properly
assertTrue(open);
for (Tuple2<Long, IntType> value : values) {
assertEquals(value.f0.intValue(), value.f1.value);
out.collect(new Tuple2<>(value.f0, new IntType(1)));
}
}
})
.addSink(new ValidatingSink<>(updaterAndChecker, updaterAndChecker, timeCharacteristic))
.setParallelism(1);
tryExecute(env, "Sliding Window Test");
}
catch (Exception e) {
e.printStackTrace();
fail(e.getMessage());
}
}
Example #17
| Source File: WindowCheckpointingITCase.java From Flink-CEPplus with Apache License 2.0 | 4 votes |
|
@Test
public void testTumblingProcessingTimeWindow() {
final int numElements = 3000;
try {
StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
env.setParallelism(PARALLELISM);
env.setStreamTimeCharacteristic(timeCharacteristic);
env.getConfig().setAutoWatermarkInterval(10);
env.enableCheckpointing(100);
env.setRestartStrategy(RestartStrategies.fixedDelayRestart(1, 0));
env.getConfig().disableSysoutLogging();
SinkValidatorUpdaterAndChecker updaterAndChecker =
new SinkValidatorUpdaterAndChecker(numElements, 1);
env
.addSource(new FailingSource(new Generator(), numElements, timeCharacteristic))
.rebalance()
.keyBy(0)
.timeWindow(Time.of(100, MILLISECONDS))
.apply(new RichWindowFunction<Tuple2<Long, IntType>, Tuple2<Long, IntType>, Tuple, TimeWindow>() {
private boolean open = false;
@Override
public void open(Configuration parameters) {
assertEquals(PARALLELISM, getRuntimeContext().getNumberOfParallelSubtasks());
open = true;
}
@Override
public void apply(
Tuple tuple,
TimeWindow window,
Iterable<Tuple2<Long, IntType>> values,
Collector<Tuple2<Long, IntType>> out) {
// validate that the function has been opened properly
assertTrue(open);
for (Tuple2<Long, IntType> value : values) {
assertEquals(value.f0.intValue(), value.f1.value);
out.collect(new Tuple2<>(value.f0, new IntType(1)));
}
}
})
.addSink(new ValidatingSink<>(updaterAndChecker, updaterAndChecker, timeCharacteristic))
.setParallelism(1);
tryExecute(env, "Tumbling Window Test");
}
catch (Exception e) {
e.printStackTrace();
fail(e.getMessage());
}
}
Example #18
| Source File: EventTimeWindowCheckpointingITCase.java From Flink-CEPplus with Apache License 2.0 | 4 votes |
|
@Test
public void testPreAggregatedSlidingTimeWindow() {
final int numElementsPerKey = numElementsPerKey();
final int windowSize = windowSize();
final int windowSlide = windowSlide();
final int numKeys = numKeys();
try {
StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
env.setParallelism(PARALLELISM);
env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);
env.enableCheckpointing(100);
env.setRestartStrategy(RestartStrategies.fixedDelayRestart(1, 0));
env.getConfig().disableSysoutLogging();
env.setStateBackend(this.stateBackend);
env.getConfig().setUseSnapshotCompression(true);
env
.addSource(new FailingSource(new KeyedEventTimeGenerator(numKeys, windowSlide), numElementsPerKey))
.rebalance()
.keyBy(0)
.timeWindow(Time.of(windowSize, MILLISECONDS), Time.of(windowSlide, MILLISECONDS))
.reduce(
new ReduceFunction<Tuple2<Long, IntType>>() {
@Override
public Tuple2<Long, IntType> reduce(
Tuple2<Long, IntType> a,
Tuple2<Long, IntType> b) {
// validate that the function has been opened properly
return new Tuple2<>(a.f0, new IntType(a.f1.value + b.f1.value));
}
},
new RichWindowFunction<Tuple2<Long, IntType>, Tuple4<Long, Long, Long, IntType>, Tuple, TimeWindow>() {
private boolean open = false;
@Override
public void open(Configuration parameters) {
assertEquals(PARALLELISM, getRuntimeContext().getNumberOfParallelSubtasks());
open = true;
}
@Override
public void apply(
Tuple tuple,
TimeWindow window,
Iterable<Tuple2<Long, IntType>> input,
Collector<Tuple4<Long, Long, Long, IntType>> out) {
// validate that the function has been opened properly
assertTrue(open);
for (Tuple2<Long, IntType> in: input) {
out.collect(new Tuple4<>(in.f0,
window.getStart(),
window.getEnd(),
in.f1));
}
}
})
.addSink(new ValidatingSink<>(
new SinkValidatorUpdateFun(numElementsPerKey),
new SinkValidatorCheckFun(numKeys, numElementsPerKey, windowSlide))).setParallelism(1);
env.execute("Tumbling Window Test");
}
catch (Exception e) {
e.printStackTrace();
fail(e.getMessage());
}
}
Example #19
| Source File: EventTimeWindowCheckpointingITCase.java From Flink-CEPplus with Apache License 2.0 | 4 votes |
|
@Test
public void testPreAggregatedTumblingTimeWindow() {
final int numElementsPerKey = numElementsPerKey();
final int windowSize = windowSize();
final int numKeys = numKeys();
try {
StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
env.setParallelism(PARALLELISM);
env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);
env.enableCheckpointing(100);
env.setRestartStrategy(RestartStrategies.fixedDelayRestart(1, 0));
env.getConfig().disableSysoutLogging();
env.setStateBackend(this.stateBackend);
env.getConfig().setUseSnapshotCompression(true);
env
.addSource(new FailingSource(new KeyedEventTimeGenerator(numKeys, windowSize), numElementsPerKey))
.rebalance()
.keyBy(0)
.timeWindow(Time.of(windowSize, MILLISECONDS))
.reduce(
new ReduceFunction<Tuple2<Long, IntType>>() {
@Override
public Tuple2<Long, IntType> reduce(
Tuple2<Long, IntType> a,
Tuple2<Long, IntType> b) {
return new Tuple2<>(a.f0, new IntType(a.f1.value + b.f1.value));
}
},
new RichWindowFunction<Tuple2<Long, IntType>, Tuple4<Long, Long, Long, IntType>, Tuple, TimeWindow>() {
private boolean open = false;
@Override
public void open(Configuration parameters) {
assertEquals(PARALLELISM, getRuntimeContext().getNumberOfParallelSubtasks());
open = true;
}
@Override
public void apply(
Tuple tuple,
TimeWindow window,
Iterable<Tuple2<Long, IntType>> input,
Collector<Tuple4<Long, Long, Long, IntType>> out) {
// validate that the function has been opened properly
assertTrue(open);
for (Tuple2<Long, IntType> in: input) {
final Tuple4<Long, Long, Long, IntType> output = new Tuple4<>(in.f0,
window.getStart(),
window.getEnd(),
in.f1);
out.collect(output);
}
}
})
.addSink(new ValidatingSink<>(
new SinkValidatorUpdateFun(numElementsPerKey),
new SinkValidatorCheckFun(numKeys, numElementsPerKey, windowSize))).setParallelism(1);
env.execute("Tumbling Window Test");
}
catch (Exception e) {
e.printStackTrace();
fail(e.getMessage());
}
}
Example #20
| Source File: EventTimeWindowCheckpointingITCase.java From Flink-CEPplus with Apache License 2.0 | 4 votes |
|
@Test
public void testSlidingTimeWindow() {
final int numElementsPerKey = numElementsPerKey();
final int windowSize = windowSize();
final int windowSlide = windowSlide();
final int numKeys = numKeys();
try {
StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
env.setMaxParallelism(2 * PARALLELISM);
env.setParallelism(PARALLELISM);
env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);
env.enableCheckpointing(100);
env.setRestartStrategy(RestartStrategies.fixedDelayRestart(1, 0));
env.getConfig().disableSysoutLogging();
env.setStateBackend(this.stateBackend);
env.getConfig().setUseSnapshotCompression(true);
env
.addSource(new FailingSource(new KeyedEventTimeGenerator(numKeys, windowSlide), numElementsPerKey))
.rebalance()
.keyBy(0)
.timeWindow(Time.of(windowSize, MILLISECONDS), Time.of(windowSlide, MILLISECONDS))
.apply(new RichWindowFunction<Tuple2<Long, IntType>, Tuple4<Long, Long, Long, IntType>, Tuple, TimeWindow>() {
private boolean open = false;
@Override
public void open(Configuration parameters) {
assertEquals(PARALLELISM, getRuntimeContext().getNumberOfParallelSubtasks());
open = true;
}
@Override
public void apply(
Tuple tuple,
TimeWindow window,
Iterable<Tuple2<Long, IntType>> values,
Collector<Tuple4<Long, Long, Long, IntType>> out) {
// validate that the function has been opened properly
assertTrue(open);
int sum = 0;
long key = -1;
for (Tuple2<Long, IntType> value : values) {
sum += value.f1.value;
key = value.f0;
}
final Tuple4<Long, Long, Long, IntType> output =
new Tuple4<>(key, window.getStart(), window.getEnd(), new IntType(sum));
out.collect(output);
}
})
.addSink(new ValidatingSink<>(
new SinkValidatorUpdateFun(numElementsPerKey),
new SinkValidatorCheckFun(numKeys, numElementsPerKey, windowSlide))).setParallelism(1);
env.execute("Tumbling Window Test");
}
catch (Exception e) {
e.printStackTrace();
fail(e.getMessage());
}
}
Example #21
| Source File: EventTimeWindowCheckpointingITCase.java From Flink-CEPplus with Apache License 2.0 | 4 votes |
|
public void doTestTumblingTimeWindowWithKVState(int maxParallelism) {
final int numElementsPerKey = numElementsPerKey();
final int windowSize = windowSize();
final int numKeys = numKeys();
try {
StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
env.setParallelism(PARALLELISM);
env.setMaxParallelism(maxParallelism);
env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);
env.enableCheckpointing(100);
env.setRestartStrategy(RestartStrategies.fixedDelayRestart(1, 0));
env.getConfig().disableSysoutLogging();
env.setStateBackend(this.stateBackend);
env.getConfig().setUseSnapshotCompression(true);
env
.addSource(new FailingSource(new KeyedEventTimeGenerator(numKeys, windowSize), numElementsPerKey))
.rebalance()
.keyBy(0)
.timeWindow(Time.of(windowSize, MILLISECONDS))
.apply(new RichWindowFunction<Tuple2<Long, IntType>, Tuple4<Long, Long, Long, IntType>, Tuple, TimeWindow>() {
private boolean open = false;
private ValueState<Integer> count;
@Override
public void open(Configuration parameters) {
assertEquals(PARALLELISM, getRuntimeContext().getNumberOfParallelSubtasks());
open = true;
count = getRuntimeContext().getState(
new ValueStateDescriptor<>("count", Integer.class, 0));
}
@Override
public void apply(
Tuple tuple,
TimeWindow window,
Iterable<Tuple2<Long, IntType>> values,
Collector<Tuple4<Long, Long, Long, IntType>> out) throws Exception {
// the window count state starts with the key, so that we get
// different count results for each key
if (count.value() == 0) {
count.update(tuple.<Long>getField(0).intValue());
}
// validate that the function has been opened properly
assertTrue(open);
count.update(count.value() + 1);
out.collect(new Tuple4<>(tuple.<Long>getField(0), window.getStart(), window.getEnd(), new IntType(count.value())));
}
})
.addSink(new ValidatingSink<>(
new CountingSinkValidatorUpdateFun(),
new SinkValidatorCheckFun(numKeys, numElementsPerKey, windowSize))).setParallelism(1);
env.execute("Tumbling Window Test");
}
catch (Exception e) {
e.printStackTrace();
fail(e.getMessage());
}
}
