org.apache.flink.streaming.api.functions.windowing.WindowFunction Java Examples

@Test
public void testApplyWindowState() throws Exception {
	final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
	env.setStreamTimeCharacteristic(TimeCharacteristic.IngestionTime);
	env.registerTypeWithKryoSerializer(File.class, JavaSerializer.class);

	DataStream<File> src = env.fromElements(new File("/"));

	SingleOutputStreamOperator<?> result = src
			.keyBy(new KeySelector<File, String>() {
				@Override
				public String getKey(File value) {
					return null;
				}
			})
			.timeWindow(Time.milliseconds(1000))
			.apply(new WindowFunction<File, String, String, TimeWindow>() {
				@Override
				public void apply(String s, TimeWindow window,
									Iterable<File> input, Collector<String> out) {}
			});

	validateListStateDescriptorConfigured(result);
}
 

@Test
public void testApplyWindowState() throws Exception {
	final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
	env.setStreamTimeCharacteristic(TimeCharacteristic.IngestionTime);
	env.registerTypeWithKryoSerializer(File.class, JavaSerializer.class);

	DataStream<File> src = env.fromElements(new File("/"));

	SingleOutputStreamOperator<?> result = src
			.keyBy(new KeySelector<File, String>() {
				@Override
				public String getKey(File value) {
					return null;
				}
			})
			.timeWindow(Time.milliseconds(1000))
			.apply(new WindowFunction<File, String, String, TimeWindow>() {
				@Override
				public void apply(String s, TimeWindow window,
									Iterable<File> input, Collector<String> out) {}
			});

	validateListStateDescriptorConfigured(result);
}
 

/**
 * Applies the given window function to each window. The window function is called for each
 * evaluation of the window for each key individually. The output of the window function is
 * interpreted as a regular non-windowed stream.
 *
 * <p>Arriving data is incrementally aggregated using the given aggregate function. This means
 * that the window function typically has only a single value to process when called.
 *
 * @param aggFunction The aggregate function that is used for incremental aggregation.
 * @param windowFunction The window function.
 *
 * @return The data stream that is the result of applying the window function to the window.
 *
 * @param <ACC> The type of the AggregateFunction's accumulator
 * @param <V> The type of AggregateFunction's result, and the WindowFunction's input
 * @param <R> The type of the elements in the resulting stream, equal to the
 *            WindowFunction's result type
 */
@PublicEvolving
public <ACC, V, R> SingleOutputStreamOperator<R> aggregate(
		AggregateFunction<T, ACC, V> aggFunction,
		WindowFunction<V, R, K, W> windowFunction) {

	checkNotNull(aggFunction, "aggFunction");
	checkNotNull(windowFunction, "windowFunction");

	TypeInformation<ACC> accumulatorType = TypeExtractor.getAggregateFunctionAccumulatorType(
			aggFunction, input.getType(), null, false);

	TypeInformation<V> aggResultType = TypeExtractor.getAggregateFunctionReturnType(
			aggFunction, input.getType(), null, false);

	TypeInformation<R> resultType = getWindowFunctionReturnType(windowFunction, aggResultType);

	return aggregate(aggFunction, windowFunction, accumulatorType, resultType);
}
 

/**
 * Applies the given window function to each window. The window function is called for each
 * evaluation of the window for each key individually. The output of the window function is
 * interpreted as a regular non-windowed stream.
 *
 * <p>Arriving data is incrementally aggregated using the given aggregate function. This means
 * that the window function typically has only a single value to process when called.
 *
 * @param aggFunction The aggregate function that is used for incremental aggregation.
 * @param windowFunction The window function.
 *
 * @return The data stream that is the result of applying the window function to the window.
 *
 * @param <ACC> The type of the AggregateFunction's accumulator
 * @param <V> The type of AggregateFunction's result, and the WindowFunction's input
 * @param <R> The type of the elements in the resulting stream, equal to the
 *            WindowFunction's result type
 */
@PublicEvolving
public <ACC, V, R> SingleOutputStreamOperator<R> aggregate(
		AggregateFunction<T, ACC, V> aggFunction,
		WindowFunction<V, R, K, W> windowFunction) {

	checkNotNull(aggFunction, "aggFunction");
	checkNotNull(windowFunction, "windowFunction");

	TypeInformation<ACC> accumulatorType = TypeExtractor.getAggregateFunctionAccumulatorType(
			aggFunction, input.getType(), null, false);

	TypeInformation<V> aggResultType = TypeExtractor.getAggregateFunctionReturnType(
			aggFunction, input.getType(), null, false);

	TypeInformation<R> resultType = getWindowFunctionReturnType(windowFunction, aggResultType);

	return aggregate(aggFunction, windowFunction, accumulatorType, resultType);
}
 

/**
 * Applies the given window function to each window. The window function is called for each
 * evaluation of the window for each key individually. The output of the window function is
 * interpreted as a regular non-windowed stream.
 *
 * <p>Arriving data is incrementally aggregated using the given aggregate function. This means
 * that the window function typically has only a single value to process when called.
 *
 * @param aggFunction The aggregate function that is used for incremental aggregation.
 * @param windowFunction The window function.
 *
 * @return The data stream that is the result of applying the window function to the window.
 *
 * @param <ACC> The type of the AggregateFunction's accumulator
 * @param <V> The type of AggregateFunction's result, and the WindowFunction's input
 * @param <R> The type of the elements in the resulting stream, equal to the
 *            WindowFunction's result type
 */
@PublicEvolving
public <ACC, V, R> SingleOutputStreamOperator<R> aggregate(
		AggregateFunction<T, ACC, V> aggFunction,
		WindowFunction<V, R, K, W> windowFunction) {

	checkNotNull(aggFunction, "aggFunction");
	checkNotNull(windowFunction, "windowFunction");

	TypeInformation<ACC> accumulatorType = TypeExtractor.getAggregateFunctionAccumulatorType(
			aggFunction, input.getType(), null, false);

	TypeInformation<V> aggResultType = TypeExtractor.getAggregateFunctionReturnType(
			aggFunction, input.getType(), null, false);

	TypeInformation<R> resultType = getWindowFunctionReturnType(windowFunction, aggResultType);

	return aggregate(aggFunction, windowFunction, accumulatorType, resultType);
}
 

@Test
public void testApplyWindowState() throws Exception {
	final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
	env.setStreamTimeCharacteristic(TimeCharacteristic.IngestionTime);
	env.registerTypeWithKryoSerializer(File.class, JavaSerializer.class);

	DataStream<File> src = env.fromElements(new File("/"));

	SingleOutputStreamOperator<?> result = src
			.keyBy(new KeySelector<File, String>() {
				@Override
				public String getKey(File value) {
					return null;
				}
			})
			.timeWindow(Time.milliseconds(1000))
			.apply(new WindowFunction<File, String, String, TimeWindow>() {
				@Override
				public void apply(String s, TimeWindow window,
									Iterable<File> input, Collector<String> out) {}
			});

	validateListStateDescriptorConfigured(result);
}
 

private void runTest(
		SourceFunction<SessionEvent<Integer, TestEventPayload>> dataSource,
		WindowFunction<SessionEvent<Integer, TestEventPayload>,
				String, Tuple, TimeWindow> windowFunction) throws Exception {

	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
	env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);
	WindowedStream<SessionEvent<Integer, TestEventPayload>, Tuple, TimeWindow> windowedStream =
			env.addSource(dataSource).keyBy("sessionKey")
			.window(EventTimeSessionWindows.withGap(Time.milliseconds(MAX_SESSION_EVENT_GAP_MS)));

	if (ALLOWED_LATENESS_MS != Long.MAX_VALUE) {
		windowedStream = windowedStream.allowedLateness(Time.milliseconds(ALLOWED_LATENESS_MS));
	}

	if (PURGE_WINDOW_ON_FIRE) {
		windowedStream = windowedStream.trigger(PurgingTrigger.of(EventTimeTrigger.create()));
	}

	windowedStream.apply(windowFunction).print();
	JobExecutionResult result = env.execute();

	// check that overall event counts match with our expectations. remember that late events within lateness will
	// each trigger a window!
	Assert.assertEquals(
		(LATE_EVENTS_PER_SESSION + 1) * NUMBER_OF_SESSIONS * EVENTS_PER_SESSION,
		(long) result.getAccumulatorResult(SESSION_COUNTER_ON_TIME_KEY));
	Assert.assertEquals(
		NUMBER_OF_SESSIONS * (LATE_EVENTS_PER_SESSION * (LATE_EVENTS_PER_SESSION + 1) / 2),
		(long) result.getAccumulatorResult(SESSION_COUNTER_LATE_KEY));
}
 

/**
 * Verifies that calls to timeWindow() instantiate a regular
 * windowOperator instead of an aligned one.
 */
@Test
public void testAlignedWindowDeprecation() throws Exception {
	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
	env.setStreamTimeCharacteristic(TimeCharacteristic.ProcessingTime);

	DataStream<Tuple2<String, Integer>> source = env.fromElements(Tuple2.of("hello", 1), Tuple2.of("hello", 2));

	DummyReducer reducer = new DummyReducer();

	DataStream<Tuple2<String, Integer>> window1 = source
			.keyBy(0)
			.timeWindow(Time.of(1000, TimeUnit.MILLISECONDS), Time.of(100, TimeUnit.MILLISECONDS))
			.reduce(reducer);

	OneInputTransformation<Tuple2<String, Integer>, Tuple2<String, Integer>> transform1 = (OneInputTransformation<Tuple2<String, Integer>, Tuple2<String, Integer>>) window1.getTransformation();
	OneInputStreamOperator<Tuple2<String, Integer>, Tuple2<String, Integer>> operator1 = transform1.getOperator();
	Assert.assertTrue(operator1 instanceof WindowOperator);

	DataStream<Tuple2<String, Integer>> window2 = source
			.keyBy(0)
			.timeWindow(Time.of(1000, TimeUnit.MILLISECONDS))
			.apply(new WindowFunction<Tuple2<String, Integer>, Tuple2<String, Integer>, Tuple, TimeWindow>() {
				private static final long serialVersionUID = 1L;

				@Override
				public void apply(Tuple tuple,
						TimeWindow window,
						Iterable<Tuple2<String, Integer>> values,
						Collector<Tuple2<String, Integer>> out) throws Exception {

				}
			});

	OneInputTransformation<Tuple2<String, Integer>, Tuple2<String, Integer>> transform2 = (OneInputTransformation<Tuple2<String, Integer>, Tuple2<String, Integer>>) window2.getTransformation();
	OneInputStreamOperator<Tuple2<String, Integer>, Tuple2<String, Integer>> operator2 = transform2.getOperator();
	Assert.assertTrue(operator2 instanceof WindowOperator);
}
 

@Test
@SuppressWarnings("rawtypes")
public void testApplyEventTimeWindows() throws Exception {
	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
	env.setStreamTimeCharacteristic(TimeCharacteristic.IngestionTime);

	DataStream<Tuple2<String, Integer>> source = env.fromElements(
			Tuple2.of("hello", 1),
			Tuple2.of("hello", 2));

	DataStream<Tuple2<String, Integer>> window1 = source
		.keyBy(0)
		.timeWindow(Time.of(1000, TimeUnit.MILLISECONDS))
		.apply(new WindowFunction<Tuple2<String, Integer>, Tuple2<String, Integer>, Tuple, TimeWindow>() {
			private static final long serialVersionUID = 1L;

			@Override
			public void apply(Tuple tuple,
				TimeWindow window,
				Iterable<Tuple2<String, Integer>> values,
				Collector<Tuple2<String, Integer>> out) throws Exception {

			}
		});

	OneInputTransformation<Tuple2<String, Integer>, Tuple2<String, Integer>> transform1 = (OneInputTransformation<Tuple2<String, Integer>, Tuple2<String, Integer>>) window1.getTransformation();
	OneInputStreamOperator<Tuple2<String, Integer>, Tuple2<String, Integer>> operator1 = transform1.getOperator();
	Assert.assertTrue(operator1 instanceof WindowOperator);
	WindowOperator winOperator1 = (WindowOperator) operator1;
	Assert.assertTrue(winOperator1.getTrigger() instanceof EventTimeTrigger);
	Assert.assertTrue(winOperator1.getWindowAssigner() instanceof TumblingEventTimeWindows);
	Assert.assertTrue(winOperator1.getStateDescriptor() instanceof ListStateDescriptor);
}
 

private static <IN, OUT, KEY> TypeInformation<OUT> getWindowFunctionReturnType(
	WindowFunction<IN, OUT, KEY, ?> function,
	TypeInformation<IN> inType) {
	return TypeExtractor.getUnaryOperatorReturnType(
		function,
		WindowFunction.class,
		0,
		1,
		new int[]{3, 0},
		inType,
		null,
		false);
}
 

/**
 * Sum - Window function, summing already happened in reduce function
 */
private static WindowFunction<Tuple3<String, String, Long>, Tuple3<String, String, Long>, Tuple, TimeWindow> sumWindowFunction() {
  return new WindowFunction<Tuple3<String, String, Long>, Tuple3<String, String, Long>, Tuple, TimeWindow>() {
    @Override
    public void apply(Tuple keyTuple, TimeWindow window, Iterable<Tuple3<String, String, Long>> values, Collector<Tuple3<String, String, Long>> out) throws Exception {
      Iterator<Tuple3<String, String, Long>> valIter = values.iterator();
      Tuple3<String, String, Long> tuple = valIter.next();
      if (valIter.hasNext()) {
        throw new IllegalStateException("Unexpected");
      }
      tuple.f1 = Long.toString(window.getEnd());
      out.collect(tuple); // collect end time here
    }
  };
}
 

private void runTest(
		SourceFunction<SessionEvent<Integer, TestEventPayload>> dataSource,
		WindowFunction<SessionEvent<Integer, TestEventPayload>,
				String, Tuple, TimeWindow> windowFunction) throws Exception {

	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
	env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);
	WindowedStream<SessionEvent<Integer, TestEventPayload>, Tuple, TimeWindow> windowedStream =
			env.addSource(dataSource).keyBy("sessionKey")
			.window(EventTimeSessionWindows.withGap(Time.milliseconds(MAX_SESSION_EVENT_GAP_MS)));

	if (ALLOWED_LATENESS_MS != Long.MAX_VALUE) {
		windowedStream = windowedStream.allowedLateness(Time.milliseconds(ALLOWED_LATENESS_MS));
	}

	if (PURGE_WINDOW_ON_FIRE) {
		windowedStream = windowedStream.trigger(PurgingTrigger.of(EventTimeTrigger.create()));
	}

	windowedStream.apply(windowFunction).print();
	JobExecutionResult result = env.execute();

	// check that overall event counts match with our expectations. remember that late events within lateness will
	// each trigger a window!
	Assert.assertEquals(
		(LATE_EVENTS_PER_SESSION + 1) * NUMBER_OF_SESSIONS * EVENTS_PER_SESSION,
		(long) result.getAccumulatorResult(SESSION_COUNTER_ON_TIME_KEY));
	Assert.assertEquals(
		NUMBER_OF_SESSIONS * (LATE_EVENTS_PER_SESSION * (LATE_EVENTS_PER_SESSION + 1) / 2),
		(long) result.getAccumulatorResult(SESSION_COUNTER_LATE_KEY));
}
 

@Test
@SuppressWarnings("rawtypes")
public void testApplyEventTimeWindows() throws Exception {
	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
	env.setStreamTimeCharacteristic(TimeCharacteristic.IngestionTime);

	DataStream<Tuple2<String, Integer>> source = env.fromElements(
			Tuple2.of("hello", 1),
			Tuple2.of("hello", 2));

	DataStream<Tuple2<String, Integer>> window1 = source
		.keyBy(0)
		.timeWindow(Time.of(1000, TimeUnit.MILLISECONDS))
		.apply(new WindowFunction<Tuple2<String, Integer>, Tuple2<String, Integer>, Tuple, TimeWindow>() {
			private static final long serialVersionUID = 1L;

			@Override
			public void apply(Tuple tuple,
				TimeWindow window,
				Iterable<Tuple2<String, Integer>> values,
				Collector<Tuple2<String, Integer>> out) throws Exception {

			}
		});

	OneInputTransformation<Tuple2<String, Integer>, Tuple2<String, Integer>> transform1 = (OneInputTransformation<Tuple2<String, Integer>, Tuple2<String, Integer>>) window1.getTransformation();
	OneInputStreamOperator<Tuple2<String, Integer>, Tuple2<String, Integer>> operator1 = transform1.getOperator();
	Assert.assertTrue(operator1 instanceof WindowOperator);
	WindowOperator winOperator1 = (WindowOperator) operator1;
	Assert.assertTrue(winOperator1.getTrigger() instanceof EventTimeTrigger);
	Assert.assertTrue(winOperator1.getWindowAssigner() instanceof TumblingEventTimeWindows);
	Assert.assertTrue(winOperator1.getStateDescriptor() instanceof ListStateDescriptor);
}
 

/**
 * Verifies that calls to timeWindow() instantiate a regular
 * windowOperator instead of an aligned one.
 */
@Test
public void testAlignedWindowDeprecation() throws Exception {
	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
	env.setStreamTimeCharacteristic(TimeCharacteristic.ProcessingTime);

	DataStream<Tuple2<String, Integer>> source = env.fromElements(Tuple2.of("hello", 1), Tuple2.of("hello", 2));

	DummyReducer reducer = new DummyReducer();

	DataStream<Tuple2<String, Integer>> window1 = source
			.keyBy(0)
			.timeWindow(Time.of(1000, TimeUnit.MILLISECONDS), Time.of(100, TimeUnit.MILLISECONDS))
			.reduce(reducer);

	OneInputTransformation<Tuple2<String, Integer>, Tuple2<String, Integer>> transform1 = (OneInputTransformation<Tuple2<String, Integer>, Tuple2<String, Integer>>) window1.getTransformation();
	OneInputStreamOperator<Tuple2<String, Integer>, Tuple2<String, Integer>> operator1 = transform1.getOperator();
	Assert.assertTrue(operator1 instanceof WindowOperator);

	DataStream<Tuple2<String, Integer>> window2 = source
			.keyBy(0)
			.timeWindow(Time.of(1000, TimeUnit.MILLISECONDS))
			.apply(new WindowFunction<Tuple2<String, Integer>, Tuple2<String, Integer>, Tuple, TimeWindow>() {
				private static final long serialVersionUID = 1L;

				@Override
				public void apply(Tuple tuple,
						TimeWindow window,
						Iterable<Tuple2<String, Integer>> values,
						Collector<Tuple2<String, Integer>> out) throws Exception {

				}
			});

	OneInputTransformation<Tuple2<String, Integer>, Tuple2<String, Integer>> transform2 = (OneInputTransformation<Tuple2<String, Integer>, Tuple2<String, Integer>>) window2.getTransformation();
	OneInputStreamOperator<Tuple2<String, Integer>, Tuple2<String, Integer>> operator2 = transform2.getOperator();
	Assert.assertTrue(operator2 instanceof WindowOperator);
}
 

private void runTest(
		SourceFunction<SessionEvent<Integer, TestEventPayload>> dataSource,
		WindowFunction<SessionEvent<Integer, TestEventPayload>,
				String, Tuple, TimeWindow> windowFunction) throws Exception {

	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
	env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);
	WindowedStream<SessionEvent<Integer, TestEventPayload>, Tuple, TimeWindow> windowedStream =
			env.addSource(dataSource).keyBy("sessionKey")
			.window(EventTimeSessionWindows.withGap(Time.milliseconds(MAX_SESSION_EVENT_GAP_MS)));

	if (ALLOWED_LATENESS_MS != Long.MAX_VALUE) {
		windowedStream = windowedStream.allowedLateness(Time.milliseconds(ALLOWED_LATENESS_MS));
	}

	if (PURGE_WINDOW_ON_FIRE) {
		windowedStream = windowedStream.trigger(PurgingTrigger.of(EventTimeTrigger.create()));
	}

	windowedStream.apply(windowFunction).print();
	JobExecutionResult result = env.execute();

	// check that overall event counts match with our expectations. remember that late events within lateness will
	// each trigger a window!
	Assert.assertEquals(
		(LATE_EVENTS_PER_SESSION + 1) * NUMBER_OF_SESSIONS * EVENTS_PER_SESSION,
		(long) result.getAccumulatorResult(SESSION_COUNTER_ON_TIME_KEY));
	Assert.assertEquals(
		NUMBER_OF_SESSIONS * (LATE_EVENTS_PER_SESSION * (LATE_EVENTS_PER_SESSION + 1) / 2),
		(long) result.getAccumulatorResult(SESSION_COUNTER_LATE_KEY));
}
 

public static void main(String[] args) throws Exception {
	final ParameterTool pt = ParameterTool.fromArgs(args);
	final boolean useValueState = pt.getBoolean("useValueState", false);
	final boolean useListState = pt.getBoolean("useListState", false);
	final boolean useMapState = pt.getBoolean("useMapState", false);

	final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();

	setupEnvironment(env, pt);
	KeyedStream<Event, Integer> keyedStream = env
		.addSource(DataStreamAllroundTestJobFactory.createEventSource(pt))
		.name(EVENT_SOURCE.getName())
		.uid(EVENT_SOURCE.getUid())
		.assignTimestampsAndWatermarks(createTimestampExtractor(pt))
		.keyBy(Event::getKey);

	keyedStream.map(new ValueStateMapper(useValueState)).name("ValueStateMapper").uid("ValueStateMapper");
	keyedStream.map(new ListStateMapper(useListState)).name("ListStateMapper").uid("ListStateMapper");
	keyedStream.map(new MapStateMapper(useMapState)).name("MapStateMapper").uid("MapStateMapper");

	boolean useWindow = pt.getBoolean("useWindow", false);
	if (useWindow) {
		applyTumblingWindows(keyedStream, pt)
			.apply(new WindowFunction<Event, Event, Integer, TimeWindow>() {
				@Override
				public void apply(Integer integer, TimeWindow window, Iterable<Event> input, Collector<Event> out) {
					for (Event e : input) {
						out.collect(e);
					}
				}
			})
			.name(TIME_WINDOW_OPER.getName())
			.uid(TIME_WINDOW_OPER.getUid());
	}

	env.execute("RocksDB test job");
}
 

private static <IN, OUT, KEY> TypeInformation<OUT> getWindowFunctionReturnType(
	WindowFunction<IN, OUT, KEY, ?> function,
	TypeInformation<IN> inType) {
	return TypeExtractor.getUnaryOperatorReturnType(
		function,
		WindowFunction.class,
		0,
		1,
		new int[]{3, 0},
		inType,
		null,
		false);
}
 

private static <IN, OUT, KEY> TypeInformation<OUT> getWindowFunctionReturnType(
	WindowFunction<IN, OUT, KEY, ?> function,
	TypeInformation<IN> inType) {
	return TypeExtractor.getUnaryOperatorReturnType(
		function,
		WindowFunction.class,
		0,
		1,
		new int[]{3, 0},
		inType,
		null,
		false);
}
 

/**
 * Verifies that calls to timeWindow() instantiate a regular
 * windowOperator instead of an aligned one.
 */
@Test
public void testAlignedWindowDeprecation() throws Exception {
	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
	env.setStreamTimeCharacteristic(TimeCharacteristic.ProcessingTime);

	DataStream<Tuple2<String, Integer>> source = env.fromElements(Tuple2.of("hello", 1), Tuple2.of("hello", 2));

	DummyReducer reducer = new DummyReducer();

	DataStream<Tuple2<String, Integer>> window1 = source
			.keyBy(0)
			.timeWindow(Time.of(1000, TimeUnit.MILLISECONDS), Time.of(100, TimeUnit.MILLISECONDS))
			.reduce(reducer);

	OneInputTransformation<Tuple2<String, Integer>, Tuple2<String, Integer>> transform1 = (OneInputTransformation<Tuple2<String, Integer>, Tuple2<String, Integer>>) window1.getTransformation();
	OneInputStreamOperator<Tuple2<String, Integer>, Tuple2<String, Integer>> operator1 = transform1.getOperator();
	Assert.assertTrue(operator1 instanceof WindowOperator);

	DataStream<Tuple2<String, Integer>> window2 = source
			.keyBy(0)
			.timeWindow(Time.of(1000, TimeUnit.MILLISECONDS))
			.apply(new WindowFunction<Tuple2<String, Integer>, Tuple2<String, Integer>, Tuple, TimeWindow>() {
				private static final long serialVersionUID = 1L;

				@Override
				public void apply(Tuple tuple,
						TimeWindow window,
						Iterable<Tuple2<String, Integer>> values,
						Collector<Tuple2<String, Integer>> out) throws Exception {

				}
			});

	OneInputTransformation<Tuple2<String, Integer>, Tuple2<String, Integer>> transform2 = (OneInputTransformation<Tuple2<String, Integer>, Tuple2<String, Integer>>) window2.getTransformation();
	OneInputStreamOperator<Tuple2<String, Integer>, Tuple2<String, Integer>> operator2 = transform2.getOperator();
	Assert.assertTrue(operator2 instanceof WindowOperator);
}
 

@Test
@SuppressWarnings("rawtypes")
public void testApplyEventTimeWindows() throws Exception {
	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
	env.setStreamTimeCharacteristic(TimeCharacteristic.IngestionTime);

	DataStream<Tuple2<String, Integer>> source = env.fromElements(
			Tuple2.of("hello", 1),
			Tuple2.of("hello", 2));

	DataStream<Tuple2<String, Integer>> window1 = source
		.keyBy(0)
		.timeWindow(Time.of(1000, TimeUnit.MILLISECONDS))
		.apply(new WindowFunction<Tuple2<String, Integer>, Tuple2<String, Integer>, Tuple, TimeWindow>() {
			private static final long serialVersionUID = 1L;

			@Override
			public void apply(Tuple tuple,
				TimeWindow window,
				Iterable<Tuple2<String, Integer>> values,
				Collector<Tuple2<String, Integer>> out) throws Exception {

			}
		});

	OneInputTransformation<Tuple2<String, Integer>, Tuple2<String, Integer>> transform1 = (OneInputTransformation<Tuple2<String, Integer>, Tuple2<String, Integer>>) window1.getTransformation();
	OneInputStreamOperator<Tuple2<String, Integer>, Tuple2<String, Integer>> operator1 = transform1.getOperator();
	Assert.assertTrue(operator1 instanceof WindowOperator);
	WindowOperator winOperator1 = (WindowOperator) operator1;
	Assert.assertTrue(winOperator1.getTrigger() instanceof EventTimeTrigger);
	Assert.assertTrue(winOperator1.getWindowAssigner() instanceof TumblingEventTimeWindows);
	Assert.assertTrue(winOperator1.getStateDescriptor() instanceof ListStateDescriptor);
}
 

@Test
public void testKeyedWindowLateArrivingEvents() throws Exception {
	TestListResultSink<String> resultSink = new TestListResultSink<>();
	TestListResultSink<Integer> lateResultSink = new TestListResultSink<>();

	StreamExecutionEnvironment see = StreamExecutionEnvironment.getExecutionEnvironment();
	see.setParallelism(3);
	see.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);

	DataStream<Integer> dataStream = see.fromCollection(elements);

	OutputTag<Integer> lateDataTag = new OutputTag<Integer>("late"){};

	SingleOutputStreamOperator<String> windowOperator = dataStream
			.assignTimestampsAndWatermarks(new TestWatermarkAssigner())
			.keyBy(new TestKeySelector())
			.timeWindow(Time.milliseconds(1), Time.milliseconds(1))
			.allowedLateness(Time.milliseconds(2))
			.sideOutputLateData(lateDataTag)
			.apply(new WindowFunction<Integer, String, Integer, TimeWindow>() {
				private static final long serialVersionUID = 1L;

				@Override
				public void apply(Integer key, TimeWindow window, Iterable<Integer> input, Collector<String> out) throws Exception {
					for (Integer val : input) {
						out.collect(String.valueOf(key) + "-" + String.valueOf(val));
					}
				}
			});

	windowOperator
			.addSink(resultSink);

	windowOperator
			.getSideOutput(lateDataTag)
			.addSink(lateResultSink);

	see.execute();
	assertEquals(Arrays.asList("1-1", "2-2", "4-4", "5-5"), resultSink.getSortedResult());
	assertEquals(Collections.singletonList(3), lateResultSink.getSortedResult());
}
 

public InternalIterableWindowFunction(WindowFunction<IN, OUT, KEY, W> wrappedFunction) {
	super(wrappedFunction);
}
 

public static void main(String[] args) throws Exception {

		final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
		env.enableCheckpointing(5000); // 要设置启动检查点
		env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);

		Properties props = new Properties();
		props.setProperty("bootstrap.servers", "kafka1:9092");
		props.setProperty("group.id", "flink-group");

		//数据源配置，是一个kafka消息的消费者
		FlinkKafkaConsumer011<String> consumer =
				new FlinkKafkaConsumer011<>("topic001", new SimpleStringSchema(), props);

		//增加时间水位设置类
		consumer.assignTimestampsAndWatermarks(new AssignerWithPunctuatedWatermarks<String> (){
			@Override
			public long extractTimestamp(String element, long previousElementTimestamp) {
				return JSONHelper.getTimeLongFromRawMessage(element);
			}

			@Nullable
			@Override
			public Watermark checkAndGetNextWatermark(String lastElement, long extractedTimestamp) {
				if (lastElement != null) {
					return new Watermark(JSONHelper.getTimeLongFromRawMessage(lastElement));
				}
				return null;
			}
		});

		env.addSource(consumer)
				//将原始消息转成Tuple2对象，保留用户名称和访问次数(每个消息访问次数为1)
				.flatMap((FlatMapFunction<String, Tuple2<String, Long>>) (s, collector) -> {
					SingleMessage singleMessage = JSONHelper.parse(s);

					if (null != singleMessage) {
						collector.collect(new Tuple2<>(singleMessage.getName(), 1L));
					}
				})
				//以用户名为key
				.keyBy(0)
				//时间窗口为2秒
				.timeWindow(Time.seconds(2))
				//将每个用户访问次数累加起来
				.apply((WindowFunction<Tuple2<String, Long>, Tuple2<String, Long>, Tuple, TimeWindow>) (tuple, window, input, out) -> {
					long sum = 0L;
					for (Tuple2<String, Long> record: input) {
						sum += record.f1;
					}

					Tuple2<String, Long> result = input.iterator().next();
					result.f1 = sum;
					out.collect(result);
				})
				//输出方式是STDOUT
				.print();

		env.execute("Flink-Kafka demo");
	}
 

public static void main(String[] args) throws Exception {
	// set up the streaming execution environment
	final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
	// env.enableCheckpointing(5000);
	env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);

	Properties properties = new Properties();
	properties.setProperty("bootstrap.servers", "localhost:9092");

	properties.setProperty("zookeeper.connect", "localhost:2181");
	properties.setProperty("group.id", "test");

	FlinkKafkaConsumer09<String> myConsumer = new FlinkKafkaConsumer09<>("temp", new SimpleStringSchema(),
			properties);
	myConsumer.assignTimestampsAndWatermarks(new CustomWatermarkEmitter());


	DataStream<Tuple2<String, Double>> keyedStream = env.addSource(myConsumer).flatMap(new Splitter()).keyBy(0)
			.timeWindow(Time.seconds(300))
			.apply(new WindowFunction<Tuple2<String, Double>, Tuple2<String, Double>, Tuple, TimeWindow>() {

				@Override
				public void apply(Tuple key, TimeWindow window, Iterable<Tuple2<String, Double>> input,
						Collector<Tuple2<String, Double>> out) throws Exception {
					double sum = 0L;
					int count = 0;
					for (Tuple2<String, Double> record : input) {
						sum += record.f1;
						count++;
					}

					Tuple2<String, Double> result = input.iterator().next();
					result.f1 = (sum/count);
					out.collect(result);

				}
			});

	keyedStream.print();

	// execute program
	env.execute("Flink Streaming Java API Skeleton");
}
 

public InternalSingleValueWindowFunction(WindowFunction<IN, OUT, KEY, W> wrappedFunction) {
	super(wrappedFunction);
}
 

@Test
public void testKeyedWindowLateArrivingEvents() throws Exception {
	TestListResultSink<String> resultSink = new TestListResultSink<>();
	TestListResultSink<Integer> lateResultSink = new TestListResultSink<>();

	StreamExecutionEnvironment see = StreamExecutionEnvironment.getExecutionEnvironment();
	see.setParallelism(3);
	see.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);

	DataStream<Integer> dataStream = see.fromCollection(elements);

	OutputTag<Integer> lateDataTag = new OutputTag<Integer>("late"){};

	SingleOutputStreamOperator<String> windowOperator = dataStream
			.assignTimestampsAndWatermarks(new TestWatermarkAssigner())
			.keyBy(new TestKeySelector())
			.timeWindow(Time.milliseconds(1), Time.milliseconds(1))
			.allowedLateness(Time.milliseconds(2))
			.sideOutputLateData(lateDataTag)
			.apply(new WindowFunction<Integer, String, Integer, TimeWindow>() {
				private static final long serialVersionUID = 1L;

				@Override
				public void apply(Integer key, TimeWindow window, Iterable<Integer> input, Collector<String> out) throws Exception {
					for (Integer val : input) {
						out.collect(String.valueOf(key) + "-" + String.valueOf(val));
					}
				}
			});

	windowOperator
			.addSink(resultSink);

	windowOperator
			.getSideOutput(lateDataTag)
			.addSink(lateResultSink);

	see.execute();
	assertEquals(Arrays.asList("1-1", "2-2", "4-4", "5-5"), resultSink.getSortedResult());
	assertEquals(Collections.singletonList(3), lateResultSink.getSortedResult());
}
 

public InternalSingleValueWindowFunction(WindowFunction<IN, OUT, KEY, W> wrappedFunction) {
	super(wrappedFunction);
}
 

public InternalIterableWindowFunction(WindowFunction<IN, OUT, KEY, W> wrappedFunction) {
	super(wrappedFunction);
}
 

public PythonApplyFunction(WindowFunction<PyObject, Object, Object, W> fun) throws IOException {
	super(fun);
}
 

@Test
public void testKeyedWindowLateArrivingEvents() throws Exception {
	TestListResultSink<String> resultSink = new TestListResultSink<>();
	TestListResultSink<Integer> lateResultSink = new TestListResultSink<>();

	StreamExecutionEnvironment see = StreamExecutionEnvironment.getExecutionEnvironment();
	see.setParallelism(3);
	see.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);

	DataStream<Integer> dataStream = see.fromCollection(elements);

	OutputTag<Integer> lateDataTag = new OutputTag<Integer>("late"){};

	SingleOutputStreamOperator<String> windowOperator = dataStream
			.assignTimestampsAndWatermarks(new TestWatermarkAssigner())
			.keyBy(new TestKeySelector())
			.timeWindow(Time.milliseconds(1), Time.milliseconds(1))
			.allowedLateness(Time.milliseconds(2))
			.sideOutputLateData(lateDataTag)
			.apply(new WindowFunction<Integer, String, Integer, TimeWindow>() {
				private static final long serialVersionUID = 1L;

				@Override
				public void apply(Integer key, TimeWindow window, Iterable<Integer> input, Collector<String> out) throws Exception {
					for (Integer val : input) {
						out.collect(String.valueOf(key) + "-" + String.valueOf(val));
					}
				}
			});

	windowOperator
			.addSink(resultSink);

	windowOperator
			.getSideOutput(lateDataTag)
			.addSink(lateResultSink);

	see.execute();
	assertEquals(Arrays.asList("1-1", "2-2", "4-4", "5-5"), resultSink.getSortedResult());
	assertEquals(Collections.singletonList(3), lateResultSink.getSortedResult());
}