Java Code Examples for org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator#addSink()

@Test
public void testSideOutputWithMultipleConsumersWithObjectReuse() throws Exception {
	final OutputTag<String> sideOutputTag = new OutputTag<String>("side"){};

	TestListResultSink<String> sideOutputResultSink1 = new TestListResultSink<>();
	TestListResultSink<String> sideOutputResultSink2 = new TestListResultSink<>();
	TestListResultSink<Integer> resultSink = new TestListResultSink<>();

	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
	env.getConfig().enableObjectReuse();
	env.setParallelism(3);

	DataStream<Integer> dataStream = env.fromCollection(elements);

	SingleOutputStreamOperator<Integer> passThroughtStream = dataStream
			.process(new ProcessFunction<Integer, Integer>() {
				private static final long serialVersionUID = 1L;

				@Override
				public void processElement(
						Integer value, Context ctx, Collector<Integer> out) throws Exception {
					out.collect(value);
					ctx.output(sideOutputTag, "sideout-" + String.valueOf(value));
				}
			});

	passThroughtStream.getSideOutput(sideOutputTag).addSink(sideOutputResultSink1);
	passThroughtStream.getSideOutput(sideOutputTag).addSink(sideOutputResultSink2);
	passThroughtStream.addSink(resultSink);
	env.execute();

	assertEquals(Arrays.asList("sideout-1", "sideout-2", "sideout-3", "sideout-4", "sideout-5"), sideOutputResultSink1.getSortedResult());
	assertEquals(Arrays.asList("sideout-1", "sideout-2", "sideout-3", "sideout-4", "sideout-5"), sideOutputResultSink2.getSortedResult());
	assertEquals(Arrays.asList(1, 2, 3, 4, 5), resultSink.getSortedResult());
}
 

public static void main(String[] args) throws Exception {
	final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();

	final DataStreamSource<Integer> source = env.fromElements(new TestUserClassLoaderJobLib().getValue(), 1, 2, 3, 4);
	final SingleOutputStreamOperator<Integer> mapper = source.map(element -> 2 * element);
	mapper.addSink(new DiscardingSink<>());

	ParameterTool parameterTool = ParameterTool.fromArgs(args);
	env.execute(TestUserClassLoaderJob.class.getCanonicalName() + "-" + parameterTool.getRequired("arg"));
}
 

/**
 * Test ProcessFunction side output.
 */
@Test
public void testProcessFunctionSideOutput() throws Exception {
	final OutputTag<String> sideOutputTag = new OutputTag<String>("side"){};

	TestListResultSink<String> sideOutputResultSink = new TestListResultSink<>();
	TestListResultSink<Integer> resultSink = new TestListResultSink<>();

	StreamExecutionEnvironment see = StreamExecutionEnvironment.getExecutionEnvironment();
	see.setParallelism(3);

	DataStream<Integer> dataStream = see.fromCollection(elements);

	SingleOutputStreamOperator<Integer> passThroughtStream = dataStream
			.process(new ProcessFunction<Integer, Integer>() {
				private static final long serialVersionUID = 1L;

				@Override
				public void processElement(
						Integer value, Context ctx, Collector<Integer> out) throws Exception {
					out.collect(value);
					ctx.output(sideOutputTag, "sideout-" + String.valueOf(value));
				}
			});

	passThroughtStream.getSideOutput(sideOutputTag).addSink(sideOutputResultSink);
	passThroughtStream.addSink(resultSink);
	see.execute();

	assertEquals(Arrays.asList("sideout-1", "sideout-2", "sideout-3", "sideout-4", "sideout-5"), sideOutputResultSink.getSortedResult());
	assertEquals(Arrays.asList(1, 2, 3, 4, 5), resultSink.getSortedResult());
}
 

@Test
public void testDifferentSideOutputTypes() throws Exception {
	final OutputTag<String> sideOutputTag1 = new OutputTag<String>("string"){};
	final OutputTag<Integer> sideOutputTag2 = new OutputTag<Integer>("int"){};

	TestListResultSink<String> sideOutputResultSink1 = new TestListResultSink<>();
	TestListResultSink<Integer> sideOutputResultSink2 = new TestListResultSink<>();
	TestListResultSink<Integer> resultSink = new TestListResultSink<>();

	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
	env.getConfig().enableObjectReuse();
	env.setParallelism(3);

	DataStream<Integer> dataStream = env.fromCollection(elements);

	SingleOutputStreamOperator<Integer> passThroughtStream = dataStream
			.process(new ProcessFunction<Integer, Integer>() {
				private static final long serialVersionUID = 1L;

				@Override
				public void processElement(
						Integer value, Context ctx, Collector<Integer> out) throws Exception {
					out.collect(value);
					ctx.output(sideOutputTag1, "sideout-" + String.valueOf(value));
					ctx.output(sideOutputTag2, 13);
				}
			});

	passThroughtStream.getSideOutput(sideOutputTag1).addSink(sideOutputResultSink1);
	passThroughtStream.getSideOutput(sideOutputTag2).addSink(sideOutputResultSink2);
	passThroughtStream.addSink(resultSink);
	env.execute();

	assertEquals(Arrays.asList("sideout-1", "sideout-2", "sideout-3", "sideout-4", "sideout-5"), sideOutputResultSink1.getSortedResult());
	assertEquals(Arrays.asList(13, 13, 13, 13, 13), sideOutputResultSink2.getSortedResult());
	assertEquals(Arrays.asList(1, 2, 3, 4, 5), resultSink.getSortedResult());
}
 

public static void main(String[] args) throws Exception {
        final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
        ParameterTool parameterTool = ExecutionEnvUtil.PARAMETER_TOOL;
        Properties props = KafkaConfigUtil.buildKafkaProps(parameterTool);

        SingleOutputStreamOperator<Tuple2<String, String>> product = env.addSource(new FlinkKafkaConsumer011<>(
                parameterTool.get(METRICS_TOPIC),   //这个 kafka topic 需要和上面的工具类的 topic 一致
                new SimpleStringSchema(),
                props))
                .map(string -> GsonUtil.fromJson(string, ProductEvent.class)) //反序列化 JSON
                .flatMap(new FlatMapFunction<ProductEvent, Tuple2<String, String>>() {
                    @Override
                    public void flatMap(ProductEvent value, Collector<Tuple2<String, String>> out) throws Exception {
                        //收集商品 id 和 price 两个属性
                        out.collect(new Tuple2<>(value.getId().toString(), value.getPrice().toString()));
                    }
                });
//        product.print();

        //单个 Redis
        FlinkJedisPoolConfig conf = new FlinkJedisPoolConfig.Builder().setHost(parameterTool.get("redis.host")).build();
        product.addSink(new RedisSink<Tuple2<String, String>>(conf, new RedisSinkMapper()));

        //Redis 的 ip 信息一般都从配置文件取出来
        //Redis 集群
/*        FlinkJedisClusterConfig clusterConfig = new FlinkJedisClusterConfig.Builder()
                .setNodes(new HashSet<InetSocketAddress>(
                        Arrays.asList(new InetSocketAddress("redis1", 6379)))).build();*/

        //Redis Sentinels
/*        FlinkJedisSentinelConfig sentinelConfig = new FlinkJedisSentinelConfig.Builder()
                .setMasterName("master")
                .setSentinels(new HashSet<>(Arrays.asList("sentinel1", "sentinel2")))
                .setPassword("")
                .setDatabase(1).build();*/

        env.execute("flink redis connector");
    }
 

public static void main(String[] args) throws Exception {
	final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();

	final DataStreamSource<Integer> source = env.fromElements(1, 2, 3, 4);
	final SingleOutputStreamOperator<Integer> mapper = source.map(element -> 2 * element);
	mapper.addSink(new DiscardingSink<>());

	ParameterTool parameterTool = ParameterTool.fromArgs(args);
	env.execute(TestJob.class.getCanonicalName() + "-" + parameterTool.getRequired("arg"));
}
 

public static void main(String[] args) throws Exception{
    final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    ParameterTool parameterTool = ExecutionEnvUtil.PARAMETER_TOOL;
    Properties props = KafkaConfigUtil.buildKafkaProps(parameterTool);

    SingleOutputStreamOperator<Student> student = env.addSource(new FlinkKafkaConsumer011<>(
            parameterTool.get(METRICS_TOPIC),   //这个 kafka topic 需要和上面的工具类的 topic 一致
            new SimpleStringSchema(),
            props)).setParallelism(1)
            .map(string -> GsonUtil.fromJson(string, Student.class)); //博客里面用的是 fastjson，这里用的是gson解析，解析字符串成 student 对象

    student.addSink(new SinkToMySQL()); //数据 sink 到 mysql

    env.execute("Flink data sink");
}
 

public static void main(String[] args) throws Exception{
    final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    ParameterTool parameterTool = ExecutionEnvUtil.PARAMETER_TOOL;
    Properties props = KafkaConfigUtil.buildKafkaProps(parameterTool);

    SingleOutputStreamOperator<Student> student = env.addSource(new FlinkKafkaConsumer011<>(
            parameterTool.get(METRICS_TOPIC),   //这个 kafka topic 需要和上面的工具类的 topic 一致
            new SimpleStringSchema(),
            props)).setParallelism(1)
            .map(string -> GsonUtil.fromJson(string, Student.class)); //博客里面用的是 fastjson，这里用的是gson解析，解析字符串成 student 对象

    student.addSink(new SinkToMySQL()); //数据 sink 到 mysql

    env.execute("Flink data sink");
}
 

@Test
public void testSideOutputWithMultipleConsumers() throws Exception {
	final OutputTag<String> sideOutputTag = new OutputTag<String>("side"){};

	TestListResultSink<String> sideOutputResultSink1 = new TestListResultSink<>();
	TestListResultSink<String> sideOutputResultSink2 = new TestListResultSink<>();
	TestListResultSink<Integer> resultSink = new TestListResultSink<>();

	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
	env.setParallelism(3);

	DataStream<Integer> dataStream = env.fromCollection(elements);

	SingleOutputStreamOperator<Integer> passThroughtStream = dataStream
			.process(new ProcessFunction<Integer, Integer>() {
				private static final long serialVersionUID = 1L;

				@Override
				public void processElement(
						Integer value, Context ctx, Collector<Integer> out) throws Exception {
					out.collect(value);
					ctx.output(sideOutputTag, "sideout-" + String.valueOf(value));
				}
			});

	passThroughtStream.getSideOutput(sideOutputTag).addSink(sideOutputResultSink1);
	passThroughtStream.getSideOutput(sideOutputTag).addSink(sideOutputResultSink2);
	passThroughtStream.addSink(resultSink);
	env.execute();

	assertEquals(Arrays.asList("sideout-1", "sideout-2", "sideout-3", "sideout-4", "sideout-5"), sideOutputResultSink1.getSortedResult());
	assertEquals(Arrays.asList("sideout-1", "sideout-2", "sideout-3", "sideout-4", "sideout-5"), sideOutputResultSink2.getSortedResult());
	assertEquals(Arrays.asList(1, 2, 3, 4, 5), resultSink.getSortedResult());
}
 

@Test
public void testSideOutputWithMultipleConsumersWithObjectReuse() throws Exception {
	final OutputTag<String> sideOutputTag = new OutputTag<String>("side"){};

	TestListResultSink<String> sideOutputResultSink1 = new TestListResultSink<>();
	TestListResultSink<String> sideOutputResultSink2 = new TestListResultSink<>();
	TestListResultSink<Integer> resultSink = new TestListResultSink<>();

	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
	env.getConfig().enableObjectReuse();
	env.setParallelism(3);

	DataStream<Integer> dataStream = env.fromCollection(elements);

	SingleOutputStreamOperator<Integer> passThroughtStream = dataStream
			.process(new ProcessFunction<Integer, Integer>() {
				private static final long serialVersionUID = 1L;

				@Override
				public void processElement(
						Integer value, Context ctx, Collector<Integer> out) throws Exception {
					out.collect(value);
					ctx.output(sideOutputTag, "sideout-" + String.valueOf(value));
				}
			});

	passThroughtStream.getSideOutput(sideOutputTag).addSink(sideOutputResultSink1);
	passThroughtStream.getSideOutput(sideOutputTag).addSink(sideOutputResultSink2);
	passThroughtStream.addSink(resultSink);
	env.execute();

	assertEquals(Arrays.asList("sideout-1", "sideout-2", "sideout-3", "sideout-4", "sideout-5"), sideOutputResultSink1.getSortedResult());
	assertEquals(Arrays.asList("sideout-1", "sideout-2", "sideout-3", "sideout-4", "sideout-5"), sideOutputResultSink2.getSortedResult());
	assertEquals(Arrays.asList(1, 2, 3, 4, 5), resultSink.getSortedResult());
}
 

@Test
public void testDifferentSideOutputTypes() throws Exception {
	final OutputTag<String> sideOutputTag1 = new OutputTag<String>("string"){};
	final OutputTag<Integer> sideOutputTag2 = new OutputTag<Integer>("int"){};

	TestListResultSink<String> sideOutputResultSink1 = new TestListResultSink<>();
	TestListResultSink<Integer> sideOutputResultSink2 = new TestListResultSink<>();
	TestListResultSink<Integer> resultSink = new TestListResultSink<>();

	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
	env.getConfig().enableObjectReuse();
	env.setParallelism(3);

	DataStream<Integer> dataStream = env.fromCollection(elements);

	SingleOutputStreamOperator<Integer> passThroughtStream = dataStream
			.process(new ProcessFunction<Integer, Integer>() {
				private static final long serialVersionUID = 1L;

				@Override
				public void processElement(
						Integer value, Context ctx, Collector<Integer> out) throws Exception {
					out.collect(value);
					ctx.output(sideOutputTag1, "sideout-" + String.valueOf(value));
					ctx.output(sideOutputTag2, 13);
				}
			});

	passThroughtStream.getSideOutput(sideOutputTag1).addSink(sideOutputResultSink1);
	passThroughtStream.getSideOutput(sideOutputTag2).addSink(sideOutputResultSink2);
	passThroughtStream.addSink(resultSink);
	env.execute();

	assertEquals(Arrays.asList("sideout-1", "sideout-2", "sideout-3", "sideout-4", "sideout-5"), sideOutputResultSink1.getSortedResult());
	assertEquals(Arrays.asList(13, 13, 13, 13, 13), sideOutputResultSink2.getSortedResult());
	assertEquals(Arrays.asList(1, 2, 3, 4, 5), resultSink.getSortedResult());
}
 

@Test
public void testSideOutputWithMultipleConsumers() throws Exception {
	final OutputTag<String> sideOutputTag = new OutputTag<String>("side"){};

	TestListResultSink<String> sideOutputResultSink1 = new TestListResultSink<>();
	TestListResultSink<String> sideOutputResultSink2 = new TestListResultSink<>();
	TestListResultSink<Integer> resultSink = new TestListResultSink<>();

	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
	env.setParallelism(3);

	DataStream<Integer> dataStream = env.fromCollection(elements);

	SingleOutputStreamOperator<Integer> passThroughtStream = dataStream
			.process(new ProcessFunction<Integer, Integer>() {
				private static final long serialVersionUID = 1L;

				@Override
				public void processElement(
						Integer value, Context ctx, Collector<Integer> out) throws Exception {
					out.collect(value);
					ctx.output(sideOutputTag, "sideout-" + String.valueOf(value));
				}
			});

	passThroughtStream.getSideOutput(sideOutputTag).addSink(sideOutputResultSink1);
	passThroughtStream.getSideOutput(sideOutputTag).addSink(sideOutputResultSink2);
	passThroughtStream.addSink(resultSink);
	env.execute();

	assertEquals(Arrays.asList("sideout-1", "sideout-2", "sideout-3", "sideout-4", "sideout-5"), sideOutputResultSink1.getSortedResult());
	assertEquals(Arrays.asList("sideout-1", "sideout-2", "sideout-3", "sideout-4", "sideout-5"), sideOutputResultSink2.getSortedResult());
	assertEquals(Arrays.asList(1, 2, 3, 4, 5), resultSink.getSortedResult());
}
 

/**
 * This tests whether virtual Transformations behave correctly.
 *
 * <p>Checks whether output selector, partitioning works correctly when applied on a union.
 */
@Test
public void testVirtualTransformations2() throws Exception {

	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();

	DataStream<Integer> source = env.fromElements(1, 10);

	DataStream<Integer> rebalanceMap = source.rebalance().map(new NoOpIntMap());

	DataStream<Integer> map1 = rebalanceMap
			.map(new NoOpIntMap());

	DataStream<Integer> map2 = rebalanceMap
			.map(new NoOpIntMap());

	DataStream<Integer> map3 = rebalanceMap
			.map(new NoOpIntMap());

	EvenOddOutputSelector selector = new EvenOddOutputSelector();

	SingleOutputStreamOperator<Integer> unionedMap = map1.union(map2).union(map3)
			.broadcast()
			.split(selector)
			.select("foo")
			.map(new NoOpIntMap());

	unionedMap.addSink(new DiscardingSink<>());

	StreamGraph graph = env.getStreamGraph();

	// verify that the properties are correctly set on all input operators
	assertTrue(graph.getStreamNode(map1.getId()).getOutEdges().get(0).getPartitioner() instanceof BroadcastPartitioner);
	assertTrue(graph.getStreamNode(map1.getId()).getOutEdges().get(0).getSelectedNames().get(0).equals("foo"));
	assertTrue(graph.getStreamNode(map1.getId()).getOutputSelectors().contains(selector));

	assertTrue(graph.getStreamNode(map2.getId()).getOutEdges().get(0).getPartitioner() instanceof BroadcastPartitioner);
	assertTrue(graph.getStreamNode(map2.getId()).getOutEdges().get(0).getSelectedNames().get(0).equals("foo"));
	assertTrue(graph.getStreamNode(map2.getId()).getOutputSelectors().contains(selector));

	assertTrue(graph.getStreamNode(map3.getId()).getOutEdges().get(0).getPartitioner() instanceof BroadcastPartitioner);
	assertTrue(graph.getStreamNode(map3.getId()).getOutEdges().get(0).getSelectedNames().get(0).equals("foo"));
	assertTrue(graph.getStreamNode(map3.getId()).getOutputSelectors().contains(selector));

}
 

/**
 * This tests whether virtual Transformations behave correctly.
 *
 * <p>Verifies that partitioning, output selector, selected names are correctly set in the
 * StreamGraph when they are intermixed.
 */
@Test
public void testVirtualTransformations() throws Exception {

	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();

	DataStream<Integer> source = env.fromElements(1, 10);

	DataStream<Integer> rebalanceMap = source.rebalance().map(new NoOpIntMap());

	// verify that only the partitioning that was set last is used
	DataStream<Integer> broadcastMap = rebalanceMap
			.forward()
			.global()
			.broadcast()
			.map(new NoOpIntMap());

	broadcastMap.addSink(new DiscardingSink<>());

	// verify that partitioning is preserved across union and split/select
	EvenOddOutputSelector selector1 = new EvenOddOutputSelector();
	EvenOddOutputSelector selector2 = new EvenOddOutputSelector();
	EvenOddOutputSelector selector3 = new EvenOddOutputSelector();

	DataStream<Integer> map1Operator = rebalanceMap
			.map(new NoOpIntMap());

	DataStream<Integer> map1 = map1Operator
			.broadcast()
			.split(selector1)
			.select("even");

	DataStream<Integer> map2Operator = rebalanceMap
			.map(new NoOpIntMap());

	DataStream<Integer> map2 = map2Operator
			.split(selector2)
			.select("odd")
			.global();

	DataStream<Integer> map3Operator = rebalanceMap
			.map(new NoOpIntMap());

	DataStream<Integer> map3 = map3Operator
			.global()
			.split(selector3)
			.select("even")
			.shuffle();

	SingleOutputStreamOperator<Integer> unionedMap = map1.union(map2).union(map3)
			.map(new NoOpIntMap());

	unionedMap.addSink(new DiscardingSink<>());

	StreamGraph graph = env.getStreamGraph();

	// rebalanceMap
	assertTrue(graph.getStreamNode(rebalanceMap.getId()).getInEdges().get(0).getPartitioner() instanceof RebalancePartitioner);

	// verify that only last partitioning takes precedence
	assertTrue(graph.getStreamNode(broadcastMap.getId()).getInEdges().get(0).getPartitioner() instanceof BroadcastPartitioner);
	assertEquals(rebalanceMap.getId(), graph.getSourceVertex(graph.getStreamNode(broadcastMap.getId()).getInEdges().get(0)).getId());

	// verify that partitioning in unions is preserved and that it works across split/select
	assertTrue(graph.getStreamNode(map1Operator.getId()).getOutEdges().get(0).getPartitioner() instanceof BroadcastPartitioner);
	assertTrue(graph.getStreamNode(map1Operator.getId()).getOutEdges().get(0).getSelectedNames().get(0).equals("even"));
	assertTrue(graph.getStreamNode(map1Operator.getId()).getOutputSelectors().contains(selector1));

	assertTrue(graph.getStreamNode(map2Operator.getId()).getOutEdges().get(0).getPartitioner() instanceof GlobalPartitioner);
	assertTrue(graph.getStreamNode(map2Operator.getId()).getOutEdges().get(0).getSelectedNames().get(0).equals("odd"));
	assertTrue(graph.getStreamNode(map2Operator.getId()).getOutputSelectors().contains(selector2));

	assertTrue(graph.getStreamNode(map3Operator.getId()).getOutEdges().get(0).getPartitioner() instanceof ShufflePartitioner);
	assertTrue(graph.getStreamNode(map3Operator.getId()).getOutEdges().get(0).getSelectedNames().get(0).equals("even"));
	assertTrue(graph.getStreamNode(map3Operator.getId()).getOutputSelectors().contains(selector3));
}
 

public static void main(String[] args) throws Exception {
	// parse input arguments
	final ParameterTool parameterTool = ParameterTool.fromArgs(args);

	if (parameterTool.getNumberOfParameters() < 6) {
		System.out.println("Missing parameters!\n" +
			"Usage: Kafka --input-topic <topic> --output-topic <topic> " +
			"--bootstrap.servers <kafka brokers> " +
			"--zookeeper.connect <zk quorum> " +
			"--schema-registry-url <confluent schema registry> --group.id <some id>");
		return;
	}
	Properties config = new Properties();
	config.setProperty("bootstrap.servers", parameterTool.getRequired("bootstrap.servers"));
	config.setProperty("group.id", parameterTool.getRequired("group.id"));
	config.setProperty("zookeeper.connect", parameterTool.getRequired("zookeeper.connect"));
	String schemaRegistryUrl = parameterTool.getRequired("schema-registry-url");

	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
	env.getConfig().disableSysoutLogging();

	DataStreamSource<User> input = env
		.addSource(
			new FlinkKafkaConsumer010<>(
				parameterTool.getRequired("input-topic"),
				ConfluentRegistryAvroDeserializationSchema.forSpecific(User.class, schemaRegistryUrl),
				config).setStartFromEarliest());

	SingleOutputStreamOperator<String> mapToString = input
		.map((MapFunction<User, String>) SpecificRecordBase::toString);

	FlinkKafkaProducer010<String> stringFlinkKafkaProducer010 = new FlinkKafkaProducer010<>(
		parameterTool.getRequired("output-topic"),
		new SimpleStringSchema(),
		config);

	mapToString.addSink(stringFlinkKafkaProducer010);
	env.execute("Kafka 0.10 Confluent Schema Registry AVRO Example");
}
 

@Test
public void testKeyedWindowLateArrivingEvents() throws Exception {
	TestListResultSink<String> resultSink = new TestListResultSink<>();
	TestListResultSink<Integer> lateResultSink = new TestListResultSink<>();

	StreamExecutionEnvironment see = StreamExecutionEnvironment.getExecutionEnvironment();
	see.setParallelism(3);
	see.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);

	DataStream<Integer> dataStream = see.fromCollection(elements);

	OutputTag<Integer> lateDataTag = new OutputTag<Integer>("late"){};

	SingleOutputStreamOperator<String> windowOperator = dataStream
			.assignTimestampsAndWatermarks(new TestWatermarkAssigner())
			.keyBy(new TestKeySelector())
			.timeWindow(Time.milliseconds(1), Time.milliseconds(1))
			.allowedLateness(Time.milliseconds(2))
			.sideOutputLateData(lateDataTag)
			.apply(new WindowFunction<Integer, String, Integer, TimeWindow>() {
				private static final long serialVersionUID = 1L;

				@Override
				public void apply(Integer key, TimeWindow window, Iterable<Integer> input, Collector<String> out) throws Exception {
					for (Integer val : input) {
						out.collect(String.valueOf(key) + "-" + String.valueOf(val));
					}
				}
			});

	windowOperator
			.addSink(resultSink);

	windowOperator
			.getSideOutput(lateDataTag)
			.addSink(lateResultSink);

	see.execute();
	assertEquals(Arrays.asList("1-1", "2-2", "4-4", "5-5"), resultSink.getSortedResult());
	assertEquals(Collections.singletonList(3), lateResultSink.getSortedResult());
}
 

@Test
public void testKeyedWindowLateArrivingEvents() throws Exception {
	TestListResultSink<String> resultSink = new TestListResultSink<>();
	TestListResultSink<Integer> lateResultSink = new TestListResultSink<>();

	StreamExecutionEnvironment see = StreamExecutionEnvironment.getExecutionEnvironment();
	see.setParallelism(3);
	see.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);

	DataStream<Integer> dataStream = see.fromCollection(elements);

	OutputTag<Integer> lateDataTag = new OutputTag<Integer>("late"){};

	SingleOutputStreamOperator<String> windowOperator = dataStream
			.assignTimestampsAndWatermarks(new TestWatermarkAssigner())
			.keyBy(new TestKeySelector())
			.timeWindow(Time.milliseconds(1), Time.milliseconds(1))
			.allowedLateness(Time.milliseconds(2))
			.sideOutputLateData(lateDataTag)
			.apply(new WindowFunction<Integer, String, Integer, TimeWindow>() {
				private static final long serialVersionUID = 1L;

				@Override
				public void apply(Integer key, TimeWindow window, Iterable<Integer> input, Collector<String> out) throws Exception {
					for (Integer val : input) {
						out.collect(String.valueOf(key) + "-" + String.valueOf(val));
					}
				}
			});

	windowOperator
			.addSink(resultSink);

	windowOperator
			.getSideOutput(lateDataTag)
			.addSink(lateResultSink);

	see.execute();
	assertEquals(Arrays.asList("1-1", "2-2", "4-4", "5-5"), resultSink.getSortedResult());
	assertEquals(Collections.singletonList(3), lateResultSink.getSortedResult());
}
 

/**
 * This tests whether virtual Transformations behave correctly.
 *
 * <p>Checks whether output selector, partitioning works correctly when applied on a union.
 */
@Test
public void testVirtualTransformations2() throws Exception {

	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();

	DataStream<Integer> source = env.fromElements(1, 10);

	DataStream<Integer> rebalanceMap = source.rebalance().map(new NoOpIntMap());

	DataStream<Integer> map1 = rebalanceMap
			.map(new NoOpIntMap());

	DataStream<Integer> map2 = rebalanceMap
			.map(new NoOpIntMap());

	DataStream<Integer> map3 = rebalanceMap
			.map(new NoOpIntMap());

	EvenOddOutputSelector selector = new EvenOddOutputSelector();

	SingleOutputStreamOperator<Integer> unionedMap = map1.union(map2).union(map3)
			.broadcast()
			.split(selector)
			.select("foo")
			.map(new NoOpIntMap());

	unionedMap.addSink(new DiscardingSink<>());

	StreamGraph graph = env.getStreamGraph();

	// verify that the properties are correctly set on all input operators
	assertTrue(graph.getStreamNode(map1.getId()).getOutEdges().get(0).getPartitioner() instanceof BroadcastPartitioner);
	assertTrue(graph.getStreamNode(map1.getId()).getOutEdges().get(0).getSelectedNames().get(0).equals("foo"));
	assertTrue(graph.getStreamNode(map1.getId()).getOutputSelectors().contains(selector));

	assertTrue(graph.getStreamNode(map2.getId()).getOutEdges().get(0).getPartitioner() instanceof BroadcastPartitioner);
	assertTrue(graph.getStreamNode(map2.getId()).getOutEdges().get(0).getSelectedNames().get(0).equals("foo"));
	assertTrue(graph.getStreamNode(map2.getId()).getOutputSelectors().contains(selector));

	assertTrue(graph.getStreamNode(map3.getId()).getOutEdges().get(0).getPartitioner() instanceof BroadcastPartitioner);
	assertTrue(graph.getStreamNode(map3.getId()).getOutEdges().get(0).getSelectedNames().get(0).equals("foo"));
	assertTrue(graph.getStreamNode(map3.getId()).getOutputSelectors().contains(selector));

}
 

/**
 * This tests whether virtual Transformations behave correctly.
 *
 * <p>Checks whether output selector, partitioning works correctly when applied on a union.
 */
@Test
public void testVirtualTransformations2() throws Exception {

	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();

	DataStream<Integer> source = env.fromElements(1, 10);

	DataStream<Integer> rebalanceMap = source.rebalance().map(new NoOpIntMap());

	DataStream<Integer> map1 = rebalanceMap
			.map(new NoOpIntMap());

	DataStream<Integer> map2 = rebalanceMap
			.map(new NoOpIntMap());

	DataStream<Integer> map3 = rebalanceMap
			.map(new NoOpIntMap());

	EvenOddOutputSelector selector = new EvenOddOutputSelector();

	SingleOutputStreamOperator<Integer> unionedMap = map1.union(map2).union(map3)
			.broadcast()
			.split(selector)
			.select("foo")
			.map(new NoOpIntMap());

	unionedMap.addSink(new DiscardingSink<>());

	StreamGraph graph = env.getStreamGraph();

	// verify that the properties are correctly set on all input operators
	assertTrue(graph.getStreamNode(map1.getId()).getOutEdges().get(0).getPartitioner() instanceof BroadcastPartitioner);
	assertTrue(graph.getStreamNode(map1.getId()).getOutEdges().get(0).getSelectedNames().get(0).equals("foo"));
	assertTrue(graph.getStreamNode(map1.getId()).getOutputSelectors().contains(selector));

	assertTrue(graph.getStreamNode(map2.getId()).getOutEdges().get(0).getPartitioner() instanceof BroadcastPartitioner);
	assertTrue(graph.getStreamNode(map2.getId()).getOutEdges().get(0).getSelectedNames().get(0).equals("foo"));
	assertTrue(graph.getStreamNode(map2.getId()).getOutputSelectors().contains(selector));

	assertTrue(graph.getStreamNode(map3.getId()).getOutEdges().get(0).getPartitioner() instanceof BroadcastPartitioner);
	assertTrue(graph.getStreamNode(map3.getId()).getOutEdges().get(0).getSelectedNames().get(0).equals("foo"));
	assertTrue(graph.getStreamNode(map3.getId()).getOutputSelectors().contains(selector));

}
 

public static void main(String[] args) throws Exception {
	// parse input arguments
	final ParameterTool parameterTool = ParameterTool.fromArgs(args);

	if (parameterTool.getNumberOfParameters() < 6) {
		System.out.println("Missing parameters!\n" +
			"Usage: Kafka --input-topic <topic> --output-string-topic <topic> --output-avro-topic <topic> " +
			"--bootstrap.servers <kafka brokers> " +
			"--schema-registry-url <confluent schema registry> --group.id <some id>");
		return;
	}
	Properties config = new Properties();
	config.setProperty("bootstrap.servers", parameterTool.getRequired("bootstrap.servers"));
	config.setProperty("group.id", parameterTool.getRequired("group.id"));
	String schemaRegistryUrl = parameterTool.getRequired("schema-registry-url");

	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();

	DataStreamSource<User> input = env
		.addSource(
			new FlinkKafkaConsumer010<>(
				parameterTool.getRequired("input-topic"),
				ConfluentRegistryAvroDeserializationSchema.forSpecific(User.class, schemaRegistryUrl),
				config).setStartFromEarliest());

	SingleOutputStreamOperator<String> mapToString = input
		.map((MapFunction<User, String>) SpecificRecordBase::toString);

	FlinkKafkaProducer010<String> stringFlinkKafkaProducer010 = new FlinkKafkaProducer010<>(
		parameterTool.getRequired("output-string-topic"),
		new SimpleStringSchema(),
		config);
	mapToString.addSink(stringFlinkKafkaProducer010);

	FlinkKafkaProducer010<User> avroFlinkKafkaProducer010 = new FlinkKafkaProducer010<>(
			parameterTool.getRequired("output-avro-topic"),
			ConfluentRegistryAvroSerializationSchema.forSpecific(User.class, parameterTool.getRequired("output-subject"), schemaRegistryUrl),
			config);
	input.addSink(avroFlinkKafkaProducer010);

	env.execute("Kafka 0.10 Confluent Schema Registry AVRO Example");
}