Java Code Examples for org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator#print()

public static void main(String[] args) throws Exception {
    final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    ParameterTool parameterTool = ExecutionEnvUtil.PARAMETER_TOOL;
    Properties props = KafkaConfigUtil.buildKafkaProps(parameterTool);

    SingleOutputStreamOperator<MetricEvent> metricData = env.addSource(new FlinkKafkaConsumer011<>(
            parameterTool.get(METRICS_TOPIC),
            new SimpleStringSchema(),
            props)).setParallelism(1)
            .map(string -> GsonUtil.fromJson(string, MetricEvent.class));

    metricData.print();

    CheckPointUtil.setCheckpointConfig(env, parameterTool)
            .execute("zhisheng --- checkpoint config example");
}
 

@Test
public void givenStreamOfEvents_whenProcessEvents_thenShouldApplyWindowingOnTransformation() throws Exception {
    // given
    final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();

    SingleOutputStreamOperator<Tuple2<Integer, Long>> windowed = env.fromElements(new Tuple2<>(16, ZonedDateTime.now().plusMinutes(25).toInstant().getEpochSecond()), new Tuple2<>(15, ZonedDateTime.now().plusMinutes(2).toInstant().getEpochSecond()))
            .assignTimestampsAndWatermarks(new BoundedOutOfOrdernessTimestampExtractor<Tuple2<Integer, Long>>(Time.seconds(20)) {
                @Override
                public long extractTimestamp(Tuple2<Integer, Long> element) {
                    return element.f1 * 1000;
                }
            });

    SingleOutputStreamOperator<Tuple2<Integer, Long>> reduced = windowed.windowAll(TumblingEventTimeWindows.of(Time.seconds(5))).maxBy(0, true);

    reduced.print();

    // when
    env.execute();
}
 

public static void main(String[] args) throws Exception {
        final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
        env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);
        env.getConfig().setAutoWatermarkInterval(5000);
        //并行度设置为 1
        env.setParallelism(1);
//        env.setParallelism(4);

        SingleOutputStreamOperator<Word> data = env.socketTextStream("localhost", 9001)
                .map(new MapFunction<String, Word>() {
                    @Override
                    public Word map(String value) throws Exception {
                        String[] split = value.split(",");
                        return new Word(split[0], Integer.valueOf(split[1]), Long.valueOf(split[2]));
                    }
                });

        //Periodic Watermark
        data.assignTimestampsAndWatermarks(new WordPeriodicWatermark());

        data.print();
        env.execute("watermark demo");
    }
 

public static void main(String[] args) throws Exception {
        final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
        env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);
        //并行度设置为 1
        env.setParallelism(1);
//        env.setParallelism(4);

        SingleOutputStreamOperator<Word> data = env.socketTextStream("localhost", 9001)
                .map(new MapFunction<String, Word>() {
                    @Override
                    public Word map(String value) throws Exception {
                        String[] split = value.split(",");
                        return new Word(split[0], Integer.valueOf(split[1]), Long.valueOf(split[2]));
                    }
                });

        //Punctuated Watermark
        data.assignTimestampsAndWatermarks(new WordPunctuatedWatermark());

        data.print();
        env.execute("watermark demo");
    }
 

public static void main(String[] args) throws Exception {
    final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    ParameterTool parameterTool = ExecutionEnvUtil.PARAMETER_TOOL;
    Properties props = KafkaConfigUtil.buildKafkaProps(parameterTool);

    SingleOutputStreamOperator<MetricEvent> metricData = env.addSource(new FlinkKafkaConsumer011<>(
            parameterTool.get(METRICS_TOPIC),
            new SimpleStringSchema(),
            props)).setParallelism(1)
            .map(string -> GsonUtil.fromJson(string, MetricEvent.class));

    metricData.print();

    CheckPointUtil.setCheckpointConfig(env, parameterTool)
            .execute("zhisheng --- checkpoint config example");
}
 

public static void main(String[] args) throws Exception {
    //参数检查
    if (args.length != 2) {
        System.err.println("USAGE:\nSocketTextStreamWordCount <hostname> <port>");
        return;
    }

    String hostname = args[0];
    Integer port = Integer.parseInt(args[1]);


    // set up the streaming execution environment
    final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();

    //获取数据
    DataStreamSource<String> stream = env.socketTextStream(hostname, port);

    //计数
    SingleOutputStreamOperator<Tuple2<String, Integer>> sum = stream.flatMap(new LineSplitter())
            .keyBy(0)
            .sum(1);

    sum.print();

    env.execute("Java WordCount from SocketTextStream Example");
}
 

public static void main(String[] args) throws Exception {
        final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
        env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);
        env.getConfig().setAutoWatermarkInterval(5000);
        //并行度设置为 1
        env.setParallelism(1);
//        env.setParallelism(4);

        SingleOutputStreamOperator<Word> data = env.socketTextStream("localhost", 9001)
                .map(new MapFunction<String, Word>() {
                    @Override
                    public Word map(String value) throws Exception {
                        String[] split = value.split(",");
                        return new Word(split[0], Integer.valueOf(split[1]), Long.valueOf(split[2]));
                    }
                });

        //Periodic Watermark
        data.assignTimestampsAndWatermarks(new WordPeriodicWatermark());

        data.print();
        env.execute("watermark demo");
    }
 

public static void main(String[] args) throws Exception {
        final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
        env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);
        //并行度设置为 1
        env.setParallelism(1);
//        env.setParallelism(4);

        SingleOutputStreamOperator<Word> data = env.socketTextStream("localhost", 9001)
                .map(new MapFunction<String, Word>() {
                    @Override
                    public Word map(String value) throws Exception {
                        String[] split = value.split(",");
                        return new Word(split[0], Integer.valueOf(split[1]), Long.valueOf(split[2]));
                    }
                });

        //Punctuated Watermark
        data.assignTimestampsAndWatermarks(new WordPunctuatedWatermark());

        data.print();
        env.execute("watermark demo");
    }
 

public static void main(String[] args) throws Exception {
    //参数检查
    if (args.length != 2) {
        System.err.println("USAGE:\nSocketTextStreamWordCount <hostname> <port>");
        return;
    }

    String hostname = args[0];
    Integer port = Integer.parseInt(args[1]);


    // set up the streaming execution environment
    final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();

    //获取数据
    DataStreamSource<String> stream = env.socketTextStream(hostname, port);

    //计数
    SingleOutputStreamOperator<Tuple2<String, Integer>> sum = stream.flatMap(new LineSplitter())
            .keyBy(0)
            .sum(1);

    sum.print();

    env.execute("Java WordCount from SocketTextStream Example");
}
 

public static void main(String[] args) throws Exception {
        final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
        env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);
        //并行度设置为 1
        env.setParallelism(1);
//        env.setParallelism(4);

        SingleOutputStreamOperator<Word> data = env.socketTextStream("localhost", 9001)
                .map(new MapFunction<String, Word>() {
                    @Override
                    public Word map(String value) throws Exception {
                        String[] split = value.split(",");
                        return new Word(split[0], Integer.valueOf(split[1]), Long.valueOf(split[2]));
                    }
                });

        //BoundedOutOfOrdernessTimestampExtractor
        data.assignTimestampsAndWatermarks(new BoundedOutOfOrdernessTimestampExtractor<Word>(Time.seconds(10)) {
            @Override
            public long extractTimestamp(Word element) {
                return element.getTimestamp();
            }
        });

        data.print();
        env.execute("watermark demo");
    }
 

public static void main(String[] args) throws Exception {
        final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
        env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);
        //并行度设置为 1
        env.setParallelism(1);
//        env.setParallelism(4);

        OutputTag<Word> lateDataTag = new OutputTag<Word>("late") {
        };

        SingleOutputStreamOperator<Word> data = env.socketTextStream("localhost", 9001)
                .map(new MapFunction<String, Word>() {
                    @Override
                    public Word map(String value) throws Exception {
                        String[] split = value.split(",");
                        return new Word(split[0], Integer.valueOf(split[1]), Long.valueOf(split[2]));
                    }
                }).assignTimestampsAndWatermarks(new WordPeriodicWatermark());

        SingleOutputStreamOperator<Word> sum = data.keyBy(0)
                .timeWindow(Time.seconds(10))
//                .allowedLateness(Time.milliseconds(2))
                .sideOutputLateData(lateDataTag)
                .sum(1);

        sum.print();

        sum.getSideOutput(lateDataTag)
                .print();

        env.execute("watermark demo");
    }
 

public static void main(String[] args) throws Exception {
        final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
        env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);
        //并行度设置为 1
        env.setParallelism(1);
//        env.setParallelism(4);

        OutputTag<Word> lateDataTag = new OutputTag<Word>("late") {
        };

        SingleOutputStreamOperator<Word> data = env.socketTextStream("localhost", 9001)
                .map(new MapFunction<String, Word>() {
                    @Override
                    public Word map(String value) throws Exception {
                        String[] split = value.split(",");
                        return new Word(split[0], Integer.valueOf(split[1]), Long.valueOf(split[2]));
                    }
                }).assignTimestampsAndWatermarks(new WordPeriodicWatermark());

        SingleOutputStreamOperator<Word> sum = data.keyBy(0)
                .timeWindow(Time.seconds(10))
//                .allowedLateness(Time.milliseconds(2))
                .sideOutputLateData(lateDataTag)
                .sum(1);

        sum.print();

        sum.getSideOutput(lateDataTag)
                .print();

        env.execute("watermark demo");
    }
 

public static void main(String[] args) throws Exception {
        final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
        env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);
        //并行度设置为 1
        env.setParallelism(1);
//        env.setParallelism(4);

        SingleOutputStreamOperator<Word> data = env.socketTextStream("localhost", 9001)
                .map(new MapFunction<String, Word>() {
                    @Override
                    public Word map(String value) throws Exception {
                        String[] split = value.split(",");
                        return new Word(split[0], Integer.valueOf(split[1]), Long.valueOf(split[2]));
                    }
                });

        //BoundedOutOfOrdernessTimestampExtractor
        data.assignTimestampsAndWatermarks(new BoundedOutOfOrdernessTimestampExtractor<Word>(Time.seconds(10)) {
            @Override
            public long extractTimestamp(Word element) {
                return element.getTimestamp();
            }
        });

        data.print();
        env.execute("watermark demo");
    }
 

public static void main(String[] args) throws Exception {

		StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
		env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);

		OutputTag<Event> lateDataTag = new OutputTag<Event>("late"){};

		// When PARTITIONS_PER_INSTANCE is greater than 1, there can be late events.
		DataStream<Event> events = env.addSource(new ParallelEventSource(PARTITIONS_PER_INSTANCE));

		// Count the number of events per user in one second event-time windows,
		// and capture late events on a side output.
		SingleOutputStreamOperator<Tuple2<String, Integer>> windowOperator = events
				.assignTimestampsAndWatermarks(new TimestampsAndWatermarks())
				.keyBy(e -> e.userId)
				.window(TumblingEventTimeWindows.of(Time.seconds(1)))
				.sideOutputLateData(lateDataTag)
				.process(new CountEventsPerUser());

		windowOperator.print();

		// Count the number of late events for every second of processing time.pri
		windowOperator.getSideOutput(lateDataTag)
				.windowAll(TumblingProcessingTimeWindows.of(Time.seconds(1)))
				.process(new CountLateEvents())
				.map(i -> new Tuple3<String, Integer, String>("LATE", i, Instant.now().toString()))
				.returns(Types.TUPLE(Types.STRING, Types.INT, Types.STRING))
				.print();

		env.execute();
	}
 

public static void main(String[] args) throws Exception {

		StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
		env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);
		env.setParallelism(1);

		DataStream<Event> eventStream = env.addSource(new OutOfOrderEventSource())
				.assignTimestampsAndWatermarks(new TimestampsAndWatermarks());

		Pattern<Event, ?> matchEverything =
				Pattern.<Event>begin("any")
						.where(new SimpleCondition<Event>() {
							@Override
							public boolean filter(Event event) throws Exception {
								return true;
							}
						});

		PatternStream<Event> patternStream = CEP.pattern(eventStream, matchEverything);
		OutputTag<Event> lateDataOutputTag = new OutputTag<Event>("late-events"){};

		SingleOutputStreamOperator<Event> sorted = patternStream
				.sideOutputLateData(lateDataOutputTag)
				.select(new PatternSelectFunction<Event, Event>() {
					@Override
					public Event select(Map<String, List<Event>> map) throws Exception {
						return map.get("any").get(0);
					}
				});

		sorted.print();
		sorted
				.getSideOutput(lateDataOutputTag)
				.map(e -> new Tuple2<>(e, "LATE"))
				.returns(Types.TUPLE(TypeInformation.of(Event.class), Types.STRING))
				.print();

		env.execute();
	}
 

@Test
public void givenStreamOfEvents_whenProcessEvents_thenShouldPrintResultsOnSinkOperation() throws Exception {
    // given
    final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();

    DataStream<String> text = env.fromElements("This is a first sentence", "This is a second sentence with a one word");

    SingleOutputStreamOperator<String> upperCase = text.map(String::toUpperCase);

    upperCase.print();

    // when
    env.execute();
}
 

public static void main(String[] args) throws Exception {
    final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();

    // 使用事件时间
    env.setStreamTimeCharacteristic(TimeCharacteristic.ProcessingTime);

    // 并行度为1
    env.setParallelism(1);

    // 设置数据源，一共三个元素
    DataStream<Tuple2<String,Integer>> dataStream = env.addSource(new SourceFunction<Tuple2<String, Integer>>() {
        @Override
        public void run(SourceContext<Tuple2<String, Integer>> ctx) throws Exception {
            for(int i=1; i<Integer.MAX_VALUE; i++) {
                // 只有aaa和bbb两种name
                String name = 0==i%2 ? "aaa" : "bbb";

                // 使用当前时间作为时间戳
                long timeStamp = System.currentTimeMillis();

                // 将数据和时间戳打印出来，用来验证数据
                System.out.println(String.format("source，%s, %s\n",
                        name,
                        time(timeStamp)));

                // 发射一个元素，并且带上了时间戳
                ctx.collectWithTimestamp(new Tuple2<String, Integer>(name, 1), timeStamp);

                // 每发射一次就延时1秒
                Thread.sleep(1000);
            }
        }

        @Override
        public void cancel() {

        }
    });

    // 将数据用5秒的滚动窗口做划分，再用ProcessAllWindowFunction
    SingleOutputStreamOperator<String> mainDataStream = dataStream
            // 5秒一次的滚动窗口
            .timeWindowAll(Time.seconds(5))
            // 统计当前窗口内的元素数量，然后把数量、窗口起止时间整理成字符串发送给下游算子
            .process(new ProcessAllWindowFunction<Tuple2<String, Integer>, String, TimeWindow>() {
                @Override
                public void process(Context context, Iterable<Tuple2<String, Integer>> iterable, Collector<String> collector) throws Exception {
                    int count = 0;

                    // iterable可以访问当前窗口内的所有数据，
                    // 这里简单处理，只统计了元素数量
                    for (Tuple2<String, Integer> tuple2 : iterable) {
                        count++;
                    }

                    // 将当前窗口的起止时间和元素数量整理成字符串
                    String value = String.format("window, %s - %s, %d\n",
                            // 当前窗口的起始时间
                            time(context.window().getStart()),
                            // 当前窗口的结束时间
                            time(context.window().getEnd()),
                            // 当前key在当前窗口内元素总数
                            count);

                    // 发射到下游算子
                    collector.collect(value);
                }
            });

    // 打印结果，通过分析打印信息，检查ProcessWindowFunction中可以处理所有key的整个窗口的数据
    mainDataStream.print();

    env.execute("processfunction demo : processallwindowfunction");
}
 

public static void main(String[] args) throws Exception {
    final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();

    // 并行度为1
    env.setParallelism(1);

    // 定义OutputTag
    final OutputTag<String> outputTag = new OutputTag<String>("side-output"){};

    // 创建一个List，里面有两个Tuple2元素
    List<Tuple2<String, Integer>> list = new ArrayList<>();
    list.add(new Tuple2("aaa", 1));
    list.add(new Tuple2("bbb", 2));
    list.add(new Tuple2("ccc", 3));

    //通过List创建DataStream
    DataStream<Tuple2<String, Integer>> fromCollectionDataStream = env.fromCollection(list);

    //所有元素都进入mainDataStream，f1字段为奇数的元素进入SideOutput
    SingleOutputStreamOperator<String> mainDataStream = fromCollectionDataStream
            .process(new ProcessFunction<Tuple2<String, Integer>, String>() {
                @Override
                public void processElement(Tuple2<String, Integer> value, Context ctx, Collector<String> out) throws Exception {

                    //进入主流程的下一个算子
                    out.collect("main, name : " + value.f0 + ", value : " + value.f1);

                    //f1字段为奇数的元素进入SideOutput
                    if(1 == value.f1 % 2) {
                        ctx.output(outputTag, "side, name : " + value.f0 + ", value : " + value.f1);
                    }
                }
            });

    // 禁止chanin，这样可以在页面上看清楚原始的DAG
    mainDataStream.disableChaining();

    // 取得旁路数据
    DataStream<String> sideDataStream = mainDataStream.getSideOutput(outputTag);


    mainDataStream.print();
    sideDataStream.print();

    env.execute("processfunction demo : sideoutput");
}