org.apache.flink.streaming.api.functions.ProcessFunction Java Examples

/**
 * Applies the given {@link ProcessFunction} on the input stream, thereby
 * creating a transformed output stream.
 *
 * <p>The function will be called for every element in the input streams and can produce zero
 * or more output elements.
 *
 * @param processFunction The {@link ProcessFunction} that is called for each element
 *                      in the stream.
 *
 * @param <R> The type of elements emitted by the {@code ProcessFunction}.
 *
 * @return The transformed {@link DataStream}.
 */
@PublicEvolving
public <R> SingleOutputStreamOperator<R> process(ProcessFunction<T, R> processFunction) {

	TypeInformation<R> outType = TypeExtractor.getUnaryOperatorReturnType(
		processFunction,
		ProcessFunction.class,
		0,
		1,
		TypeExtractor.NO_INDEX,
		getType(),
		Utils.getCallLocationName(),
		true);

	return process(processFunction, outType);
}
 

/**
 * Returns an initialized test harness for {@link ProcessFunction}.
 *
 * @param function instance of a {@link ProcessFunction} under test
 * @param <IN> type of input stream elements
 * @param <OUT> type of output stream elements
 * @return {@link OneInputStreamOperatorTestHarness} wrapped around {@code function}
 */
public static <IN, OUT>
OneInputStreamOperatorTestHarness<IN, OUT> forProcessFunction(
	final ProcessFunction<IN, OUT> function)
	throws Exception {

	OneInputStreamOperatorTestHarness<IN, OUT> testHarness =
		new OneInputStreamOperatorTestHarness<>(
			new ProcessOperator<>(
				Preconditions.checkNotNull(function)),
			1,
			1,
			0);
	testHarness.setup();
	testHarness.open();
	return testHarness;
}
 

@Test
public void testHarnessForProcessFunction() throws Exception {
	ProcessFunction<Integer, Integer> function = new ProcessFunction<Integer, Integer> () {

		@Override
		public void processElement(
			Integer value, Context ctx, Collector<Integer> out) throws Exception {
			out.collect(value);
		}
	};
	OneInputStreamOperatorTestHarness<Integer, Integer> harness = ProcessFunctionTestHarnesses
		.forProcessFunction(function);

	harness.processElement(1, 10);

	assertEquals(harness.extractOutputValues(), Collections.singletonList(1));
}
 

/**
 * Applies the given {@link ProcessFunction} on the input stream, thereby creating a transformed output stream.
 *
 * <p>The function will be called for every element in the input streams and can produce zero
 * or more output elements. Contrary to the {@link DataStream#flatMap(FlatMapFunction)}
 * function, this function can also query the time and set timers. When reacting to the firing
 * of set timers the function can directly emit elements and/or register yet more timers.
 *
 * @param processFunction The {@link ProcessFunction} that is called for each element
 *                      in the stream.
 *
 * @param <R> The type of elements emitted by the {@code ProcessFunction}.
 *
 * @return The transformed {@link DataStream}.
 *
 * @deprecated Use {@link KeyedStream#process(KeyedProcessFunction)}
 */
@Deprecated
@Override
@PublicEvolving
public <R> SingleOutputStreamOperator<R> process(ProcessFunction<T, R> processFunction) {

	TypeInformation<R> outType = TypeExtractor.getUnaryOperatorReturnType(
		processFunction,
		ProcessFunction.class,
		0,
		1,
		TypeExtractor.NO_INDEX,
		getType(),
		Utils.getCallLocationName(),
		true);

	return process(processFunction, outType);
}
 

/**
 * Applies the given {@link ProcessFunction} on the input stream, thereby
 * creating a transformed output stream.
 *
 * <p>The function will be called for every element in the input streams and can produce zero
 * or more output elements.
 *
 * @param processFunction The {@link ProcessFunction} that is called for each element
 *                      in the stream.
 *
 * @param <R> The type of elements emitted by the {@code ProcessFunction}.
 *
 * @return The transformed {@link DataStream}.
 */
@PublicEvolving
public <R> SingleOutputStreamOperator<R> process(ProcessFunction<T, R> processFunction) {

	TypeInformation<R> outType = TypeExtractor.getUnaryOperatorReturnType(
		processFunction,
		ProcessFunction.class,
		0,
		1,
		TypeExtractor.NO_INDEX,
		getType(),
		Utils.getCallLocationName(),
		true);

	return process(processFunction, outType);
}
 

/**
 * Applies the given {@link ProcessFunction} on the input stream, thereby
 * creating a transformed output stream.
 *
 * <p>The function will be called for every element in the input streams and can produce zero
 * or more output elements.
 *
 * @param processFunction The {@link ProcessFunction} that is called for each element
 *                      in the stream.
 *
 * @param <R> The type of elements emitted by the {@code ProcessFunction}.
 *
 * @return The transformed {@link DataStream}.
 */
@PublicEvolving
public <R> SingleOutputStreamOperator<R> process(ProcessFunction<T, R> processFunction) {

	TypeInformation<R> outType = TypeExtractor.getUnaryOperatorReturnType(
		processFunction,
		ProcessFunction.class,
		0,
		1,
		TypeExtractor.NO_INDEX,
		getType(),
		Utils.getCallLocationName(),
		true);

	return process(processFunction, outType);
}
 

/**
 * Applies the given {@link ProcessFunction} on the input stream, thereby creating a transformed output stream.
 *
 * <p>The function will be called for every element in the input streams and can produce zero
 * or more output elements. Contrary to the {@link DataStream#flatMap(FlatMapFunction)}
 * function, this function can also query the time and set timers. When reacting to the firing
 * of set timers the function can directly emit elements and/or register yet more timers.
 *
 * @param processFunction The {@link ProcessFunction} that is called for each element
 *                      in the stream.
 *
 * @param <R> The type of elements emitted by the {@code ProcessFunction}.
 *
 * @return The transformed {@link DataStream}.
 *
 * @deprecated Use {@link KeyedStream#process(KeyedProcessFunction)}
 */
@Deprecated
@Override
@PublicEvolving
public <R> SingleOutputStreamOperator<R> process(ProcessFunction<T, R> processFunction) {

	TypeInformation<R> outType = TypeExtractor.getUnaryOperatorReturnType(
		processFunction,
		ProcessFunction.class,
		0,
		1,
		TypeExtractor.NO_INDEX,
		getType(),
		Utils.getCallLocationName(),
		true);

	return process(processFunction, outType);
}
 

/**
 * Applies the given {@link ProcessFunction} on the input stream, thereby creating a transformed output stream.
 *
 * <p>The function will be called for every element in the input streams and can produce zero
 * or more output elements. Contrary to the {@link DataStream#flatMap(FlatMapFunction)}
 * function, this function can also query the time and set timers. When reacting to the firing
 * of set timers the function can directly emit elements and/or register yet more timers.
 *
 * @param processFunction The {@link ProcessFunction} that is called for each element
 *                      in the stream.
 *
 * @param <R> The type of elements emitted by the {@code ProcessFunction}.
 *
 * @return The transformed {@link DataStream}.
 *
 * @deprecated Use {@link KeyedStream#process(KeyedProcessFunction)}
 */
@Deprecated
@Override
@PublicEvolving
public <R> SingleOutputStreamOperator<R> process(ProcessFunction<T, R> processFunction) {

	TypeInformation<R> outType = TypeExtractor.getUnaryOperatorReturnType(
		processFunction,
		ProcessFunction.class,
		0,
		1,
		TypeExtractor.NO_INDEX,
		getType(),
		Utils.getCallLocationName(),
		true);

	return process(processFunction, outType);
}
 

@SuppressWarnings("unchecked")
private OneInputStreamOperatorTestHarness<BaseRow, BaseRow> createHarness(
		JoinType joinType,
		FilterOnTable filterOnTable) throws Exception {
	boolean isLeftJoin = joinType == JoinType.LEFT_JOIN;
	ProcessFunction<BaseRow, BaseRow> joinRunner;
	if (filterOnTable == FilterOnTable.WITHOUT_FILTER) {
		joinRunner = new LookupJoinRunner(
			new GeneratedFunctionWrapper<>(new TestingFetcherFunction()),
			new GeneratedCollectorWrapper<>(new TestingFetcherCollector()),
			isLeftJoin,
			2);
	} else {
		joinRunner = new LookupJoinWithCalcRunner(
			new GeneratedFunctionWrapper<>(new TestingFetcherFunction()),
			new GeneratedFunctionWrapper<>(new CalculateOnTemporalTable()),
			new GeneratedCollectorWrapper<>(new TestingFetcherCollector()),
			isLeftJoin,
			2);
	}

	ProcessOperator<BaseRow, BaseRow> operator = new ProcessOperator<>(joinRunner);
	return new OneInputStreamOperatorTestHarness<>(
		operator,
		inSerializer);
}
 

@SuppressWarnings("unchecked")
private OneInputStreamOperatorTestHarness<RowData, RowData> createHarness(
		JoinType joinType,
		FilterOnTable filterOnTable) throws Exception {
	boolean isLeftJoin = joinType == JoinType.LEFT_JOIN;
	ProcessFunction<RowData, RowData> joinRunner;
	if (filterOnTable == FilterOnTable.WITHOUT_FILTER) {
		joinRunner = new LookupJoinRunner(
			new GeneratedFunctionWrapper<>(new TestingFetcherFunction()),
			new GeneratedCollectorWrapper<>(new TestingFetcherCollector()),
			isLeftJoin,
			2);
	} else {
		joinRunner = new LookupJoinWithCalcRunner(
			new GeneratedFunctionWrapper<>(new TestingFetcherFunction()),
			new GeneratedFunctionWrapper<>(new CalculateOnTemporalTable()),
			new GeneratedCollectorWrapper<>(new TestingFetcherCollector()),
			isLeftJoin,
			2);
	}

	ProcessOperator<RowData, RowData> operator = new ProcessOperator<>(joinRunner);
	return new OneInputStreamOperatorTestHarness<>(
		operator,
		inSerializer);
}
 

/**
 * Test ProcessFunction side outputs with wrong {@code OutputTag}.
 */
@Test
public void testProcessFunctionSideOutputWithWrongTag() throws Exception {
	final OutputTag<String> sideOutputTag1 = new OutputTag<String>("side"){};
	final OutputTag<String> sideOutputTag2 = new OutputTag<String>("other-side"){};

	TestListResultSink<String> sideOutputResultSink = new TestListResultSink<>();

	StreamExecutionEnvironment see = StreamExecutionEnvironment.getExecutionEnvironment();
	see.setParallelism(3);

	DataStream<Integer> dataStream = see.fromCollection(elements);

	dataStream
			.process(new ProcessFunction<Integer, Integer>() {
				private static final long serialVersionUID = 1L;

				@Override
				public void processElement(
						Integer value, Context ctx, Collector<Integer> out) throws Exception {
					out.collect(value);
					ctx.output(sideOutputTag2, "sideout-" + String.valueOf(value));
				}
			}).getSideOutput(sideOutputTag1).addSink(sideOutputResultSink);

	see.execute();

	assertEquals(Arrays.asList(), sideOutputResultSink.getSortedResult());
}
 

@Test
public void testSideOutputWithMultipleConsumersWithObjectReuse() throws Exception {
	final OutputTag<String> sideOutputTag = new OutputTag<String>("side"){};

	TestListResultSink<String> sideOutputResultSink1 = new TestListResultSink<>();
	TestListResultSink<String> sideOutputResultSink2 = new TestListResultSink<>();
	TestListResultSink<Integer> resultSink = new TestListResultSink<>();

	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
	env.getConfig().enableObjectReuse();
	env.setParallelism(3);

	DataStream<Integer> dataStream = env.fromCollection(elements);

	SingleOutputStreamOperator<Integer> passThroughtStream = dataStream
			.process(new ProcessFunction<Integer, Integer>() {
				private static final long serialVersionUID = 1L;

				@Override
				public void processElement(
						Integer value, Context ctx, Collector<Integer> out) throws Exception {
					out.collect(value);
					ctx.output(sideOutputTag, "sideout-" + String.valueOf(value));
				}
			});

	passThroughtStream.getSideOutput(sideOutputTag).addSink(sideOutputResultSink1);
	passThroughtStream.getSideOutput(sideOutputTag).addSink(sideOutputResultSink2);
	passThroughtStream.addSink(resultSink);
	env.execute();

	assertEquals(Arrays.asList("sideout-1", "sideout-2", "sideout-3", "sideout-4", "sideout-5"), sideOutputResultSink1.getSortedResult());
	assertEquals(Arrays.asList("sideout-1", "sideout-2", "sideout-3", "sideout-4", "sideout-5"), sideOutputResultSink2.getSortedResult());
	assertEquals(Arrays.asList(1, 2, 3, 4, 5), resultSink.getSortedResult());
}
 

public DataStream<String> constructTestPipeline(DataStream<String> source) {

		OutputTag<Integer> filtered = new OutputTag<>("filter", BasicTypeInfo.INT_TYPE_INFO);
		OutputTag<Integer> process = new OutputTag<>("process", BasicTypeInfo.INT_TYPE_INFO);

		SingleOutputStreamOperator<String> input = source.process(new ProcessFunction<String, String>() {
			private static final long serialVersionUID = 1L;

			@Override
			public void processElement(String s, Context ctx,
					Collector<String> out) throws Exception {

				if (s.startsWith("filter ")) {
					ctx.output(filtered, Integer.parseInt(s.substring(7)));
				} else if (s.startsWith("process ")) {
					ctx.output(process, Integer.parseInt(s.substring(8)));
				} else {
					throw new RuntimeException("oOoO");
				}

			}
		});

		BroadcastStream<Integer> broadcast = input.getSideOutput(filtered).broadcast(bcstate);

		return input.getSideOutput(process).keyBy(i -> i).connect(broadcast).process(new BroadcastProcessor(bcstate))
				.uid("stateful");
	}
 

/**
 * Creates and adds two synthetic sources for {@link DepositEvent} and {@link TransactionEvent}.
 *
 * @param env              the streaming environment to add the sources to.
 * @param recordsPerSecond the number of {@link TransactionEvent} per second to generate.
 * @return a {@link DataStream} for each event type generated.
 */
public static SyntheticSources create(StreamExecutionEnvironment env, int recordsPerSecond) {

    final DataStreamSource<Either<DepositEvent, TransactionEvent>> depositsAndTransactions = env.addSource(
            new DepositsThenTransactionsSource(recordsPerSecond));

    final OutputTag<TransactionEvent> transactionsSideOutput = new OutputTag<>(
            "transactions side output",
            TypeInformation.of(TransactionEvent.class));

    final SingleOutputStreamOperator<DepositEvent> deposits = depositsAndTransactions.process(
            new ProcessFunction<Either<DepositEvent, TransactionEvent>, DepositEvent>() {

                @Override
                public void processElement(
                        Either<DepositEvent, TransactionEvent> depositOrTransaction,
                        Context context,
                        Collector<DepositEvent> out) {

                    if (depositOrTransaction.isLeft()) {
                        out.collect(depositOrTransaction.left());
                    }
                    else {
                        context.output(transactionsSideOutput, depositOrTransaction.right());
                    }
                }
            });

    final DataStream<TransactionEvent> transactions = deposits.getSideOutput(transactionsSideOutput);

    return new SyntheticSources(deposits, transactions);
}
 

public static void main(String[] args) throws Exception {
    final ParameterTool params = ParameterTool.fromArgs(args);
    final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    env.getConfig().setGlobalJobParameters(params);

    DataStreamSource<MetricEvent> data = KafkaConfigUtil.buildSource(env);  //从 Kafka 获取到所有的数据流
    SingleOutputStreamOperator<MetricEvent> sideOutputData = data.process(new ProcessFunction<MetricEvent, MetricEvent>() {
        @Override
        public void processElement(MetricEvent metricEvent, Context context, Collector<MetricEvent> collector) throws Exception {
            String type = metricEvent.getTags().get("type");
            switch (type) {
                case "machine":
                    context.output(machineTag, metricEvent);
                case "docker":
                    context.output(dockerTag, metricEvent);
                case "application":
                    context.output(applicationTag, metricEvent);
                case "middleware":
                    context.output(middlewareTag, metricEvent);
                default:
                    collector.collect(metricEvent);
            }
        }
    });
    DataStream<MetricEvent> machine = sideOutputData.getSideOutput(machineTag);
    DataStream<MetricEvent> docker = sideOutputData.getSideOutput(dockerTag);
    DataStream<MetricEvent> application = sideOutputData.getSideOutput(applicationTag);
    DataStream<MetricEvent> middleware = sideOutputData.getSideOutput(middlewareTag);
}
 

public static void main(String[] args) throws Exception {
    StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();

    env.enableCheckpointing(3000);

    Properties consumerProps = new Properties();
    consumerProps.setProperty(RocketMQConfig.NAME_SERVER_ADDR, "localhost:9876");
    consumerProps.setProperty(RocketMQConfig.CONSUMER_GROUP, "c002");
    consumerProps.setProperty(RocketMQConfig.CONSUMER_TOPIC, "zhisheng");


    Properties producerProps = new Properties();
    producerProps.setProperty(RocketMQConfig.NAME_SERVER_ADDR, "localhost:9876");
    int msgDelayLevel = RocketMQConfig.MSG_DELAY_LEVEL05;
    producerProps.setProperty(RocketMQConfig.MSG_DELAY_LEVEL, String.valueOf(msgDelayLevel));
    // TimeDelayLevel is not supported for batching
    boolean batchFlag = msgDelayLevel <= 0;

    env.addSource(new RocketMQSource(new SimpleKeyValueDeserializationSchema("id", "address"), consumerProps))
            .name("rocketmq-source")
            .setParallelism(2)
            .process(new ProcessFunction<Map, Map>() {
                @Override
                public void processElement(Map in, Context ctx, Collector<Map> out) throws Exception {
                    HashMap result = new HashMap();
                    result.put("id", in.get("id"));
                    String[] arr = in.get("address").toString().split("\\s+");
                    result.put("province", arr[arr.length - 1]);
                    out.collect(result);
                }
            })
            .name("upper-processor")
            .setParallelism(2)
            .addSink(new RocketMQSink(new SimpleKeyValueSerializationSchema("id", "province"),
                    new DefaultTopicSelector("zhisheng"), producerProps).withBatchFlushOnCheckpoint(batchFlag))
            .name("rocketmq-sink")
            .setParallelism(2);

    env.execute("rocketmq-flink-example");
}
 

public static void main(String[] args) throws Exception {
    final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();


    //并行度为1
    env.setParallelism(1);

    //创建一个List，里面有两个Tuple2元素
    List<Tuple2<String, Integer>> list = new ArrayList<>();
    list.add(new Tuple2("aaa", 1));
    list.add(new Tuple2("bbb", 2));
    list.add(new Tuple2("ccc", 3));
    list.add(new Tuple2("ddd", 4));
    list.add(new Tuple2("eee", 5));
    list.add(new Tuple2("fff", 6));

    //通过List创建DataStream
    DataStream<Tuple2<String, Integer>> fromCollectionDataStream = env.fromCollection(list);

    //统计每个单词的数量
    fromCollectionDataStream
            .process(new ProcessFunction<Tuple2<String, Integer>, String>() {
                @Override
                public void processElement(Tuple2<String, Integer> value, Context ctx, Collector<String> out) throws Exception {
                    out.collect(value.f1 + "个【" + value.f0 + "】");
                }
            })
            .print();


    env.execute("processfunction demo : processfunction");
}
 

@Test
public void testSideOutputWithMultipleConsumers() throws Exception {
	final OutputTag<String> sideOutputTag = new OutputTag<String>("side"){};

	TestListResultSink<String> sideOutputResultSink1 = new TestListResultSink<>();
	TestListResultSink<String> sideOutputResultSink2 = new TestListResultSink<>();
	TestListResultSink<Integer> resultSink = new TestListResultSink<>();

	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
	env.setParallelism(3);

	DataStream<Integer> dataStream = env.fromCollection(elements);

	SingleOutputStreamOperator<Integer> passThroughtStream = dataStream
			.process(new ProcessFunction<Integer, Integer>() {
				private static final long serialVersionUID = 1L;

				@Override
				public void processElement(
						Integer value, Context ctx, Collector<Integer> out) throws Exception {
					out.collect(value);
					ctx.output(sideOutputTag, "sideout-" + String.valueOf(value));
				}
			});

	passThroughtStream.getSideOutput(sideOutputTag).addSink(sideOutputResultSink1);
	passThroughtStream.getSideOutput(sideOutputTag).addSink(sideOutputResultSink2);
	passThroughtStream.addSink(resultSink);
	env.execute();

	assertEquals(Arrays.asList("sideout-1", "sideout-2", "sideout-3", "sideout-4", "sideout-5"), sideOutputResultSink1.getSortedResult());
	assertEquals(Arrays.asList("sideout-1", "sideout-2", "sideout-3", "sideout-4", "sideout-5"), sideOutputResultSink2.getSortedResult());
	assertEquals(Arrays.asList(1, 2, 3, 4, 5), resultSink.getSortedResult());
}
 

@Test
public void testSideOutputWithMultipleConsumersWithObjectReuse() throws Exception {
	final OutputTag<String> sideOutputTag = new OutputTag<String>("side"){};

	TestListResultSink<String> sideOutputResultSink1 = new TestListResultSink<>();
	TestListResultSink<String> sideOutputResultSink2 = new TestListResultSink<>();
	TestListResultSink<Integer> resultSink = new TestListResultSink<>();

	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
	env.getConfig().enableObjectReuse();
	env.setParallelism(3);

	DataStream<Integer> dataStream = env.fromCollection(elements);

	SingleOutputStreamOperator<Integer> passThroughtStream = dataStream
			.process(new ProcessFunction<Integer, Integer>() {
				private static final long serialVersionUID = 1L;

				@Override
				public void processElement(
						Integer value, Context ctx, Collector<Integer> out) throws Exception {
					out.collect(value);
					ctx.output(sideOutputTag, "sideout-" + String.valueOf(value));
				}
			});

	passThroughtStream.getSideOutput(sideOutputTag).addSink(sideOutputResultSink1);
	passThroughtStream.getSideOutput(sideOutputTag).addSink(sideOutputResultSink2);
	passThroughtStream.addSink(resultSink);
	env.execute();

	assertEquals(Arrays.asList("sideout-1", "sideout-2", "sideout-3", "sideout-4", "sideout-5"), sideOutputResultSink1.getSortedResult());
	assertEquals(Arrays.asList("sideout-1", "sideout-2", "sideout-3", "sideout-4", "sideout-5"), sideOutputResultSink2.getSortedResult());
	assertEquals(Arrays.asList(1, 2, 3, 4, 5), resultSink.getSortedResult());
}
 

@Test
public void testDifferentSideOutputTypes() throws Exception {
	final OutputTag<String> sideOutputTag1 = new OutputTag<String>("string"){};
	final OutputTag<Integer> sideOutputTag2 = new OutputTag<Integer>("int"){};

	TestListResultSink<String> sideOutputResultSink1 = new TestListResultSink<>();
	TestListResultSink<Integer> sideOutputResultSink2 = new TestListResultSink<>();
	TestListResultSink<Integer> resultSink = new TestListResultSink<>();

	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
	env.getConfig().enableObjectReuse();
	env.setParallelism(3);

	DataStream<Integer> dataStream = env.fromCollection(elements);

	SingleOutputStreamOperator<Integer> passThroughtStream = dataStream
			.process(new ProcessFunction<Integer, Integer>() {
				private static final long serialVersionUID = 1L;

				@Override
				public void processElement(
						Integer value, Context ctx, Collector<Integer> out) throws Exception {
					out.collect(value);
					ctx.output(sideOutputTag1, "sideout-" + String.valueOf(value));
					ctx.output(sideOutputTag2, 13);
				}
			});

	passThroughtStream.getSideOutput(sideOutputTag1).addSink(sideOutputResultSink1);
	passThroughtStream.getSideOutput(sideOutputTag2).addSink(sideOutputResultSink2);
	passThroughtStream.addSink(resultSink);
	env.execute();

	assertEquals(Arrays.asList("sideout-1", "sideout-2", "sideout-3", "sideout-4", "sideout-5"), sideOutputResultSink1.getSortedResult());
	assertEquals(Arrays.asList(13, 13, 13, 13, 13), sideOutputResultSink2.getSortedResult());
	assertEquals(Arrays.asList(1, 2, 3, 4, 5), resultSink.getSortedResult());
}
 

@Test
public void testSideOutputNameClash() throws Exception {
	final OutputTag<String> sideOutputTag1 = new OutputTag<String>("side"){};
	final OutputTag<Integer> sideOutputTag2 = new OutputTag<Integer>("side"){};

	TestListResultSink<String> sideOutputResultSink1 = new TestListResultSink<>();
	TestListResultSink<Integer> sideOutputResultSink2 = new TestListResultSink<>();

	StreamExecutionEnvironment see = StreamExecutionEnvironment.getExecutionEnvironment();
	see.setParallelism(3);

	DataStream<Integer> dataStream = see.fromCollection(elements);

	SingleOutputStreamOperator<Integer> passThroughtStream = dataStream
			.process(new ProcessFunction<Integer, Integer>() {
				private static final long serialVersionUID = 1L;

				@Override
				public void processElement(
						Integer value, Context ctx, Collector<Integer> out) throws Exception {
					out.collect(value);
					ctx.output(sideOutputTag1, "sideout-" + String.valueOf(value));
					ctx.output(sideOutputTag2, 13);
				}
			});

	passThroughtStream.getSideOutput(sideOutputTag1).addSink(sideOutputResultSink1);

	expectedException.expect(UnsupportedOperationException.class);
	passThroughtStream.getSideOutput(sideOutputTag2).addSink(sideOutputResultSink2);
}
 

/**
 * Test ProcessFunction side output.
 */
@Test
public void testProcessFunctionSideOutput() throws Exception {
	final OutputTag<String> sideOutputTag = new OutputTag<String>("side"){};

	TestListResultSink<String> sideOutputResultSink = new TestListResultSink<>();
	TestListResultSink<Integer> resultSink = new TestListResultSink<>();

	StreamExecutionEnvironment see = StreamExecutionEnvironment.getExecutionEnvironment();
	see.setParallelism(3);

	DataStream<Integer> dataStream = see.fromCollection(elements);

	SingleOutputStreamOperator<Integer> passThroughtStream = dataStream
			.process(new ProcessFunction<Integer, Integer>() {
				private static final long serialVersionUID = 1L;

				@Override
				public void processElement(
						Integer value, Context ctx, Collector<Integer> out) throws Exception {
					out.collect(value);
					ctx.output(sideOutputTag, "sideout-" + String.valueOf(value));
				}
			});

	passThroughtStream.getSideOutput(sideOutputTag).addSink(sideOutputResultSink);
	passThroughtStream.addSink(resultSink);
	see.execute();

	assertEquals(Arrays.asList("sideout-1", "sideout-2", "sideout-3", "sideout-4", "sideout-5"), sideOutputResultSink.getSortedResult());
	assertEquals(Arrays.asList(1, 2, 3, 4, 5), resultSink.getSortedResult());
}
 

/**
 * Test ProcessFunction side outputs with wrong {@code OutputTag}.
 */
@Test
public void testProcessFunctionSideOutputWithWrongTag() throws Exception {
	final OutputTag<String> sideOutputTag1 = new OutputTag<String>("side"){};
	final OutputTag<String> sideOutputTag2 = new OutputTag<String>("other-side"){};

	TestListResultSink<String> sideOutputResultSink = new TestListResultSink<>();

	StreamExecutionEnvironment see = StreamExecutionEnvironment.getExecutionEnvironment();
	see.setParallelism(3);

	DataStream<Integer> dataStream = see.fromCollection(elements);

	dataStream
			.process(new ProcessFunction<Integer, Integer>() {
				private static final long serialVersionUID = 1L;

				@Override
				public void processElement(
						Integer value, Context ctx, Collector<Integer> out) throws Exception {
					out.collect(value);
					ctx.output(sideOutputTag2, "sideout-" + String.valueOf(value));
				}
			}).getSideOutput(sideOutputTag1).addSink(sideOutputResultSink);

	see.execute();

	assertEquals(Arrays.asList(), sideOutputResultSink.getSortedResult());
}
 

@Test
public void testSideOutputNameClash() throws Exception {
	final OutputTag<String> sideOutputTag1 = new OutputTag<String>("side"){};
	final OutputTag<Integer> sideOutputTag2 = new OutputTag<Integer>("side"){};

	TestListResultSink<String> sideOutputResultSink1 = new TestListResultSink<>();
	TestListResultSink<Integer> sideOutputResultSink2 = new TestListResultSink<>();

	StreamExecutionEnvironment see = StreamExecutionEnvironment.getExecutionEnvironment();
	see.setParallelism(3);

	DataStream<Integer> dataStream = see.fromCollection(elements);

	SingleOutputStreamOperator<Integer> passThroughtStream = dataStream
			.process(new ProcessFunction<Integer, Integer>() {
				private static final long serialVersionUID = 1L;

				@Override
				public void processElement(
						Integer value, Context ctx, Collector<Integer> out) throws Exception {
					out.collect(value);
					ctx.output(sideOutputTag1, "sideout-" + String.valueOf(value));
					ctx.output(sideOutputTag2, 13);
				}
			});

	passThroughtStream.getSideOutput(sideOutputTag1).addSink(sideOutputResultSink1);

	expectedException.expect(UnsupportedOperationException.class);
	passThroughtStream.getSideOutput(sideOutputTag2).addSink(sideOutputResultSink2);
}
 

@Test
public void testSideOutputWithMultipleConsumers() throws Exception {
	final OutputTag<String> sideOutputTag = new OutputTag<String>("side"){};

	TestListResultSink<String> sideOutputResultSink1 = new TestListResultSink<>();
	TestListResultSink<String> sideOutputResultSink2 = new TestListResultSink<>();
	TestListResultSink<Integer> resultSink = new TestListResultSink<>();

	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
	env.setParallelism(3);

	DataStream<Integer> dataStream = env.fromCollection(elements);

	SingleOutputStreamOperator<Integer> passThroughtStream = dataStream
			.process(new ProcessFunction<Integer, Integer>() {
				private static final long serialVersionUID = 1L;

				@Override
				public void processElement(
						Integer value, Context ctx, Collector<Integer> out) throws Exception {
					out.collect(value);
					ctx.output(sideOutputTag, "sideout-" + String.valueOf(value));
				}
			});

	passThroughtStream.getSideOutput(sideOutputTag).addSink(sideOutputResultSink1);
	passThroughtStream.getSideOutput(sideOutputTag).addSink(sideOutputResultSink2);
	passThroughtStream.addSink(resultSink);
	env.execute();

	assertEquals(Arrays.asList("sideout-1", "sideout-2", "sideout-3", "sideout-4", "sideout-5"), sideOutputResultSink1.getSortedResult());
	assertEquals(Arrays.asList("sideout-1", "sideout-2", "sideout-3", "sideout-4", "sideout-5"), sideOutputResultSink2.getSortedResult());
	assertEquals(Arrays.asList(1, 2, 3, 4, 5), resultSink.getSortedResult());
}
 

@Test
public void testDifferentSideOutputTypes() throws Exception {
	final OutputTag<String> sideOutputTag1 = new OutputTag<String>("string"){};
	final OutputTag<Integer> sideOutputTag2 = new OutputTag<Integer>("int"){};

	TestListResultSink<String> sideOutputResultSink1 = new TestListResultSink<>();
	TestListResultSink<Integer> sideOutputResultSink2 = new TestListResultSink<>();
	TestListResultSink<Integer> resultSink = new TestListResultSink<>();

	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
	env.getConfig().enableObjectReuse();
	env.setParallelism(3);

	DataStream<Integer> dataStream = env.fromCollection(elements);

	SingleOutputStreamOperator<Integer> passThroughtStream = dataStream
			.process(new ProcessFunction<Integer, Integer>() {
				private static final long serialVersionUID = 1L;

				@Override
				public void processElement(
						Integer value, Context ctx, Collector<Integer> out) throws Exception {
					out.collect(value);
					ctx.output(sideOutputTag1, "sideout-" + String.valueOf(value));
					ctx.output(sideOutputTag2, 13);
				}
			});

	passThroughtStream.getSideOutput(sideOutputTag1).addSink(sideOutputResultSink1);
	passThroughtStream.getSideOutput(sideOutputTag2).addSink(sideOutputResultSink2);
	passThroughtStream.addSink(resultSink);
	env.execute();

	assertEquals(Arrays.asList("sideout-1", "sideout-2", "sideout-3", "sideout-4", "sideout-5"), sideOutputResultSink1.getSortedResult());
	assertEquals(Arrays.asList(13, 13, 13, 13, 13), sideOutputResultSink2.getSortedResult());
	assertEquals(Arrays.asList(1, 2, 3, 4, 5), resultSink.getSortedResult());
}
 

@Test
public void testSideOutputNameClash() throws Exception {
	final OutputTag<String> sideOutputTag1 = new OutputTag<String>("side"){};
	final OutputTag<Integer> sideOutputTag2 = new OutputTag<Integer>("side"){};

	TestListResultSink<String> sideOutputResultSink1 = new TestListResultSink<>();
	TestListResultSink<Integer> sideOutputResultSink2 = new TestListResultSink<>();

	StreamExecutionEnvironment see = StreamExecutionEnvironment.getExecutionEnvironment();
	see.setParallelism(3);

	DataStream<Integer> dataStream = see.fromCollection(elements);

	SingleOutputStreamOperator<Integer> passThroughtStream = dataStream
			.process(new ProcessFunction<Integer, Integer>() {
				private static final long serialVersionUID = 1L;

				@Override
				public void processElement(
						Integer value, Context ctx, Collector<Integer> out) throws Exception {
					out.collect(value);
					ctx.output(sideOutputTag1, "sideout-" + String.valueOf(value));
					ctx.output(sideOutputTag2, 13);
				}
			});

	passThroughtStream.getSideOutput(sideOutputTag1).addSink(sideOutputResultSink1);

	expectedException.expect(UnsupportedOperationException.class);
	passThroughtStream.getSideOutput(sideOutputTag2).addSink(sideOutputResultSink2);
}
 

/**
 * Test ProcessFunction side output.
 */
@Test
public void testProcessFunctionSideOutput() throws Exception {
	final OutputTag<String> sideOutputTag = new OutputTag<String>("side"){};

	TestListResultSink<String> sideOutputResultSink = new TestListResultSink<>();
	TestListResultSink<Integer> resultSink = new TestListResultSink<>();

	StreamExecutionEnvironment see = StreamExecutionEnvironment.getExecutionEnvironment();
	see.setParallelism(3);

	DataStream<Integer> dataStream = see.fromCollection(elements);

	SingleOutputStreamOperator<Integer> passThroughtStream = dataStream
			.process(new ProcessFunction<Integer, Integer>() {
				private static final long serialVersionUID = 1L;

				@Override
				public void processElement(
						Integer value, Context ctx, Collector<Integer> out) throws Exception {
					out.collect(value);
					ctx.output(sideOutputTag, "sideout-" + String.valueOf(value));
				}
			});

	passThroughtStream.getSideOutput(sideOutputTag).addSink(sideOutputResultSink);
	passThroughtStream.addSink(resultSink);
	see.execute();

	assertEquals(Arrays.asList("sideout-1", "sideout-2", "sideout-3", "sideout-4", "sideout-5"), sideOutputResultSink.getSortedResult());
	assertEquals(Arrays.asList(1, 2, 3, 4, 5), resultSink.getSortedResult());
}
 

/**
 * Test ProcessFunction side outputs with wrong {@code OutputTag}.
 */
@Test
public void testProcessFunctionSideOutputWithWrongTag() throws Exception {
	final OutputTag<String> sideOutputTag1 = new OutputTag<String>("side"){};
	final OutputTag<String> sideOutputTag2 = new OutputTag<String>("other-side"){};

	TestListResultSink<String> sideOutputResultSink = new TestListResultSink<>();

	StreamExecutionEnvironment see = StreamExecutionEnvironment.getExecutionEnvironment();
	see.setParallelism(3);

	DataStream<Integer> dataStream = see.fromCollection(elements);

	dataStream
			.process(new ProcessFunction<Integer, Integer>() {
				private static final long serialVersionUID = 1L;

				@Override
				public void processElement(
						Integer value, Context ctx, Collector<Integer> out) throws Exception {
					out.collect(value);
					ctx.output(sideOutputTag2, "sideout-" + String.valueOf(value));
				}
			}).getSideOutput(sideOutputTag1).addSink(sideOutputResultSink);

	see.execute();

	assertEquals(Arrays.asList(), sideOutputResultSink.getSortedResult());
}
 

public static void main(String[] args) throws Exception {
    final ParameterTool params = ParameterTool.fromArgs(args);
    final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    env.getConfig().setGlobalJobParameters(params);

    DataStreamSource<MetricEvent> data = KafkaConfigUtil.buildSource(env);  //从 Kafka 获取到所有的数据流
    SingleOutputStreamOperator<MetricEvent> sideOutputData = data.process(new ProcessFunction<MetricEvent, MetricEvent>() {
        @Override
        public void processElement(MetricEvent metricEvent, Context context, Collector<MetricEvent> collector) throws Exception {
            String type = metricEvent.getTags().get("type");
            switch (type) {
                case "machine":
                    context.output(machineTag, metricEvent);
                case "docker":
                    context.output(dockerTag, metricEvent);
                case "application":
                    context.output(applicationTag, metricEvent);
                case "middleware":
                    context.output(middlewareTag, metricEvent);
                default:
                    collector.collect(metricEvent);
            }
        }
    });
    DataStream<MetricEvent> machine = sideOutputData.getSideOutput(machineTag);
    DataStream<MetricEvent> docker = sideOutputData.getSideOutput(dockerTag);
    DataStream<MetricEvent> application = sideOutputData.getSideOutput(applicationTag);
    DataStream<MetricEvent> middleware = sideOutputData.getSideOutput(middlewareTag);
}