Java Code Examples for org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator#getSideOutput()

The following examples show how to use org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator#getSideOutput() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: SerialStreamingLedgerRuntimeProvider.java    From da-streamingledger with Apache License 2.0 6 votes vote down vote up
@Override
public ResultStreams translate(String name, List<InputAndSpec<?, ?>> streamLedgerSpecs) {
    List<OutputTag<?>> sideOutputTags = createSideOutputTags(streamLedgerSpecs);

    // the input stream is a union of different streams.
    KeyedStream<TaggedElement, Boolean> input = union(streamLedgerSpecs)
            .keyBy(unused -> true);

    // main pipeline
    String serialTransactorName = "SerialTransactor(" + name + ")";
    SingleOutputStreamOperator<Void> resultStream = input
            .process(new SerialTransactor(specs(streamLedgerSpecs), sideOutputTags))
            .name(serialTransactorName)
            .uid(serialTransactorName + "___SERIAL_TX")
            .forceNonParallel()
            .returns(Void.class);

    // gather the sideOutputs.
    Map<String, DataStream<?>> output = new HashMap<>();
    for (OutputTag<?> outputTag : sideOutputTags) {
        DataStream<?> rs = resultStream.getSideOutput(outputTag);
        output.put(outputTag.getId(), rs);
    }
    return new ResultStreams(output);
}
 
Example 2
Source File: SideOutputEvent.java    From flink-learning with Apache License 2.0 5 votes vote down vote up
public static void main(String[] args) throws Exception {
    final ParameterTool params = ParameterTool.fromArgs(args);
    final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    env.getConfig().setGlobalJobParameters(params);

    DataStreamSource<MetricEvent> data = KafkaConfigUtil.buildSource(env);  //从 Kafka 获取到所有的数据流
    SingleOutputStreamOperator<MetricEvent> sideOutputData = data.process(new ProcessFunction<MetricEvent, MetricEvent>() {
        @Override
        public void processElement(MetricEvent metricEvent, Context context, Collector<MetricEvent> collector) throws Exception {
            String type = metricEvent.getTags().get("type");
            switch (type) {
                case "machine":
                    context.output(machineTag, metricEvent);
                case "docker":
                    context.output(dockerTag, metricEvent);
                case "application":
                    context.output(applicationTag, metricEvent);
                case "middleware":
                    context.output(middlewareTag, metricEvent);
                default:
                    collector.collect(metricEvent);
            }
        }
    });
    DataStream<MetricEvent> machine = sideOutputData.getSideOutput(machineTag);
    DataStream<MetricEvent> docker = sideOutputData.getSideOutput(dockerTag);
    DataStream<MetricEvent> application = sideOutputData.getSideOutput(applicationTag);
    DataStream<MetricEvent> middleware = sideOutputData.getSideOutput(middlewareTag);
}
 
Example 3
Source File: SyntheticSources.java    From da-streamingledger with Apache License 2.0 5 votes vote down vote up
/**
 * Creates and adds two synthetic sources for {@link DepositEvent} and {@link TransactionEvent}.
 *
 * @param env              the streaming environment to add the sources to.
 * @param recordsPerSecond the number of {@link TransactionEvent} per second to generate.
 * @return a {@link DataStream} for each event type generated.
 */
public static SyntheticSources create(StreamExecutionEnvironment env, int recordsPerSecond) {

    final DataStreamSource<Either<DepositEvent, TransactionEvent>> depositsAndTransactions = env.addSource(
            new DepositsThenTransactionsSource(recordsPerSecond));

    final OutputTag<TransactionEvent> transactionsSideOutput = new OutputTag<>(
            "transactions side output",
            TypeInformation.of(TransactionEvent.class));

    final SingleOutputStreamOperator<DepositEvent> deposits = depositsAndTransactions.process(
            new ProcessFunction<Either<DepositEvent, TransactionEvent>, DepositEvent>() {

                @Override
                public void processElement(
                        Either<DepositEvent, TransactionEvent> depositOrTransaction,
                        Context context,
                        Collector<DepositEvent> out) {

                    if (depositOrTransaction.isLeft()) {
                        out.collect(depositOrTransaction.left());
                    }
                    else {
                        context.output(transactionsSideOutput, depositOrTransaction.right());
                    }
                }
            });

    final DataStream<TransactionEvent> transactions = deposits.getSideOutput(transactionsSideOutput);

    return new SyntheticSources(deposits, transactions);
}
 
Example 4
Source File: SideOutputEvent.java    From flink-learning with Apache License 2.0 5 votes vote down vote up
public static void main(String[] args) throws Exception {
    final ParameterTool params = ParameterTool.fromArgs(args);
    final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    env.getConfig().setGlobalJobParameters(params);

    DataStreamSource<MetricEvent> data = KafkaConfigUtil.buildSource(env);  //从 Kafka 获取到所有的数据流
    SingleOutputStreamOperator<MetricEvent> sideOutputData = data.process(new ProcessFunction<MetricEvent, MetricEvent>() {
        @Override
        public void processElement(MetricEvent metricEvent, Context context, Collector<MetricEvent> collector) throws Exception {
            String type = metricEvent.getTags().get("type");
            switch (type) {
                case "machine":
                    context.output(machineTag, metricEvent);
                case "docker":
                    context.output(dockerTag, metricEvent);
                case "application":
                    context.output(applicationTag, metricEvent);
                case "middleware":
                    context.output(middlewareTag, metricEvent);
                default:
                    collector.collect(metricEvent);
            }
        }
    });
    DataStream<MetricEvent> machine = sideOutputData.getSideOutput(machineTag);
    DataStream<MetricEvent> docker = sideOutputData.getSideOutput(dockerTag);
    DataStream<MetricEvent> application = sideOutputData.getSideOutput(applicationTag);
    DataStream<MetricEvent> middleware = sideOutputData.getSideOutput(middlewareTag);
}
 
Example 5
Source File: PatternStream.java    From Flink-CEPplus with Apache License 2.0 4 votes vote down vote up
/**
 * Applies a select function to the detected pattern sequence. For each pattern sequence the
 * provided {@link PatternSelectFunction} is called. The pattern select function can produce
 * exactly one resulting element.
 *
 * <p>Applies a timeout function to a partial pattern sequence which has timed out. For each
 * partial pattern sequence the provided {@link PatternTimeoutFunction} is called. The pattern
 * timeout function can produce exactly one resulting element.
 *
 * @param patternTimeoutFunction The pattern timeout function which is called for each partial
 *                               pattern sequence which has timed out.
 * @param patternSelectFunction The pattern select function which is called for each detected
 *                              pattern sequence.
 * @param <L> Type of the resulting timeout elements
 * @param <R> Type of the resulting elements
 *
 * @deprecated Use {@link PatternStream#select(OutputTag, PatternTimeoutFunction, PatternSelectFunction)}
 * that returns timed out events as a side-output
 *
 * @return {@link DataStream} which contains the resulting elements or the resulting timeout
 * elements wrapped in an {@link Either} type.
 */
@Deprecated
public <L, R> SingleOutputStreamOperator<Either<L, R>> select(
		final PatternTimeoutFunction<T, L> patternTimeoutFunction,
		final PatternSelectFunction<T, R> patternSelectFunction) {

	final TypeInformation<R> mainTypeInfo = TypeExtractor.getUnaryOperatorReturnType(
		patternSelectFunction,
		PatternSelectFunction.class,
		0,
		1,
		TypeExtractor.NO_INDEX,
		builder.getInputType(),
		null,
		false);

	final TypeInformation<L> timeoutTypeInfo = TypeExtractor.getUnaryOperatorReturnType(
		patternTimeoutFunction,
		PatternTimeoutFunction.class,
		0,
		1,
		TypeExtractor.NO_INDEX,
		builder.getInputType(),
		null,
		false);

	final TypeInformation<Either<L, R>> outTypeInfo = new EitherTypeInfo<>(timeoutTypeInfo, mainTypeInfo);

	final OutputTag<L> outputTag = new OutputTag<>(UUID.randomUUID().toString(), timeoutTypeInfo);

	final PatternProcessFunction<T, R> processFunction =
		fromSelect(builder.clean(patternSelectFunction))
			.withTimeoutHandler(outputTag, builder.clean(patternTimeoutFunction))
			.build();

	final SingleOutputStreamOperator<R> mainStream = process(processFunction, mainTypeInfo);
	final DataStream<L> timedOutStream = mainStream.getSideOutput(outputTag);

	return mainStream
		.connect(timedOutStream)
		.map(new CoMapTimeout<>())
		.returns(outTypeInfo);
}
 
Example 6
Source File: PatternStream.java    From Flink-CEPplus with Apache License 2.0 4 votes vote down vote up
/**
 * Applies a flat select function to the detected pattern sequence. For each pattern sequence
 * the provided {@link PatternFlatSelectFunction} is called. The pattern flat select function
 * can produce an arbitrary number of resulting elements.
 *
 * <p>Applies a timeout function to a partial pattern sequence which has timed out. For each
 * partial pattern sequence the provided {@link PatternFlatTimeoutFunction} is called. The
 * pattern timeout function can produce an arbitrary number of resulting elements.
 *
 * @param patternFlatTimeoutFunction The pattern flat timeout function which is called for each
 *                                   partial pattern sequence which has timed out.
 * @param patternFlatSelectFunction The pattern flat select function which is called for each
 *                                  detected pattern sequence.
 * @param <L> Type of the resulting timeout events
 * @param <R> Type of the resulting events
 *
 * @deprecated Use {@link PatternStream#flatSelect(OutputTag, PatternFlatTimeoutFunction, PatternFlatSelectFunction)}
 * that returns timed out events as a side-output
 *
 * @return {@link DataStream} which contains the resulting events from the pattern flat select
 * function or the resulting timeout events from the pattern flat timeout function wrapped in an
 * {@link Either} type.
 */
@Deprecated
public <L, R> SingleOutputStreamOperator<Either<L, R>> flatSelect(
		final PatternFlatTimeoutFunction<T, L> patternFlatTimeoutFunction,
		final PatternFlatSelectFunction<T, R> patternFlatSelectFunction) {

	final TypeInformation<L> timedOutTypeInfo = TypeExtractor.getUnaryOperatorReturnType(
		patternFlatTimeoutFunction,
		PatternFlatTimeoutFunction.class,
		0,
		1,
		new int[]{2, 0},
		builder.getInputType(),
		null,
		false);

	final TypeInformation<R> mainTypeInfo = TypeExtractor.getUnaryOperatorReturnType(
		patternFlatSelectFunction,
		PatternFlatSelectFunction.class,
		0,
		1,
		new int[]{1, 0},
		builder.getInputType(),
		null,
		false);

	final OutputTag<L> outputTag = new OutputTag<>(UUID.randomUUID().toString(), timedOutTypeInfo);

	final PatternProcessFunction<T, R> processFunction =
		fromFlatSelect(builder.clean(patternFlatSelectFunction))
			.withTimeoutHandler(outputTag, builder.clean(patternFlatTimeoutFunction))
			.build();

	final SingleOutputStreamOperator<R> mainStream = process(processFunction, mainTypeInfo);
	final DataStream<L> timedOutStream = mainStream.getSideOutput(outputTag);
	final TypeInformation<Either<L, R>> outTypeInfo = new EitherTypeInfo<>(timedOutTypeInfo, mainTypeInfo);

	return mainStream
			.connect(timedOutStream)
			.map(new CoMapTimeout<>())
			.returns(outTypeInfo);
}
 
Example 7
Source File: PatternStream.java    From flink with Apache License 2.0 4 votes vote down vote up
/**
 * Applies a select function to the detected pattern sequence. For each pattern sequence the
 * provided {@link PatternSelectFunction} is called. The pattern select function can produce
 * exactly one resulting element.
 *
 * <p>Applies a timeout function to a partial pattern sequence which has timed out. For each
 * partial pattern sequence the provided {@link PatternTimeoutFunction} is called. The pattern
 * timeout function can produce exactly one resulting element.
 *
 * @param patternTimeoutFunction The pattern timeout function which is called for each partial
 *                               pattern sequence which has timed out.
 * @param patternSelectFunction The pattern select function which is called for each detected
 *                              pattern sequence.
 * @param <L> Type of the resulting timeout elements
 * @param <R> Type of the resulting elements
 *
 * @deprecated Use {@link PatternStream#select(OutputTag, PatternTimeoutFunction, PatternSelectFunction)}
 * that returns timed out events as a side-output
 *
 * @return {@link DataStream} which contains the resulting elements or the resulting timeout
 * elements wrapped in an {@link Either} type.
 */
@Deprecated
public <L, R> SingleOutputStreamOperator<Either<L, R>> select(
		final PatternTimeoutFunction<T, L> patternTimeoutFunction,
		final PatternSelectFunction<T, R> patternSelectFunction) {

	final TypeInformation<R> mainTypeInfo = TypeExtractor.getUnaryOperatorReturnType(
		patternSelectFunction,
		PatternSelectFunction.class,
		0,
		1,
		TypeExtractor.NO_INDEX,
		builder.getInputType(),
		null,
		false);

	final TypeInformation<L> timeoutTypeInfo = TypeExtractor.getUnaryOperatorReturnType(
		patternTimeoutFunction,
		PatternTimeoutFunction.class,
		0,
		1,
		TypeExtractor.NO_INDEX,
		builder.getInputType(),
		null,
		false);

	final TypeInformation<Either<L, R>> outTypeInfo = new EitherTypeInfo<>(timeoutTypeInfo, mainTypeInfo);

	final OutputTag<L> outputTag = new OutputTag<>(UUID.randomUUID().toString(), timeoutTypeInfo);

	final PatternProcessFunction<T, R> processFunction =
		fromSelect(builder.clean(patternSelectFunction))
			.withTimeoutHandler(outputTag, builder.clean(patternTimeoutFunction))
			.build();

	final SingleOutputStreamOperator<R> mainStream = process(processFunction, mainTypeInfo);
	final DataStream<L> timedOutStream = mainStream.getSideOutput(outputTag);

	return mainStream
		.connect(timedOutStream)
		.map(new CoMapTimeout<>())
		.returns(outTypeInfo);
}
 
Example 8
Source File: PatternStream.java    From flink with Apache License 2.0 4 votes vote down vote up
/**
 * Applies a flat select function to the detected pattern sequence. For each pattern sequence
 * the provided {@link PatternFlatSelectFunction} is called. The pattern flat select function
 * can produce an arbitrary number of resulting elements.
 *
 * <p>Applies a timeout function to a partial pattern sequence which has timed out. For each
 * partial pattern sequence the provided {@link PatternFlatTimeoutFunction} is called. The
 * pattern timeout function can produce an arbitrary number of resulting elements.
 *
 * @param patternFlatTimeoutFunction The pattern flat timeout function which is called for each
 *                                   partial pattern sequence which has timed out.
 * @param patternFlatSelectFunction The pattern flat select function which is called for each
 *                                  detected pattern sequence.
 * @param <L> Type of the resulting timeout events
 * @param <R> Type of the resulting events
 *
 * @deprecated Use {@link PatternStream#flatSelect(OutputTag, PatternFlatTimeoutFunction, PatternFlatSelectFunction)}
 * that returns timed out events as a side-output
 *
 * @return {@link DataStream} which contains the resulting events from the pattern flat select
 * function or the resulting timeout events from the pattern flat timeout function wrapped in an
 * {@link Either} type.
 */
@Deprecated
public <L, R> SingleOutputStreamOperator<Either<L, R>> flatSelect(
		final PatternFlatTimeoutFunction<T, L> patternFlatTimeoutFunction,
		final PatternFlatSelectFunction<T, R> patternFlatSelectFunction) {

	final TypeInformation<L> timedOutTypeInfo = TypeExtractor.getUnaryOperatorReturnType(
		patternFlatTimeoutFunction,
		PatternFlatTimeoutFunction.class,
		0,
		1,
		new int[]{2, 0},
		builder.getInputType(),
		null,
		false);

	final TypeInformation<R> mainTypeInfo = TypeExtractor.getUnaryOperatorReturnType(
		patternFlatSelectFunction,
		PatternFlatSelectFunction.class,
		0,
		1,
		new int[]{1, 0},
		builder.getInputType(),
		null,
		false);

	final OutputTag<L> outputTag = new OutputTag<>(UUID.randomUUID().toString(), timedOutTypeInfo);

	final PatternProcessFunction<T, R> processFunction =
		fromFlatSelect(builder.clean(patternFlatSelectFunction))
			.withTimeoutHandler(outputTag, builder.clean(patternFlatTimeoutFunction))
			.build();

	final SingleOutputStreamOperator<R> mainStream = process(processFunction, mainTypeInfo);
	final DataStream<L> timedOutStream = mainStream.getSideOutput(outputTag);
	final TypeInformation<Either<L, R>> outTypeInfo = new EitherTypeInfo<>(timedOutTypeInfo, mainTypeInfo);

	return mainStream
			.connect(timedOutStream)
			.map(new CoMapTimeout<>())
			.returns(outTypeInfo);
}
 
Example 9
Source File: ItemTransactionJob.java    From flink-tutorials with Apache License 2.0 4 votes vote down vote up
public final StreamExecutionEnvironment createApplicationPipeline(ParameterTool params) throws Exception {

		// Create and configure the StreamExecutionEnvironment
		StreamExecutionEnvironment env = createExecutionEnvironment(params);

		// Read transaction stream
		DataStream<ItemTransaction> transactionStream = readTransactionStream(params, env);

		// We read the query stream and exclude it from watermark tracking by assigning Long.MAX_VALUE watermark
		DataStream<Query> queryStream = readQueryStream(params, env)
				.assignTimestampsAndWatermarks(new MaxWatermark<>())
				.name("MaxWatermark");

		// Connect transactions with queries using the same itemId key and apply our transaction processor
		// The main output is the transaction result, query results are accessed as a side output.
		SingleOutputStreamOperator<TransactionResult> processedTransactions = transactionStream.keyBy("itemId")
				.connect(queryStream.keyBy("itemId"))
				.process(new TransactionProcessor())
				.name("Transaction Processor")
				.uid("Transaction Processor");

		// Query results are accessed as a sideoutput of the transaction processor
		DataStream<QueryResult> queryResultStream = processedTransactions.getSideOutput(QUERY_RESULT);

		if (params.getBoolean(ENABLE_DB_ENRICHMENT, false)) {
			queryResultStream = AsyncDataStream.unorderedWait(
					queryResultStream,
					new ItemInfoEnrichment(params.getInt(ASYNC_TP_SIZE, 5), params.getRequired(DB_CONN_STRING)),
					10, TimeUnit.SECONDS
			);
		}

		// Handle the output of transaction and query results separately
		writeTransactionResults(params, processedTransactions);
		writeQueryOutput(params, queryResultStream);

		// If needed we create a window computation of the transaction summaries by item and time window
		if (params.getBoolean(ENABLE_SUMMARIES_KEY, false)) {
			DataStream<TransactionSummary> transactionSummaryStream = processedTransactions
					.keyBy("transaction.itemId")
					.timeWindow(Time.minutes(10))
					.aggregate(new TransactionSummaryAggregator())
					.name("Create Transaction Summary")
					.uid("Create Transaction Summary")
					.filter(new SummaryAlertingCondition(params))
					.name("Filter High failure rate");

			writeTransactionSummaries(params, transactionSummaryStream);
		}

		return env;
	}
 
Example 10
Source File: SideOutput.java    From blog_demos with Apache License 2.0 4 votes vote down vote up
public static void main(String[] args) throws Exception {
    final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();

    // 并行度为1
    env.setParallelism(1);

    // 定义OutputTag
    final OutputTag<String> outputTag = new OutputTag<String>("side-output"){};

    // 创建一个List,里面有两个Tuple2元素
    List<Tuple2<String, Integer>> list = new ArrayList<>();
    list.add(new Tuple2("aaa", 1));
    list.add(new Tuple2("bbb", 2));
    list.add(new Tuple2("ccc", 3));

    //通过List创建DataStream
    DataStream<Tuple2<String, Integer>> fromCollectionDataStream = env.fromCollection(list);

    //所有元素都进入mainDataStream,f1字段为奇数的元素进入SideOutput
    SingleOutputStreamOperator<String> mainDataStream = fromCollectionDataStream
            .process(new ProcessFunction<Tuple2<String, Integer>, String>() {
                @Override
                public void processElement(Tuple2<String, Integer> value, Context ctx, Collector<String> out) throws Exception {

                    //进入主流程的下一个算子
                    out.collect("main, name : " + value.f0 + ", value : " + value.f1);

                    //f1字段为奇数的元素进入SideOutput
                    if(1 == value.f1 % 2) {
                        ctx.output(outputTag, "side, name : " + value.f0 + ", value : " + value.f1);
                    }
                }
            });

    // 禁止chanin,这样可以在页面上看清楚原始的DAG
    mainDataStream.disableChaining();

    // 取得旁路数据
    DataStream<String> sideDataStream = mainDataStream.getSideOutput(outputTag);


    mainDataStream.print();
    sideDataStream.print();

    env.execute("processfunction demo : sideoutput");
}
 
Example 11
Source File: SideOutputITCase.java    From flink with Apache License 2.0 4 votes vote down vote up
@Test
public void testUnionOfTwoSideOutputs() throws Exception {
	TestListResultSink<Integer> evensResultSink = new TestListResultSink<>();
	TestListResultSink<Integer> oddsResultSink = new TestListResultSink<>();
	TestListResultSink<Integer> oddsUEvensResultSink = new TestListResultSink<>();
	TestListResultSink<Integer> evensUOddsResultSink = new TestListResultSink<>();
	TestListResultSink<Integer> oddsUOddsResultSink = new TestListResultSink<>();
	TestListResultSink<Integer> evensUEvensResultSink = new TestListResultSink<>();
	TestListResultSink<Integer> oddsUEvensExternalResultSink = new TestListResultSink<>();
	TestListResultSink<Integer> resultSink = new TestListResultSink<>();

	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
	env.setParallelism(3);

	DataStream<Integer> input = env.fromElements(1, 2, 3, 4);

	OutputTag<Integer> oddTag = new OutputTag<Integer>("odds"){};
	OutputTag<Integer> evenTag = new OutputTag<Integer>("even"){};

	SingleOutputStreamOperator<Integer> passThroughStream =
		input.process(new ProcessFunction<Integer, Integer>() {
			@Override
			public void processElement(Integer value, Context ctx, Collector<Integer> out) throws Exception {
				if (value % 2 != 0) {
					ctx.output(oddTag, value);
				}
				else {
					ctx.output(evenTag, value);
				}
				out.collect(value);
			}
		});

	DataStream<Integer> evens = passThroughStream.getSideOutput(evenTag);
	DataStream<Integer> odds = passThroughStream.getSideOutput(oddTag);

	evens.addSink(evensResultSink);
	odds.addSink(oddsResultSink);
	passThroughStream.addSink(resultSink);

	odds.union(evens).addSink(oddsUEvensResultSink);
	evens.union(odds).addSink(evensUOddsResultSink);

	odds.union(odds).addSink(oddsUOddsResultSink);
	evens.union(evens).addSink(evensUEvensResultSink);

	odds.union(env.fromElements(2, 4)).addSink(oddsUEvensExternalResultSink);

	env.execute();

	assertEquals(
		Arrays.asList(1, 3),
		oddsResultSink.getSortedResult());

	assertEquals(
		Arrays.asList(2, 4),
		evensResultSink.getSortedResult());

	assertEquals(
		Arrays.asList(1, 2, 3, 4),
		resultSink.getSortedResult());

	assertEquals(
		Arrays.asList(1, 2, 3, 4),
		oddsUEvensResultSink.getSortedResult());

	assertEquals(
		Arrays.asList(1, 2, 3, 4),
		evensUOddsResultSink.getSortedResult());

	assertEquals(
		Arrays.asList(1, 1, 3, 3),
		oddsUOddsResultSink.getSortedResult());

	assertEquals(
		Arrays.asList(2, 2, 4, 4),
		evensUEvensResultSink.getSortedResult());

	assertEquals(
		Arrays.asList(1, 2, 3, 4),
		oddsUEvensExternalResultSink.getSortedResult());
}
 
Example 12
Source File: PatternStream.java    From flink with Apache License 2.0 4 votes vote down vote up
/**
 * Applies a select function to the detected pattern sequence. For each pattern sequence the
 * provided {@link PatternSelectFunction} is called. The pattern select function can produce
 * exactly one resulting element.
 *
 * <p>Applies a timeout function to a partial pattern sequence which has timed out. For each
 * partial pattern sequence the provided {@link PatternTimeoutFunction} is called. The pattern
 * timeout function can produce exactly one resulting element.
 *
 * @param patternTimeoutFunction The pattern timeout function which is called for each partial
 *                               pattern sequence which has timed out.
 * @param patternSelectFunction The pattern select function which is called for each detected
 *                              pattern sequence.
 * @param <L> Type of the resulting timeout elements
 * @param <R> Type of the resulting elements
 *
 * @deprecated Use {@link PatternStream#select(OutputTag, PatternTimeoutFunction, PatternSelectFunction)}
 * that returns timed out events as a side-output
 *
 * @return {@link DataStream} which contains the resulting elements or the resulting timeout
 * elements wrapped in an {@link Either} type.
 */
@Deprecated
public <L, R> SingleOutputStreamOperator<Either<L, R>> select(
		final PatternTimeoutFunction<T, L> patternTimeoutFunction,
		final PatternSelectFunction<T, R> patternSelectFunction) {

	final TypeInformation<R> mainTypeInfo = TypeExtractor.getUnaryOperatorReturnType(
		patternSelectFunction,
		PatternSelectFunction.class,
		0,
		1,
		TypeExtractor.NO_INDEX,
		builder.getInputType(),
		null,
		false);

	final TypeInformation<L> timeoutTypeInfo = TypeExtractor.getUnaryOperatorReturnType(
		patternTimeoutFunction,
		PatternTimeoutFunction.class,
		0,
		1,
		TypeExtractor.NO_INDEX,
		builder.getInputType(),
		null,
		false);

	final TypeInformation<Either<L, R>> outTypeInfo = new EitherTypeInfo<>(timeoutTypeInfo, mainTypeInfo);

	final OutputTag<L> outputTag = new OutputTag<>(UUID.randomUUID().toString(), timeoutTypeInfo);

	final PatternProcessFunction<T, R> processFunction =
		fromSelect(builder.clean(patternSelectFunction))
			.withTimeoutHandler(outputTag, builder.clean(patternTimeoutFunction))
			.build();

	final SingleOutputStreamOperator<R> mainStream = process(processFunction, mainTypeInfo);
	final DataStream<L> timedOutStream = mainStream.getSideOutput(outputTag);

	return mainStream
		.connect(timedOutStream)
		.map(new CoMapTimeout<>())
		.returns(outTypeInfo);
}
 
Example 13
Source File: PatternStream.java    From flink with Apache License 2.0 4 votes vote down vote up
/**
 * Applies a flat select function to the detected pattern sequence. For each pattern sequence
 * the provided {@link PatternFlatSelectFunction} is called. The pattern flat select function
 * can produce an arbitrary number of resulting elements.
 *
 * <p>Applies a timeout function to a partial pattern sequence which has timed out. For each
 * partial pattern sequence the provided {@link PatternFlatTimeoutFunction} is called. The
 * pattern timeout function can produce an arbitrary number of resulting elements.
 *
 * @param patternFlatTimeoutFunction The pattern flat timeout function which is called for each
 *                                   partial pattern sequence which has timed out.
 * @param patternFlatSelectFunction The pattern flat select function which is called for each
 *                                  detected pattern sequence.
 * @param <L> Type of the resulting timeout events
 * @param <R> Type of the resulting events
 *
 * @deprecated Use {@link PatternStream#flatSelect(OutputTag, PatternFlatTimeoutFunction, PatternFlatSelectFunction)}
 * that returns timed out events as a side-output
 *
 * @return {@link DataStream} which contains the resulting events from the pattern flat select
 * function or the resulting timeout events from the pattern flat timeout function wrapped in an
 * {@link Either} type.
 */
@Deprecated
public <L, R> SingleOutputStreamOperator<Either<L, R>> flatSelect(
		final PatternFlatTimeoutFunction<T, L> patternFlatTimeoutFunction,
		final PatternFlatSelectFunction<T, R> patternFlatSelectFunction) {

	final TypeInformation<L> timedOutTypeInfo = TypeExtractor.getUnaryOperatorReturnType(
		patternFlatTimeoutFunction,
		PatternFlatTimeoutFunction.class,
		0,
		1,
		new int[]{2, 0},
		builder.getInputType(),
		null,
		false);

	final TypeInformation<R> mainTypeInfo = TypeExtractor.getUnaryOperatorReturnType(
		patternFlatSelectFunction,
		PatternFlatSelectFunction.class,
		0,
		1,
		new int[]{1, 0},
		builder.getInputType(),
		null,
		false);

	final OutputTag<L> outputTag = new OutputTag<>(UUID.randomUUID().toString(), timedOutTypeInfo);

	final PatternProcessFunction<T, R> processFunction =
		fromFlatSelect(builder.clean(patternFlatSelectFunction))
			.withTimeoutHandler(outputTag, builder.clean(patternFlatTimeoutFunction))
			.build();

	final SingleOutputStreamOperator<R> mainStream = process(processFunction, mainTypeInfo);
	final DataStream<L> timedOutStream = mainStream.getSideOutput(outputTag);
	final TypeInformation<Either<L, R>> outTypeInfo = new EitherTypeInfo<>(timedOutTypeInfo, mainTypeInfo);

	return mainStream
			.connect(timedOutStream)
			.map(new CoMapTimeout<>())
			.returns(outTypeInfo);
}