org.apache.flink.streaming.api.operators.co.KeyedCoProcessOperator Java Examples

/**
 * Applies the given {@link KeyedCoProcessFunction} on the connected input streams,
 * thereby creating a transformed output stream.
 *
 * <p>The function will be called for every element in the input streams and can produce zero
 * or more output elements. Contrary to the {@link #flatMap(CoFlatMapFunction)} function,
 * this function can also query the time and set timers. When reacting to the firing of set
 * timers the function can directly emit elements and/or register yet more timers.
 *
 * @param keyedCoProcessFunction The {@link KeyedCoProcessFunction} that is called for each element
 *                      in the stream.
 *
 * @param <R> The type of elements emitted by the {@code CoProcessFunction}.
 *
 * @return The transformed {@link DataStream}.
 */
@Internal
public <K, R> SingleOutputStreamOperator<R> process(
	KeyedCoProcessFunction<K, IN1, IN2, R> keyedCoProcessFunction,
	TypeInformation<R> outputType) {

	TwoInputStreamOperator<IN1, IN2, R> operator;

	if ((inputStream1 instanceof KeyedStream) && (inputStream2 instanceof KeyedStream)) {
		operator = new KeyedCoProcessOperator<>(inputStream1.clean(keyedCoProcessFunction));
	} else {
		throw new UnsupportedOperationException("KeyedCoProcessFunction can only be used " +
			"when both input streams are of type KeyedStream.");
	}

	return transform("Co-Keyed-Process", outputType, operator);
}
 

private KeyedTwoInputStreamOperatorTestHarness<String, CrawlStateUrl, DomainScore, FetchUrl> makeTestHarness(
        int parallelism, int subTaskIndex, OperatorSubtaskState savedState) throws Exception {

    BaseUrlStateMerger merger = new DefaultUrlStateMerger();
    FetchQueue fetchQueue = new ReFetchingQueue();
    KeyedCoProcessOperator<String, CrawlStateUrl, DomainScore, FetchUrl> operator = new KeyedCoProcessOperator<>(
            new UrlDBFunction(_terminator, merger, fetchQueue));
    KeyedTwoInputStreamOperatorTestHarness<String, CrawlStateUrl, DomainScore, FetchUrl> result = new KeyedTwoInputStreamOperatorTestHarness<>(
            operator, _pldKeySelector, _domainScoreKeySelector, BasicTypeInfo.STRING_TYPE_INFO,
            MAX_PARALLELISM, parallelism, subTaskIndex);
    result.setStateBackend(new MemoryStateBackend());
    result.setup();
    if (savedState != null) {
        result.initializeState(savedState);
    }
    result.open();
    return result;
}
 

private KeyedTwoInputStreamOperatorTestHarness<Long, TaxiRide, TaxiFare, Tuple2<TaxiRide, TaxiFare>> setupHarness() throws Exception {
	// instantiate operator
	KeyedCoProcessOperator<Long, TaxiRide, TaxiFare, Tuple2<TaxiRide, TaxiFare>> operator =
			new KeyedCoProcessOperator<>(new ExpiringStateSolution.EnrichmentFunction());

	// setup test harness
	KeyedTwoInputStreamOperatorTestHarness<Long, TaxiRide, TaxiFare, Tuple2<TaxiRide, TaxiFare>> testHarness =
			new KeyedTwoInputStreamOperatorTestHarness<>(operator,
					(TaxiRide r) -> r.rideId,
					(TaxiFare f) -> f.rideId,
					BasicTypeInfo.LONG_TYPE_INFO);

	testHarness.setup();
	testHarness.open();

	return testHarness;
}
 

private TwoInputStreamOperatorTestHarness<Trade, Customer, EnrichedTrade> setupHarness() throws Exception {
	// instantiate operator
	KeyedCoProcessOperator<Long, Trade, Customer, EnrichedTrade> operator =
			new KeyedCoProcessOperator<>(new EventTimeJoinExercise.EventTimeJoinFunction());

	// setup test harness
	TwoInputStreamOperatorTestHarness<Trade, Customer, EnrichedTrade> testHarness =
			new KeyedTwoInputStreamOperatorTestHarness<>(operator,
					(Trade t) -> t.customerId,
					(Customer c) -> c.customerId,
					BasicTypeInfo.LONG_TYPE_INFO);

	testHarness.setup();
	testHarness.open();

	return testHarness;
}
 

/**
 * Applies the given {@link KeyedCoProcessFunction} on the connected input streams,
 * thereby creating a transformed output stream.
 *
 * <p>The function will be called for every element in the input streams and can produce zero
 * or more output elements. Contrary to the {@link #flatMap(CoFlatMapFunction)} function,
 * this function can also query the time and set timers. When reacting to the firing of set
 * timers the function can directly emit elements and/or register yet more timers.
 *
 * @param keyedCoProcessFunction The {@link KeyedCoProcessFunction} that is called for each element
 *                      in the stream.
 *
 * @param <R> The type of elements emitted by the {@code CoProcessFunction}.
 *
 * @return The transformed {@link DataStream}.
 */
@Internal
public <K, R> SingleOutputStreamOperator<R> process(
	KeyedCoProcessFunction<K, IN1, IN2, R> keyedCoProcessFunction,
	TypeInformation<R> outputType) {

	TwoInputStreamOperator<IN1, IN2, R> operator;

	if ((inputStream1 instanceof KeyedStream) && (inputStream2 instanceof KeyedStream)) {
		operator = new KeyedCoProcessOperator<>(inputStream1.clean(keyedCoProcessFunction));
	} else {
		throw new UnsupportedOperationException("KeyedCoProcessFunction can only be used " +
			"when both input streams are of type KeyedStream.");
	}

	return transform("Co-Keyed-Process", outputType, operator);
}
 

/**
 * Returns an initialized test harness for {@link KeyedCoProcessFunction} with two input streams.
 *
 * @param function instance of a {@link KeyedCoProcessFunction} under test
 * @param <K> key type
 * @param <IN1> type of first input stream elements
 * @param <IN2> type of second input stream elements
 * @param <OUT> type of output stream elements
 * @return {@link KeyedOneInputStreamOperatorTestHarness} wrapped around {@code function}
 */
public static <K, IN1, IN2, OUT>
KeyedTwoInputStreamOperatorTestHarness<K, IN1, IN2, OUT> forKeyedCoProcessFunction(
	final KeyedCoProcessFunction<K, IN1, IN2, OUT> function,
	final KeySelector<IN1, K> keySelector1,
	final KeySelector<IN2, K> keySelector2,
	final TypeInformation<K> keyType)
	throws Exception {

	KeyedTwoInputStreamOperatorTestHarness<K, IN1, IN2, OUT> testHarness =
		new KeyedTwoInputStreamOperatorTestHarness<>(
			new KeyedCoProcessOperator<>(Preconditions.checkNotNull(function)),
			keySelector1,
			keySelector2,
			keyType,
			1,
			1,
			0);

	testHarness.open();
	return testHarness;
}
 

private KeyedTwoInputStreamOperatorTestHarness<RowData, RowData, RowData, RowData> createTestHarness(
		RowTimeIntervalJoin intervalJoinFunc)
		throws Exception {
	KeyedCoProcessOperator<RowData, RowData, RowData, RowData> operator = new KeyedCoProcessOperatorWithWatermarkDelay<>(
			intervalJoinFunc, intervalJoinFunc.getMaxOutputDelay());
	KeyedTwoInputStreamOperatorTestHarness<RowData, RowData, RowData, RowData> testHarness =
			new KeyedTwoInputStreamOperatorTestHarness<>(operator, keySelector, keySelector, keyType);
	return testHarness;
}
 

private KeyedTwoInputStreamOperatorTestHarness<RowData, RowData, RowData, RowData> createTestHarness(
		ProcTimeIntervalJoin intervalJoinFunc)
		throws Exception {
	KeyedCoProcessOperator<RowData, RowData, RowData, RowData> operator = new KeyedCoProcessOperator<>(
			intervalJoinFunc);
	KeyedTwoInputStreamOperatorTestHarness<RowData, RowData, RowData, RowData> testHarness =
			new KeyedTwoInputStreamOperatorTestHarness<>(operator, keySelector, keySelector, keyType);
	return testHarness;
}
 

/**
 * Applies the given {@link CoProcessFunction} on the connected input streams,
 * thereby creating a transformed output stream.
 *
 * <p>The function will be called for every element in the input streams and can produce zero
 * or more output elements. Contrary to the {@link #flatMap(CoFlatMapFunction)} function,
 * this function can also query the time and set timers. When reacting to the firing of set
 * timers the function can directly emit elements and/or register yet more timers.
 *
 * @param coProcessFunction The {@link CoProcessFunction} that is called for each element
 *                      in the stream.
 *
 * @param <R> The type of elements emitted by the {@code CoProcessFunction}.
 *
 * @return The transformed {@link DataStream}.
 */
@Internal
public <R> SingleOutputStreamOperator<R> process(
		CoProcessFunction<IN1, IN2, R> coProcessFunction,
		TypeInformation<R> outputType) {

	TwoInputStreamOperator<IN1, IN2, R> operator;

	if ((inputStream1 instanceof KeyedStream) && (inputStream2 instanceof KeyedStream)) {
		operator = new KeyedCoProcessOperator<>(inputStream1.clean(coProcessFunction));
	} else {
		operator = new CoProcessOperator<>(inputStream1.clean(coProcessFunction));
	}

	return transform("Co-Process", outputType, operator);
}