org.apache.flink.streaming.api.functions.co.CoProcessFunction Java Examples

/**
 * Returns an initialized test harness for {@link CoProcessFunction} with two input streams.
 *
 * @param function instance of a {@link CoProcessFunction} under test
 * @param <IN1> type of first input stream elements
 * @param <IN2> type of second input stream elements
 * @param <OUT> type of output stream elements
 * @return {@link TwoInputStreamOperatorTestHarness} wrapped around {@code function}
 */
public static <IN1, IN2, OUT>
TwoInputStreamOperatorTestHarness<IN1, IN2, OUT> forCoProcessFunction(
	final CoProcessFunction<IN1, IN2, OUT> function)
	throws Exception {

	TwoInputStreamOperatorTestHarness<IN1, IN2, OUT> testHarness =
		new TwoInputStreamOperatorTestHarness<>(
			new CoProcessOperator<>(
				Preconditions.checkNotNull(function)),
			1,
			1,
			0);
	testHarness.open();
	return testHarness;
}
 

/**
 * Applies the given {@link CoProcessFunction} on the connected input streams,
 * thereby creating a transformed output stream.
 *
 * <p>The function will be called for every element in the input streams and can produce zero or
 * more output elements. Contrary to the {@link #flatMap(CoFlatMapFunction)} function, this
 * function can also query the time and set timers. When reacting to the firing of set timers
 * the function can directly emit elements and/or register yet more timers.
 *
 * @param coProcessFunction The {@link CoProcessFunction} that is called for each element
 *                      in the stream.
 *
 * @param <R> The type of elements emitted by the {@code CoProcessFunction}.
 *
 * @return The transformed {@link DataStream}.
 */
@PublicEvolving
public <R> SingleOutputStreamOperator<R> process(
		CoProcessFunction<IN1, IN2, R> coProcessFunction) {

	TypeInformation<R> outTypeInfo = TypeExtractor.getBinaryOperatorReturnType(
		coProcessFunction,
		CoProcessFunction.class,
		0,
		1,
		2,
		TypeExtractor.NO_INDEX,
		getType1(),
		getType2(),
		Utils.getCallLocationName(),
		true);

	return process(coProcessFunction, outTypeInfo);
}
 

/**
 * Applies the given {@link CoProcessFunction} on the connected input streams,
 * thereby creating a transformed output stream.
 *
 * <p>The function will be called for every element in the input streams and can produce zero or
 * more output elements. Contrary to the {@link #flatMap(CoFlatMapFunction)} function, this
 * function can also query the time and set timers. When reacting to the firing of set timers
 * the function can directly emit elements and/or register yet more timers.
 *
 * @param coProcessFunction The {@link CoProcessFunction} that is called for each element
 *                      in the stream.
 *
 * @param <R> The type of elements emitted by the {@code CoProcessFunction}.
 *
 * @return The transformed {@link DataStream}.
 */
@PublicEvolving
public <R> SingleOutputStreamOperator<R> process(
		CoProcessFunction<IN1, IN2, R> coProcessFunction) {

	TypeInformation<R> outTypeInfo = TypeExtractor.getBinaryOperatorReturnType(
		coProcessFunction,
		CoProcessFunction.class,
		0,
		1,
		2,
		TypeExtractor.NO_INDEX,
		getType1(),
		getType2(),
		Utils.getCallLocationName(),
		true);

	return process(coProcessFunction, outTypeInfo);
}
 

/**
 * Applies the given {@link CoProcessFunction} on the connected input streams,
 * thereby creating a transformed output stream.
 *
 * <p>The function will be called for every element in the input streams and can produce zero or
 * more output elements. Contrary to the {@link #flatMap(CoFlatMapFunction)} function, this
 * function can also query the time and set timers. When reacting to the firing of set timers
 * the function can directly emit elements and/or register yet more timers.
 *
 * @param coProcessFunction The {@link CoProcessFunction} that is called for each element
 *                      in the stream.
 *
 * @param <R> The type of elements emitted by the {@code CoProcessFunction}.
 *
 * @return The transformed {@link DataStream}.
 */
@PublicEvolving
public <R> SingleOutputStreamOperator<R> process(
		CoProcessFunction<IN1, IN2, R> coProcessFunction) {

	TypeInformation<R> outTypeInfo = TypeExtractor.getBinaryOperatorReturnType(
		coProcessFunction,
		CoProcessFunction.class,
		0,
		1,
		2,
		TypeExtractor.NO_INDEX,
		getType1(),
		getType2(),
		Utils.getCallLocationName(),
		true);

	return process(coProcessFunction, outTypeInfo);
}
 

public KeyedCoProcessOperatorWithWatermarkDelay(CoProcessFunction<IN1, IN2, OUT> flatMapper, long watermarkDelay) {
	super(flatMapper);
	Preconditions.checkArgument(watermarkDelay >= 0, "The watermark delay should be non-negative.");
	if (watermarkDelay == 0) {
		// emits watermark without delay
		emitter = (Consumer<Watermark> & Serializable) (Watermark mark) -> output.emitWatermark(mark);
	} else {
		// emits watermark with delay
		emitter = (Consumer<Watermark> & Serializable) (Watermark mark) -> output
				.emitWatermark(new Watermark(mark.getTimestamp() - watermarkDelay));
	}
}
 

ContextImpl(CoProcessFunction<IN1, IN2, OUT> function, ProcessingTimeService timerService) {
	function.super();
	this.timerService = checkNotNull(timerService);
}
 

OnTimerContextImpl(CoProcessFunction<IN1, IN2, OUT> function, TimerService timerService) {
	function.super();
	this.timerService = checkNotNull(timerService);
}
 

ContextImpl(CoProcessFunction<IN1, IN2, OUT> function, TimerService timerService) {
	function.super();
	this.timerService = checkNotNull(timerService);
}
 

public LegacyKeyedCoProcessOperator(CoProcessFunction<IN1, IN2, OUT> flatMapper) {
	super(flatMapper);
}
 

ContextImpl(CoProcessFunction<IN1, IN2, OUT> function, ProcessingTimeService timerService) {
	function.super();
	this.timerService = checkNotNull(timerService);
}
 

public CoProcessOperator(CoProcessFunction<IN1, IN2, OUT> flatMapper) {
	super(flatMapper);
}
 

/**
 * Applies the given {@link CoProcessFunction} on the connected input streams,
 * thereby creating a transformed output stream.
 *
 * <p>The function will be called for every element in the input streams and can produce zero
 * or more output elements. Contrary to the {@link #flatMap(CoFlatMapFunction)} function,
 * this function can also query the time and set timers. When reacting to the firing of set
 * timers the function can directly emit elements and/or register yet more timers.
 *
 * @param coProcessFunction The {@link CoProcessFunction} that is called for each element
 *                      in the stream.
 *
 * @param <R> The type of elements emitted by the {@code CoProcessFunction}.
 *
 * @return The transformed {@link DataStream}.
 */
@Internal
public <R> SingleOutputStreamOperator<R> process(
		CoProcessFunction<IN1, IN2, R> coProcessFunction,
		TypeInformation<R> outputType) {

	TwoInputStreamOperator<IN1, IN2, R> operator;

	if ((inputStream1 instanceof KeyedStream) && (inputStream2 instanceof KeyedStream)) {
		operator = new LegacyKeyedCoProcessOperator<>(inputStream1.clean(coProcessFunction));
	} else {
		operator = new CoProcessOperator<>(inputStream1.clean(coProcessFunction));
	}

	return transform("Co-Process", outputType, operator);
}
 

OnTimerContextImpl(CoProcessFunction<IN1, IN2, OUT> function, TimerService timerService) {
	function.super();
	this.timerService = checkNotNull(timerService);
}
 

ContextImpl(CoProcessFunction<IN1, IN2, OUT> function, TimerService timerService) {
	function.super();
	this.timerService = checkNotNull(timerService);
}
 

public LegacyKeyedCoProcessOperator(CoProcessFunction<IN1, IN2, OUT> flatMapper) {
	super(flatMapper);
}
 

public CoProcessOperator(CoProcessFunction<IN1, IN2, OUT> flatMapper) {
	super(flatMapper);
}
 

/**
 * Applies the given {@link CoProcessFunction} on the connected input streams,
 * thereby creating a transformed output stream.
 *
 * <p>The function will be called for every element in the input streams and can produce zero
 * or more output elements. Contrary to the {@link #flatMap(CoFlatMapFunction)} function,
 * this function can also query the time and set timers. When reacting to the firing of set
 * timers the function can directly emit elements and/or register yet more timers.
 *
 * @param coProcessFunction The {@link CoProcessFunction} that is called for each element
 *                      in the stream.
 *
 * @param <R> The type of elements emitted by the {@code CoProcessFunction}.
 *
 * @return The transformed {@link DataStream}.
 */
@Internal
public <R> SingleOutputStreamOperator<R> process(
		CoProcessFunction<IN1, IN2, R> coProcessFunction,
		TypeInformation<R> outputType) {

	TwoInputStreamOperator<IN1, IN2, R> operator;

	if ((inputStream1 instanceof KeyedStream) && (inputStream2 instanceof KeyedStream)) {
		operator = new LegacyKeyedCoProcessOperator<>(inputStream1.clean(coProcessFunction));
	} else {
		operator = new CoProcessOperator<>(inputStream1.clean(coProcessFunction));
	}

	return transform("Co-Process", outputType, operator);
}
 

OnTimerContextImpl(CoProcessFunction<IN1, IN2, OUT> function, TimerService timerService) {
	function.super();
	this.timerService = checkNotNull(timerService);
}
 

ContextImpl(CoProcessFunction<IN1, IN2, OUT> function, TimerService timerService) {
	function.super();
	this.timerService = checkNotNull(timerService);
}
 

public KeyedCoProcessOperator(CoProcessFunction<IN1, IN2, OUT> coProcessFunction) {
	super(coProcessFunction);
}
 

ContextImpl(CoProcessFunction<IN1, IN2, OUT> function, ProcessingTimeService timerService) {
	function.super();
	this.timerService = checkNotNull(timerService);
}
 

public CoProcessOperator(CoProcessFunction<IN1, IN2, OUT> flatMapper) {
	super(flatMapper);
}
 

/**
 * Applies the given {@link CoProcessFunction} on the connected input streams,
 * thereby creating a transformed output stream.
 *
 * <p>The function will be called for every element in the input streams and can produce zero
 * or more output elements. Contrary to the {@link #flatMap(CoFlatMapFunction)} function,
 * this function can also query the time and set timers. When reacting to the firing of set
 * timers the function can directly emit elements and/or register yet more timers.
 *
 * @param coProcessFunction The {@link CoProcessFunction} that is called for each element
 *                      in the stream.
 *
 * @param <R> The type of elements emitted by the {@code CoProcessFunction}.
 *
 * @return The transformed {@link DataStream}.
 */
@Internal
public <R> SingleOutputStreamOperator<R> process(
		CoProcessFunction<IN1, IN2, R> coProcessFunction,
		TypeInformation<R> outputType) {

	TwoInputStreamOperator<IN1, IN2, R> operator;

	if ((inputStream1 instanceof KeyedStream) && (inputStream2 instanceof KeyedStream)) {
		operator = new KeyedCoProcessOperator<>(inputStream1.clean(coProcessFunction));
	} else {
		operator = new CoProcessOperator<>(inputStream1.clean(coProcessFunction));
	}

	return transform("Co-Process", outputType, operator);
}