org.apache.flink.streaming.api.functions.co.KeyedBroadcastProcessFunction Java Examples

/**
 * Returns an initialized test harness for {@link KeyedBroadcastProcessFunction}.
 *
 * @param function instance of a {@link KeyedBroadcastProcessFunction} under test
 * @param descriptors broadcast state descriptors used in the {@code function}
 * @param <K> key type
 * @param <IN1> type of input stream elements
 * @param <IN2> type of broadcast stream elements
 * @param <OUT> type of output elements
 * @return {@link BroadcastOperatorTestHarness} wrapped around {@code function}
 */
public static <K, IN1, IN2, OUT>
KeyedBroadcastOperatorTestHarness<K, IN1, IN2, OUT> forKeyedBroadcastProcessFunction(
	final KeyedBroadcastProcessFunction<K, IN1, IN2, OUT> function,
	final KeySelector<IN1, K> keySelector,
	final TypeInformation<K> keyType,
	final MapStateDescriptor<?, ?>... descriptors)
	throws Exception {

	KeyedBroadcastOperatorTestHarness<K, IN1, IN2, OUT> testHarness =
		new KeyedBroadcastOperatorTestHarness<>(
			new CoBroadcastWithKeyedOperator<>(
				Preconditions.checkNotNull(function), Arrays.asList(descriptors)),
			keySelector,
			keyType,
			1,
			1,
			0);

	testHarness.open();
	return testHarness;
}
 

private static <KEY, IN1, IN2, OUT> TwoInputStreamOperatorTestHarness<IN1, IN2, OUT> getInitializedTestHarness(
		final TypeInformation<KEY> keyTypeInfo,
		final KeySelector<IN1, KEY> keyKeySelector,
		final KeyedBroadcastProcessFunction<KEY, IN1, IN2, OUT> function,
		final int maxParallelism,
		final int numTasks,
		final int taskIdx) throws Exception {

	return getInitializedTestHarness(
			keyTypeInfo,
			keyKeySelector,
			function,
			maxParallelism,
			numTasks,
			taskIdx,
			null);
}
 

private static <KEY, IN1, IN2, OUT> TwoInputStreamOperatorTestHarness<IN1, IN2, OUT> getInitializedTestHarness(
		final TypeInformation<KEY> keyTypeInfo,
		final KeySelector<IN1, KEY> keyKeySelector,
		final KeyedBroadcastProcessFunction<KEY, IN1, IN2, OUT> function,
		final int maxParallelism,
		final int numTasks,
		final int taskIdx) throws Exception {

	return getInitializedTestHarness(
			keyTypeInfo,
			keyKeySelector,
			function,
			maxParallelism,
			numTasks,
			taskIdx,
			null);
}
 

/**
 * Assumes as inputs a {@link BroadcastStream} and a {@link KeyedStream} and applies the given
 * {@link KeyedBroadcastProcessFunction} on them, thereby creating a transformed output stream.
 *
 * @param function The {@link KeyedBroadcastProcessFunction} that is called for each element in the stream.
 * @param <KS> The type of the keys in the keyed stream.
 * @param <OUT> The type of the output elements.
 * @return The transformed {@link DataStream}.
 */
@PublicEvolving
public <KS, OUT> SingleOutputStreamOperator<OUT> process(final KeyedBroadcastProcessFunction<KS, IN1, IN2, OUT> function) {

	TypeInformation<OUT> outTypeInfo = TypeExtractor.getBinaryOperatorReturnType(
			function,
			KeyedBroadcastProcessFunction.class,
			1,
			2,
			3,
			TypeExtractor.NO_INDEX,
			getType1(),
			getType2(),
			Utils.getCallLocationName(),
			true);

	return process(function, outTypeInfo);
}
 

/**
 * Assumes as inputs a {@link BroadcastStream} and a {@link KeyedStream} and applies the given
 * {@link KeyedBroadcastProcessFunction} on them, thereby creating a transformed output stream.
 *
 * @param function The {@link KeyedBroadcastProcessFunction} that is called for each element in the stream.
 * @param <KS> The type of the keys in the keyed stream.
 * @param <OUT> The type of the output elements.
 * @return The transformed {@link DataStream}.
 */
@PublicEvolving
public <KS, OUT> SingleOutputStreamOperator<OUT> process(final KeyedBroadcastProcessFunction<KS, IN1, IN2, OUT> function) {

	TypeInformation<OUT> outTypeInfo = TypeExtractor.getBinaryOperatorReturnType(
			function,
			KeyedBroadcastProcessFunction.class,
			1,
			2,
			3,
			TypeExtractor.NO_INDEX,
			getType1(),
			getType2(),
			Utils.getCallLocationName(),
			true);

	return process(function, outTypeInfo);
}
 

private static <KEY, IN1, IN2, OUT> TwoInputStreamOperatorTestHarness<IN1, IN2, OUT> getInitializedTestHarness(
		final TypeInformation<KEY> keyTypeInfo,
		final KeySelector<IN1, KEY> keyKeySelector,
		final KeyedBroadcastProcessFunction<KEY, IN1, IN2, OUT> function,
		final int maxParallelism,
		final int numTasks,
		final int taskIdx) throws Exception {

	return getInitializedTestHarness(
			keyTypeInfo,
			keyKeySelector,
			function,
			maxParallelism,
			numTasks,
			taskIdx,
			null);
}
 

/**
 * Assumes as inputs a {@link BroadcastStream} and a {@link KeyedStream} and applies the given
 * {@link KeyedBroadcastProcessFunction} on them, thereby creating a transformed output stream.
 *
 * @param function The {@link KeyedBroadcastProcessFunction} that is called for each element in the stream.
 * @param <KS> The type of the keys in the keyed stream.
 * @param <OUT> The type of the output elements.
 * @return The transformed {@link DataStream}.
 */
@PublicEvolving
public <KS, OUT> SingleOutputStreamOperator<OUT> process(final KeyedBroadcastProcessFunction<KS, IN1, IN2, OUT> function) {

	TypeInformation<OUT> outTypeInfo = TypeExtractor.getBinaryOperatorReturnType(
			function,
			KeyedBroadcastProcessFunction.class,
			1,
			2,
			3,
			TypeExtractor.NO_INDEX,
			getType1(),
			getType2(),
			Utils.getCallLocationName(),
			true);

	return process(function, outTypeInfo);
}
 

OnTimerContextImpl(
		final ExecutionConfig executionConfig,
		final KeyedBroadcastProcessFunction<KS, IN1, IN2, OUT> function,
		final Map<MapStateDescriptor<?, ?>, BroadcastState<?, ?>> broadcastStates,
		final TimerService timerService) {

	function.super();
	this.config = Preconditions.checkNotNull(executionConfig);
	this.states = Preconditions.checkNotNull(broadcastStates);
	this.timerService = Preconditions.checkNotNull(timerService);
}
 

private static <KEY, IN1, IN2, OUT> TwoInputStreamOperatorTestHarness<IN1, IN2, OUT> getInitializedTestHarness(
		final TypeInformation<KEY> keyTypeInfo,
		final KeySelector<IN1, KEY> keyKeySelector,
		final KeyedBroadcastProcessFunction<KEY, IN1, IN2, OUT> function,
		final int maxParallelism,
		final int numTasks,
		final int taskIdx,
		final OperatorSubtaskState initState) throws Exception {

	final TwoInputStreamOperatorTestHarness<IN1, IN2, OUT>  testHarness =
			new KeyedTwoInputStreamOperatorTestHarness<>(
					new CoBroadcastWithKeyedOperator<>(
							Preconditions.checkNotNull(function),
							Collections.singletonList(STATE_DESCRIPTOR)),
					keyKeySelector,
					null,
					keyTypeInfo,
					maxParallelism,
					numTasks,
					taskIdx
			);

	testHarness.setup();
	testHarness.initializeState(initState);
	testHarness.open();

	return testHarness;
}
 

private static <KEY, IN1, IN2, OUT> TwoInputStreamOperatorTestHarness<IN1, IN2, OUT> getInitializedTestHarness(
		final TypeInformation<KEY> keyTypeInfo,
		final KeySelector<IN1, KEY> keyKeySelector,
		final KeyedBroadcastProcessFunction<KEY, IN1, IN2, OUT> function) throws Exception {

	return getInitializedTestHarness(
			keyTypeInfo,
			keyKeySelector,
			function,
			1,
			1,
			0);
}
 

OnTimerContextImpl(
		final ExecutionConfig executionConfig,
		final KeyedBroadcastProcessFunction<KS, IN1, IN2, OUT> function,
		final Map<MapStateDescriptor<?, ?>, BroadcastState<?, ?>> broadcastStates,
		final TimerService timerService) {

	function.super();
	this.config = Preconditions.checkNotNull(executionConfig);
	this.states = Preconditions.checkNotNull(broadcastStates);
	this.timerService = Preconditions.checkNotNull(timerService);
}
 

ReadOnlyContextImpl(
		final ExecutionConfig executionConfig,
		final KeyedBroadcastProcessFunction<KS, IN1, IN2, OUT> function,
		final Map<MapStateDescriptor<?, ?>, BroadcastState<?, ?>> broadcastStates,
		final TimerService timerService) {

	function.super();
	this.config = Preconditions.checkNotNull(executionConfig);
	this.states = Preconditions.checkNotNull(broadcastStates);
	this.timerService = Preconditions.checkNotNull(timerService);
}
 

ReadWriteContextImpl (
		final ExecutionConfig executionConfig,
		final KeyedStateBackend<KS> keyedStateBackend,
		final KeyedBroadcastProcessFunction<KS, IN1, IN2, OUT> function,
		final Map<MapStateDescriptor<?, ?>, BroadcastState<?, ?>> broadcastStates,
		final TimerService timerService) {

	function.super();
	this.config = Preconditions.checkNotNull(executionConfig);
	this.keyedStateBackend = Preconditions.checkNotNull(keyedStateBackend);
	this.states = Preconditions.checkNotNull(broadcastStates);
	this.timerService = Preconditions.checkNotNull(timerService);
}
 

/**
 * Assumes as inputs a {@link BroadcastStream} and a {@link KeyedStream} and applies the given
 * {@link KeyedBroadcastProcessFunction} on them, thereby creating a transformed output stream.
 *
 * @param function The {@link KeyedBroadcastProcessFunction} that is called for each element in the stream.
 * @param outTypeInfo The type of the output elements.
 * @param <KS> The type of the keys in the keyed stream.
 * @param <OUT> The type of the output elements.
 * @return The transformed {@link DataStream}.
 */
@PublicEvolving
public <KS, OUT> SingleOutputStreamOperator<OUT> process(
		final KeyedBroadcastProcessFunction<KS, IN1, IN2, OUT> function,
		final TypeInformation<OUT> outTypeInfo) {

	Preconditions.checkNotNull(function);
	Preconditions.checkArgument(inputStream1 instanceof KeyedStream,
			"A KeyedBroadcastProcessFunction can only be used on a keyed stream.");

	TwoInputStreamOperator<IN1, IN2, OUT> operator =
			new CoBroadcastWithKeyedOperator<>(clean(function), broadcastStateDescriptors);
	return transform("Co-Process-Broadcast-Keyed", outTypeInfo, operator);
}
 

private static <KEY, IN1, IN2, OUT> TwoInputStreamOperatorTestHarness<IN1, IN2, OUT> getInitializedTestHarness(
		final TypeInformation<KEY> keyTypeInfo,
		final KeySelector<IN1, KEY> keyKeySelector,
		final KeyedBroadcastProcessFunction<KEY, IN1, IN2, OUT> function,
		final int maxParallelism,
		final int numTasks,
		final int taskIdx,
		final OperatorSubtaskState initState) throws Exception {

	final TwoInputStreamOperatorTestHarness<IN1, IN2, OUT>  testHarness =
			new KeyedTwoInputStreamOperatorTestHarness<>(
					new CoBroadcastWithKeyedOperator<>(
							Preconditions.checkNotNull(function),
							Collections.singletonList(STATE_DESCRIPTOR)),
					keyKeySelector,
					null,
					keyTypeInfo,
					maxParallelism,
					numTasks,
					taskIdx
			);

	testHarness.setup();
	testHarness.initializeState(initState);
	testHarness.open();

	return testHarness;
}
 

private static <KEY, IN1, IN2, OUT> TwoInputStreamOperatorTestHarness<IN1, IN2, OUT> getInitializedTestHarness(
		final TypeInformation<KEY> keyTypeInfo,
		final KeySelector<IN1, KEY> keyKeySelector,
		final KeyedBroadcastProcessFunction<KEY, IN1, IN2, OUT> function) throws Exception {

	return getInitializedTestHarness(
			keyTypeInfo,
			keyKeySelector,
			function,
			1,
			1,
			0);
}
 

ReadOnlyContextImpl(
		final ExecutionConfig executionConfig,
		final KeyedBroadcastProcessFunction<KS, IN1, IN2, OUT> function,
		final Map<MapStateDescriptor<?, ?>, BroadcastState<?, ?>> broadcastStates,
		final TimerService timerService) {

	function.super();
	this.config = Preconditions.checkNotNull(executionConfig);
	this.states = Preconditions.checkNotNull(broadcastStates);
	this.timerService = Preconditions.checkNotNull(timerService);
}
 

ReadWriteContextImpl (
		final ExecutionConfig executionConfig,
		final KeyedStateBackend<KS> keyedStateBackend,
		final KeyedBroadcastProcessFunction<KS, IN1, IN2, OUT> function,
		final Map<MapStateDescriptor<?, ?>, BroadcastState<?, ?>> broadcastStates,
		final TimerService timerService) {

	function.super();
	this.config = Preconditions.checkNotNull(executionConfig);
	this.keyedStateBackend = Preconditions.checkNotNull(keyedStateBackend);
	this.states = Preconditions.checkNotNull(broadcastStates);
	this.timerService = Preconditions.checkNotNull(timerService);
}
 

/**
 * Assumes as inputs a {@link BroadcastStream} and a {@link KeyedStream} and applies the given
 * {@link KeyedBroadcastProcessFunction} on them, thereby creating a transformed output stream.
 *
 * @param function The {@link KeyedBroadcastProcessFunction} that is called for each element in the stream.
 * @param outTypeInfo The type of the output elements.
 * @param <KS> The type of the keys in the keyed stream.
 * @param <OUT> The type of the output elements.
 * @return The transformed {@link DataStream}.
 */
@PublicEvolving
public <KS, OUT> SingleOutputStreamOperator<OUT> process(
		final KeyedBroadcastProcessFunction<KS, IN1, IN2, OUT> function,
		final TypeInformation<OUT> outTypeInfo) {

	Preconditions.checkNotNull(function);
	Preconditions.checkArgument(inputStream1 instanceof KeyedStream,
			"A KeyedBroadcastProcessFunction can only be used on a keyed stream.");

	TwoInputStreamOperator<IN1, IN2, OUT> operator =
			new CoBroadcastWithKeyedOperator<>(clean(function), broadcastStateDescriptors);
	return transform("Co-Process-Broadcast-Keyed", outTypeInfo, operator);
}
 

private static <KEY, IN1, IN2, OUT> TwoInputStreamOperatorTestHarness<IN1, IN2, OUT> getInitializedTestHarness(
		final TypeInformation<KEY> keyTypeInfo,
		final KeySelector<IN1, KEY> keyKeySelector,
		final KeyedBroadcastProcessFunction<KEY, IN1, IN2, OUT> function,
		final int maxParallelism,
		final int numTasks,
		final int taskIdx,
		final OperatorSubtaskState initState) throws Exception {

	final TwoInputStreamOperatorTestHarness<IN1, IN2, OUT>  testHarness =
			new KeyedTwoInputStreamOperatorTestHarness<>(
					new CoBroadcastWithKeyedOperator<>(
							Preconditions.checkNotNull(function),
							Collections.singletonList(STATE_DESCRIPTOR)),
					keyKeySelector,
					null,
					keyTypeInfo,
					maxParallelism,
					numTasks,
					taskIdx
			);

	testHarness.setup();
	testHarness.initializeState(initState);
	testHarness.open();

	return testHarness;
}
 

private static <KEY, IN1, IN2, OUT> TwoInputStreamOperatorTestHarness<IN1, IN2, OUT> getInitializedTestHarness(
		final TypeInformation<KEY> keyTypeInfo,
		final KeySelector<IN1, KEY> keyKeySelector,
		final KeyedBroadcastProcessFunction<KEY, IN1, IN2, OUT> function) throws Exception {

	return getInitializedTestHarness(
			keyTypeInfo,
			keyKeySelector,
			function,
			1,
			1,
			0);
}
 

OnTimerContextImpl(
		final ExecutionConfig executionConfig,
		final KeyedBroadcastProcessFunction<KS, IN1, IN2, OUT> function,
		final Map<MapStateDescriptor<?, ?>, BroadcastState<?, ?>> broadcastStates,
		final TimerService timerService) {

	function.super();
	this.config = Preconditions.checkNotNull(executionConfig);
	this.states = Preconditions.checkNotNull(broadcastStates);
	this.timerService = Preconditions.checkNotNull(timerService);
}
 

ReadOnlyContextImpl(
		final ExecutionConfig executionConfig,
		final KeyedBroadcastProcessFunction<KS, IN1, IN2, OUT> function,
		final Map<MapStateDescriptor<?, ?>, BroadcastState<?, ?>> broadcastStates,
		final TimerService timerService) {

	function.super();
	this.config = Preconditions.checkNotNull(executionConfig);
	this.states = Preconditions.checkNotNull(broadcastStates);
	this.timerService = Preconditions.checkNotNull(timerService);
}
 

ReadWriteContextImpl (
		final ExecutionConfig executionConfig,
		final KeyedStateBackend<KS> keyedStateBackend,
		final KeyedBroadcastProcessFunction<KS, IN1, IN2, OUT> function,
		final Map<MapStateDescriptor<?, ?>, BroadcastState<?, ?>> broadcastStates,
		final TimerService timerService) {

	function.super();
	this.config = Preconditions.checkNotNull(executionConfig);
	this.keyedStateBackend = Preconditions.checkNotNull(keyedStateBackend);
	this.states = Preconditions.checkNotNull(broadcastStates);
	this.timerService = Preconditions.checkNotNull(timerService);
}
 

/**
 * Assumes as inputs a {@link BroadcastStream} and a {@link KeyedStream} and applies the given
 * {@link KeyedBroadcastProcessFunction} on them, thereby creating a transformed output stream.
 *
 * @param function The {@link KeyedBroadcastProcessFunction} that is called for each element in the stream.
 * @param outTypeInfo The type of the output elements.
 * @param <KS> The type of the keys in the keyed stream.
 * @param <OUT> The type of the output elements.
 * @return The transformed {@link DataStream}.
 */
@PublicEvolving
public <KS, OUT> SingleOutputStreamOperator<OUT> process(
		final KeyedBroadcastProcessFunction<KS, IN1, IN2, OUT> function,
		final TypeInformation<OUT> outTypeInfo) {

	Preconditions.checkNotNull(function);
	Preconditions.checkArgument(inputStream1 instanceof KeyedStream,
			"A KeyedBroadcastProcessFunction can only be used on a keyed stream.");

	TwoInputStreamOperator<IN1, IN2, OUT> operator =
			new CoBroadcastWithKeyedOperator<>(clean(function), broadcastStateDescriptors);
	return transform("Co-Process-Broadcast-Keyed", outTypeInfo, operator);
}
 

public CoBroadcastWithKeyedOperator(
		final KeyedBroadcastProcessFunction<KS, IN1, IN2, OUT> function,
		final List<MapStateDescriptor<?, ?>> broadcastStateDescriptors) {
	super(function);
	this.broadcastStateDescriptors = Preconditions.checkNotNull(broadcastStateDescriptors);
}
 

public CoBroadcastWithKeyedOperator(
		final KeyedBroadcastProcessFunction<KS, IN1, IN2, OUT> function,
		final List<MapStateDescriptor<?, ?>> broadcastStateDescriptors) {
	super(function);
	this.broadcastStateDescriptors = Preconditions.checkNotNull(broadcastStateDescriptors);
}
 

public CoBroadcastWithKeyedOperator(
		final KeyedBroadcastProcessFunction<KS, IN1, IN2, OUT> function,
		final List<MapStateDescriptor<?, ?>> broadcastStateDescriptors) {
	super(function);
	this.broadcastStateDescriptors = Preconditions.checkNotNull(broadcastStateDescriptors);
}