org.apache.flink.streaming.api.functions.co.BroadcastProcessFunction Java Examples

/**
 * Returns an initialized test harness for {@link BroadcastProcessFunction}.
 *
 * @param function instance of a {@link BroadcastProcessFunction} under test
 * @param descriptors broadcast state descriptors used in the {@code function}
 * @param <IN1> type of input stream elements
 * @param <IN2> type of broadcast stream elements
 * @param <OUT> type of output elements
 * @return {@link BroadcastOperatorTestHarness} wrapped around {@code function}
 */
public static <IN1, IN2, OUT>
BroadcastOperatorTestHarness<IN1, IN2, OUT> forBroadcastProcessFunction(
	final BroadcastProcessFunction<IN1, IN2, OUT> function,
	final MapStateDescriptor<?, ?>... descriptors)
	throws Exception {

	BroadcastOperatorTestHarness<IN1, IN2, OUT> testHarness =
		new BroadcastOperatorTestHarness<>(
			new CoBroadcastWithNonKeyedOperator<>(
				Preconditions.checkNotNull(function), Arrays.asList(descriptors)),
			1,
			1,
			0);
	testHarness.open();
	return testHarness;
}
 

private static <IN1, IN2, OUT> TwoInputStreamOperatorTestHarness<IN1, IN2, OUT> getInitializedTestHarness(
		final BroadcastProcessFunction<IN1, IN2, OUT> function,
		final int maxParallelism,
		final int numTasks,
		final int taskIdx,
		final OperatorSubtaskState initState,
		final MapStateDescriptor<?, ?>... descriptors) throws Exception {

	TwoInputStreamOperatorTestHarness<IN1, IN2, OUT> testHarness = new TwoInputStreamOperatorTestHarness<>(
			new CoBroadcastWithNonKeyedOperator<>(
					Preconditions.checkNotNull(function),
					Arrays.asList(descriptors)),
			maxParallelism, numTasks, taskIdx
	);
	testHarness.setup();
	testHarness.initializeState(initState);
	testHarness.open();

	return testHarness;
}
 

private static <IN1, IN2, OUT> TwoInputStreamOperatorTestHarness<IN1, IN2, OUT> getInitializedTestHarness(
		final BroadcastProcessFunction<IN1, IN2, OUT> function,
		final int maxParallelism,
		final int numTasks,
		final int taskIdx,
		final OperatorSubtaskState initState,
		final MapStateDescriptor<?, ?>... descriptors) throws Exception {

	TwoInputStreamOperatorTestHarness<IN1, IN2, OUT> testHarness = new TwoInputStreamOperatorTestHarness<>(
			new CoBroadcastWithNonKeyedOperator<>(
					Preconditions.checkNotNull(function),
					Arrays.asList(descriptors)),
			maxParallelism, numTasks, taskIdx
	);
	testHarness.setup();
	testHarness.initializeState(initState);
	testHarness.open();

	return testHarness;
}
 

/**
 * Assumes as inputs a {@link BroadcastStream} and a non-keyed {@link DataStream} and applies the given
 * {@link BroadcastProcessFunction} on them, thereby creating a transformed output stream.
 *
 * @param function The {@link BroadcastProcessFunction} that is called for each element in the stream.
 * @param <OUT> The type of the output elements.
 * @return The transformed {@link DataStream}.
 */
@PublicEvolving
public <OUT> SingleOutputStreamOperator<OUT> process(final BroadcastProcessFunction<IN1, IN2, OUT> function) {

	TypeInformation<OUT> outTypeInfo = TypeExtractor.getBinaryOperatorReturnType(
			function,
			BroadcastProcessFunction.class,
			0,
			1,
			2,
			TypeExtractor.NO_INDEX,
			getType1(),
			getType2(),
			Utils.getCallLocationName(),
			true);

	return process(function, outTypeInfo);
}
 

/**
 * Assumes as inputs a {@link BroadcastStream} and a non-keyed {@link DataStream} and applies the given
 * {@link BroadcastProcessFunction} on them, thereby creating a transformed output stream.
 *
 * @param function The {@link BroadcastProcessFunction} that is called for each element in the stream.
 * @param <OUT> The type of the output elements.
 * @return The transformed {@link DataStream}.
 */
@PublicEvolving
public <OUT> SingleOutputStreamOperator<OUT> process(final BroadcastProcessFunction<IN1, IN2, OUT> function) {

	TypeInformation<OUT> outTypeInfo = TypeExtractor.getBinaryOperatorReturnType(
			function,
			BroadcastProcessFunction.class,
			0,
			1,
			2,
			TypeExtractor.NO_INDEX,
			getType1(),
			getType2(),
			Utils.getCallLocationName(),
			true);

	return process(function, outTypeInfo);
}
 

private static <IN1, IN2, OUT> TwoInputStreamOperatorTestHarness<IN1, IN2, OUT> getInitializedTestHarness(
		final BroadcastProcessFunction<IN1, IN2, OUT> function,
		final int maxParallelism,
		final int numTasks,
		final int taskIdx,
		final OperatorSubtaskState initState,
		final MapStateDescriptor<?, ?>... descriptors) throws Exception {

	TwoInputStreamOperatorTestHarness<IN1, IN2, OUT> testHarness = new TwoInputStreamOperatorTestHarness<>(
			new CoBroadcastWithNonKeyedOperator<>(
					Preconditions.checkNotNull(function),
					Arrays.asList(descriptors)),
			maxParallelism, numTasks, taskIdx
	);
	testHarness.setup();
	testHarness.initializeState(initState);
	testHarness.open();

	return testHarness;
}
 

/**
 * Assumes as inputs a {@link BroadcastStream} and a non-keyed {@link DataStream} and applies the given
 * {@link BroadcastProcessFunction} on them, thereby creating a transformed output stream.
 *
 * @param function The {@link BroadcastProcessFunction} that is called for each element in the stream.
 * @param <OUT> The type of the output elements.
 * @return The transformed {@link DataStream}.
 */
@PublicEvolving
public <OUT> SingleOutputStreamOperator<OUT> process(final BroadcastProcessFunction<IN1, IN2, OUT> function) {

	TypeInformation<OUT> outTypeInfo = TypeExtractor.getBinaryOperatorReturnType(
			function,
			BroadcastProcessFunction.class,
			0,
			1,
			2,
			TypeExtractor.NO_INDEX,
			getType1(),
			getType2(),
			Utils.getCallLocationName(),
			true);

	return process(function, outTypeInfo);
}
 

ReadOnlyContextImpl(
		final ExecutionConfig executionConfig,
		final BroadcastProcessFunction<IN1, IN2, OUT> function,
		final Map<MapStateDescriptor<?, ?>, BroadcastState<?, ?>> broadcastStates,
		final ProcessingTimeService timerService) {

	function.super();
	this.config = Preconditions.checkNotNull(executionConfig);
	this.states = Preconditions.checkNotNull(broadcastStates);
	this.timerService = Preconditions.checkNotNull(timerService);
}
 

private static <IN1, IN2, OUT> TwoInputStreamOperatorTestHarness<IN1, IN2, OUT> getInitializedTestHarness(
		final BroadcastProcessFunction<IN1, IN2, OUT> function,
		final int maxParallelism,
		final int numTasks,
		final int taskIdx,
		final MapStateDescriptor<?, ?>... descriptors) throws Exception {

	return getInitializedTestHarness(
			function,
			maxParallelism,
			numTasks,
			taskIdx,
			null,
			descriptors);
}
 

private static <IN1, IN2, OUT> TwoInputStreamOperatorTestHarness<IN1, IN2, OUT> getInitializedTestHarness(
		final BroadcastProcessFunction<IN1, IN2, OUT> function,
		final MapStateDescriptor<?, ?>... descriptors) throws Exception {

	return getInitializedTestHarness(
			function,
			1,
			1,
			0,
			descriptors);
}
 

ReadOnlyContextImpl(
		final ExecutionConfig executionConfig,
		final BroadcastProcessFunction<IN1, IN2, OUT> function,
		final Map<MapStateDescriptor<?, ?>, BroadcastState<?, ?>> broadcastStates,
		final ProcessingTimeService timerService) {

	function.super();
	this.config = Preconditions.checkNotNull(executionConfig);
	this.states = Preconditions.checkNotNull(broadcastStates);
	this.timerService = Preconditions.checkNotNull(timerService);
}
 

ReadWriteContextImpl(
		final ExecutionConfig executionConfig,
		final BroadcastProcessFunction<IN1, IN2, OUT> function,
		final Map<MapStateDescriptor<?, ?>, BroadcastState<?, ?>> broadcastStates,
		final ProcessingTimeService timerService) {

	function.super();
	this.config = Preconditions.checkNotNull(executionConfig);
	this.states = Preconditions.checkNotNull(broadcastStates);
	this.timerService = Preconditions.checkNotNull(timerService);
}
 

/**
 * Assumes as inputs a {@link BroadcastStream} and a non-keyed {@link DataStream} and applies the given
 * {@link BroadcastProcessFunction} on them, thereby creating a transformed output stream.
 *
 * @param function The {@link BroadcastProcessFunction} that is called for each element in the stream.
 * @param outTypeInfo The type of the output elements.
 * @param <OUT> The type of the output elements.
 * @return The transformed {@link DataStream}.
 */
@PublicEvolving
public <OUT> SingleOutputStreamOperator<OUT> process(
		final BroadcastProcessFunction<IN1, IN2, OUT> function,
		final TypeInformation<OUT> outTypeInfo) {

	Preconditions.checkNotNull(function);
	Preconditions.checkArgument(!(inputStream1 instanceof KeyedStream),
			"A BroadcastProcessFunction can only be used on a non-keyed stream.");

	TwoInputStreamOperator<IN1, IN2, OUT> operator =
			new CoBroadcastWithNonKeyedOperator<>(clean(function), broadcastStateDescriptors);
	return transform("Co-Process-Broadcast", outTypeInfo, operator);
}
 

private static <IN1, IN2, OUT> TwoInputStreamOperatorTestHarness<IN1, IN2, OUT> getInitializedTestHarness(
		final BroadcastProcessFunction<IN1, IN2, OUT> function,
		final int maxParallelism,
		final int numTasks,
		final int taskIdx,
		final MapStateDescriptor<?, ?>... descriptors) throws Exception {

	return getInitializedTestHarness(
			function,
			maxParallelism,
			numTasks,
			taskIdx,
			null,
			descriptors);
}
 

private static <IN1, IN2, OUT> TwoInputStreamOperatorTestHarness<IN1, IN2, OUT> getInitializedTestHarness(
		final BroadcastProcessFunction<IN1, IN2, OUT> function,
		final MapStateDescriptor<?, ?>... descriptors) throws Exception {

	return getInitializedTestHarness(
			function,
			1,
			1,
			0,
			descriptors);
}
 

ReadWriteContextImpl(
		final ExecutionConfig executionConfig,
		final BroadcastProcessFunction<IN1, IN2, OUT> function,
		final Map<MapStateDescriptor<?, ?>, BroadcastState<?, ?>> broadcastStates,
		final ProcessingTimeService timerService) {

	function.super();
	this.config = Preconditions.checkNotNull(executionConfig);
	this.states = Preconditions.checkNotNull(broadcastStates);
	this.timerService = Preconditions.checkNotNull(timerService);
}
 

/**
 * Assumes as inputs a {@link BroadcastStream} and a non-keyed {@link DataStream} and applies the given
 * {@link BroadcastProcessFunction} on them, thereby creating a transformed output stream.
 *
 * @param function The {@link BroadcastProcessFunction} that is called for each element in the stream.
 * @param outTypeInfo The type of the output elements.
 * @param <OUT> The type of the output elements.
 * @return The transformed {@link DataStream}.
 */
@PublicEvolving
public <OUT> SingleOutputStreamOperator<OUT> process(
		final BroadcastProcessFunction<IN1, IN2, OUT> function,
		final TypeInformation<OUT> outTypeInfo) {

	Preconditions.checkNotNull(function);
	Preconditions.checkArgument(!(inputStream1 instanceof KeyedStream),
			"A BroadcastProcessFunction can only be used on a non-keyed stream.");

	TwoInputStreamOperator<IN1, IN2, OUT> operator =
			new CoBroadcastWithNonKeyedOperator<>(clean(function), broadcastStateDescriptors);
	return transform("Co-Process-Broadcast", outTypeInfo, operator);
}
 

private static <IN1, IN2, OUT> TwoInputStreamOperatorTestHarness<IN1, IN2, OUT> getInitializedTestHarness(
		final BroadcastProcessFunction<IN1, IN2, OUT> function,
		final int maxParallelism,
		final int numTasks,
		final int taskIdx,
		final MapStateDescriptor<?, ?>... descriptors) throws Exception {

	return getInitializedTestHarness(
			function,
			maxParallelism,
			numTasks,
			taskIdx,
			null,
			descriptors);
}
 

private static <IN1, IN2, OUT> TwoInputStreamOperatorTestHarness<IN1, IN2, OUT> getInitializedTestHarness(
		final BroadcastProcessFunction<IN1, IN2, OUT> function,
		final MapStateDescriptor<?, ?>... descriptors) throws Exception {

	return getInitializedTestHarness(
			function,
			1,
			1,
			0,
			descriptors);
}
 

ReadOnlyContextImpl(
		final ExecutionConfig executionConfig,
		final BroadcastProcessFunction<IN1, IN2, OUT> function,
		final Map<MapStateDescriptor<?, ?>, BroadcastState<?, ?>> broadcastStates,
		final ProcessingTimeService timerService) {

	function.super();
	this.config = Preconditions.checkNotNull(executionConfig);
	this.states = Preconditions.checkNotNull(broadcastStates);
	this.timerService = Preconditions.checkNotNull(timerService);
}
 

ReadWriteContextImpl(
		final ExecutionConfig executionConfig,
		final BroadcastProcessFunction<IN1, IN2, OUT> function,
		final Map<MapStateDescriptor<?, ?>, BroadcastState<?, ?>> broadcastStates,
		final ProcessingTimeService timerService) {

	function.super();
	this.config = Preconditions.checkNotNull(executionConfig);
	this.states = Preconditions.checkNotNull(broadcastStates);
	this.timerService = Preconditions.checkNotNull(timerService);
}
 

/**
 * Assumes as inputs a {@link BroadcastStream} and a non-keyed {@link DataStream} and applies the given
 * {@link BroadcastProcessFunction} on them, thereby creating a transformed output stream.
 *
 * @param function The {@link BroadcastProcessFunction} that is called for each element in the stream.
 * @param outTypeInfo The type of the output elements.
 * @param <OUT> The type of the output elements.
 * @return The transformed {@link DataStream}.
 */
@PublicEvolving
public <OUT> SingleOutputStreamOperator<OUT> process(
		final BroadcastProcessFunction<IN1, IN2, OUT> function,
		final TypeInformation<OUT> outTypeInfo) {

	Preconditions.checkNotNull(function);
	Preconditions.checkArgument(!(inputStream1 instanceof KeyedStream),
			"A BroadcastProcessFunction can only be used on a non-keyed stream.");

	TwoInputStreamOperator<IN1, IN2, OUT> operator =
			new CoBroadcastWithNonKeyedOperator<>(clean(function), broadcastStateDescriptors);
	return transform("Co-Process-Broadcast", outTypeInfo, operator);
}
 

public CoBroadcastWithNonKeyedOperator(
		final BroadcastProcessFunction<IN1, IN2, OUT> function,
		final List<MapStateDescriptor<?, ?>> broadcastStateDescriptors) {
	super(function);
	this.broadcastStateDescriptors = Preconditions.checkNotNull(broadcastStateDescriptors);
}
 

public CoBroadcastWithNonKeyedOperator(
		final BroadcastProcessFunction<IN1, IN2, OUT> function,
		final List<MapStateDescriptor<?, ?>> broadcastStateDescriptors) {
	super(function);
	this.broadcastStateDescriptors = Preconditions.checkNotNull(broadcastStateDescriptors);
}
 

public CoBroadcastWithNonKeyedOperator(
		final BroadcastProcessFunction<IN1, IN2, OUT> function,
		final List<MapStateDescriptor<?, ?>> broadcastStateDescriptors) {
	super(function);
	this.broadcastStateDescriptors = Preconditions.checkNotNull(broadcastStateDescriptors);
}