org.apache.flink.streaming.api.operators.TwoInputStreamOperator Java Examples

public KeyedTwoInputStreamOperatorTestHarness(
		TwoInputStreamOperator<IN1, IN2, OUT> operator,
		KeySelector<IN1, K> keySelector1,
		KeySelector<IN2, K> keySelector2,
		TypeInformation<K> keyType,
		int maxParallelism,
		int numSubtasks,
		int subtaskIndex) throws Exception {
	super(operator, maxParallelism, numSubtasks, subtaskIndex);

	ClosureCleaner.clean(keySelector1, ExecutionConfig.ClosureCleanerLevel.RECURSIVE, false);
	ClosureCleaner.clean(keySelector2, ExecutionConfig.ClosureCleanerLevel.RECURSIVE, false);
	config.setStatePartitioner(0, keySelector1);
	config.setStatePartitioner(1, keySelector2);
	config.setStateKeySerializer(keyType.createSerializer(executionConfig));
}
 

/**
 * Applies the given {@link KeyedCoProcessFunction} on the connected input streams,
 * thereby creating a transformed output stream.
 *
 * <p>The function will be called for every element in the input streams and can produce zero
 * or more output elements. Contrary to the {@link #flatMap(CoFlatMapFunction)} function,
 * this function can also query the time and set timers. When reacting to the firing of set
 * timers the function can directly emit elements and/or register yet more timers.
 *
 * @param keyedCoProcessFunction The {@link KeyedCoProcessFunction} that is called for each element
 *                      in the stream.
 *
 * @param <R> The type of elements emitted by the {@code CoProcessFunction}.
 *
 * @return The transformed {@link DataStream}.
 */
@Internal
public <K, R> SingleOutputStreamOperator<R> process(
	KeyedCoProcessFunction<K, IN1, IN2, R> keyedCoProcessFunction,
	TypeInformation<R> outputType) {

	TwoInputStreamOperator<IN1, IN2, R> operator;

	if ((inputStream1 instanceof KeyedStream) && (inputStream2 instanceof KeyedStream)) {
		operator = new KeyedCoProcessOperator<>(inputStream1.clean(keyedCoProcessFunction));
	} else {
		throw new UnsupportedOperationException("KeyedCoProcessFunction can only be used " +
			"when both input streams are of type KeyedStream.");
	}

	return transform("Co-Keyed-Process", outputType, operator);
}
 

private ForwardingValveOutputHandler(
	TwoInputStreamOperator<IN1, IN2, ?> operator,
	Object lock,
	StreamStatusMaintainer streamStatusMaintainer,
	WatermarkGauge inputWatermarkGauge,
	int inputIndex) {

	this.operator = checkNotNull(operator);
	this.lock = checkNotNull(lock);

	this.streamStatusMaintainer = checkNotNull(streamStatusMaintainer);

	this.inputWatermarkGauge = inputWatermarkGauge;

	this.inputIndex = inputIndex;
}
 

public KeyedTwoInputStreamOperatorTestHarness(
		TwoInputStreamOperator<IN1, IN2, OUT> operator,
		KeySelector<IN1, K> keySelector1,
		KeySelector<IN2, K> keySelector2,
		TypeInformation<K> keyType,
		int maxParallelism,
		int numSubtasks,
		int subtaskIndex) throws Exception {
	super(operator, maxParallelism, numSubtasks, subtaskIndex);

	ClosureCleaner.clean(keySelector1, ExecutionConfig.ClosureCleanerLevel.RECURSIVE, false);
	ClosureCleaner.clean(keySelector2, ExecutionConfig.ClosureCleanerLevel.RECURSIVE, false);
	config.setStatePartitioner(0, keySelector1);
	config.setStatePartitioner(1, keySelector2);
	config.setStateKeySerializer(keyType.createSerializer(executionConfig));
}
 

public KeyedTwoInputStreamOperatorTestHarness(
		TwoInputStreamOperator<IN1, IN2, OUT> operator,
		KeySelector<IN1, K> keySelector1,
		KeySelector<IN2, K> keySelector2,
		TypeInformation<K> keyType,
		int maxParallelism,
		int numSubtasks,
		int subtaskIndex) throws Exception {
	super(operator, maxParallelism, numSubtasks, subtaskIndex);

	ClosureCleaner.clean(keySelector1, ExecutionConfig.ClosureCleanerLevel.RECURSIVE, false);
	ClosureCleaner.clean(keySelector2, ExecutionConfig.ClosureCleanerLevel.RECURSIVE, false);
	config.setStatePartitioner(0, keySelector1);
	config.setStatePartitioner(1, keySelector2);
	config.setStateKeySerializer(keyType.createSerializer(executionConfig));
}
 

/**
 * Applies the given {@link KeyedCoProcessFunction} on the connected input streams,
 * thereby creating a transformed output stream.
 *
 * <p>The function will be called for every element in the input streams and can produce zero
 * or more output elements. Contrary to the {@link #flatMap(CoFlatMapFunction)} function,
 * this function can also query the time and set timers. When reacting to the firing of set
 * timers the function can directly emit elements and/or register yet more timers.
 *
 * @param keyedCoProcessFunction The {@link KeyedCoProcessFunction} that is called for each element
 *                      in the stream.
 *
 * @param <R> The type of elements emitted by the {@code CoProcessFunction}.
 *
 * @return The transformed {@link DataStream}.
 */
@Internal
public <K, R> SingleOutputStreamOperator<R> process(
	KeyedCoProcessFunction<K, IN1, IN2, R> keyedCoProcessFunction,
	TypeInformation<R> outputType) {

	TwoInputStreamOperator<IN1, IN2, R> operator;

	if ((inputStream1 instanceof KeyedStream) && (inputStream2 instanceof KeyedStream)) {
		operator = new KeyedCoProcessOperator<>(inputStream1.clean(keyedCoProcessFunction));
	} else {
		throw new UnsupportedOperationException("KeyedCoProcessFunction can only be used " +
			"when both input streams are of type KeyedStream.");
	}

	return transform("Co-Keyed-Process", outputType, operator);
}
 

/**
 * Assumes as inputs a {@link BroadcastStream} and a {@link KeyedStream} and applies the given
 * {@link KeyedBroadcastProcessFunction} on them, thereby creating a transformed output stream.
 *
 * @param function The {@link KeyedBroadcastProcessFunction} that is called for each element in the stream.
 * @param outTypeInfo The type of the output elements.
 * @param <KS> The type of the keys in the keyed stream.
 * @param <OUT> The type of the output elements.
 * @return The transformed {@link DataStream}.
 */
@PublicEvolving
public <KS, OUT> SingleOutputStreamOperator<OUT> process(
		final KeyedBroadcastProcessFunction<KS, IN1, IN2, OUT> function,
		final TypeInformation<OUT> outTypeInfo) {

	Preconditions.checkNotNull(function);
	Preconditions.checkArgument(inputStream1 instanceof KeyedStream,
			"A KeyedBroadcastProcessFunction can only be used on a keyed stream.");

	TwoInputStreamOperator<IN1, IN2, OUT> operator =
			new CoBroadcastWithKeyedOperator<>(clean(function), broadcastStateDescriptors);
	return transform("Co-Process-Broadcast-Keyed", outTypeInfo, operator);
}
 

/**
 * Ends an input (specified by {@code inputId}) of the {@link StreamTask}. The {@code inputId}
 * is numbered starting from 1, and `1` indicates the first input.
 *
 * @param inputId The ID of the input.
 * @throws Exception if some exception happens in the endInput function of an operator.
 */
public void endInput(int inputId) throws Exception {
	if (finishedInputs.areAllInputsSelected()) {
		return;
	}

	if (headOperator instanceof TwoInputStreamOperator) {
		if (finishedInputs.isInputSelected(inputId)) {
			return;
		}

		if (headOperator instanceof BoundedMultiInput) {
			((BoundedMultiInput) headOperator).endInput(inputId);
		}

		finishedInputs = InputSelection.Builder
			.from(finishedInputs)
			.select(finishedInputs.getInputMask() == 0 ? inputId : -1)
			.build();
	} else {
		// here, the head operator is a stream source or an one-input stream operator,
		// so all inputs are finished
		finishedInputs = new InputSelection.Builder()
			.select(-1)
			.build();
	}

	if (finishedInputs.areAllInputsSelected()) {
		// executing #endInput() happens from head to tail operator in the chain
		for (int i = allOperators.length - 1; i >= 0; i--) {
			StreamOperator<?> operator = allOperators[i];
			if (operator instanceof BoundedOneInput) {
				((BoundedOneInput) operator).endInput();
			}
		}
	}
}
 

TestHarness(
	TwoInputStreamOperator<TestElem, TestElem, Tuple2<TestElem, TestElem>> operator,
	KeySelector<TestElem, String> keySelector1,
	KeySelector<TestElem, String> keySelector2,
	TypeInformation<String> keyType) throws Exception {
	super(operator, keySelector1, keySelector2, keyType);
}
 

public TwoInputStreamOperatorTestHarness(
		TwoInputStreamOperator<IN1, IN2, OUT> operator,
		int maxParallelism,
		int numSubtasks,
		int subtaskIndex) throws Exception {
	super(operator, maxParallelism, numSubtasks, subtaskIndex);

	this.twoInputOperator = operator;
}
 

public KeyedTwoInputStreamOperatorTestHarness(
		TwoInputStreamOperator<IN1, IN2, OUT> operator,
		final KeySelector<IN1, K> keySelector1,
		final KeySelector<IN2, K> keySelector2,
		TypeInformation<K> keyType) throws Exception {
	this(operator, keySelector1, keySelector2, keyType, 1, 1, 0);
}
 

/**
 * Assumes as inputs a {@link BroadcastStream} and a {@link KeyedStream} and applies the given
 * {@link KeyedBroadcastProcessFunction} on them, thereby creating a transformed output stream.
 *
 * @param function The {@link KeyedBroadcastProcessFunction} that is called for each element in the stream.
 * @param outTypeInfo The type of the output elements.
 * @param <KS> The type of the keys in the keyed stream.
 * @param <OUT> The type of the output elements.
 * @return The transformed {@link DataStream}.
 */
@PublicEvolving
public <KS, OUT> SingleOutputStreamOperator<OUT> process(
		final KeyedBroadcastProcessFunction<KS, IN1, IN2, OUT> function,
		final TypeInformation<OUT> outTypeInfo) {

	Preconditions.checkNotNull(function);
	Preconditions.checkArgument(inputStream1 instanceof KeyedStream,
			"A KeyedBroadcastProcessFunction can only be used on a keyed stream.");

	TwoInputStreamOperator<IN1, IN2, OUT> operator =
			new CoBroadcastWithKeyedOperator<>(clean(function), broadcastStateDescriptors);
	return transform("Co-Process-Broadcast-Keyed", outTypeInfo, operator);
}
 

/**
 * Assumes as inputs a {@link BroadcastStream} and a non-keyed {@link DataStream} and applies the given
 * {@link BroadcastProcessFunction} on them, thereby creating a transformed output stream.
 *
 * @param function The {@link BroadcastProcessFunction} that is called for each element in the stream.
 * @param outTypeInfo The type of the output elements.
 * @param <OUT> The type of the output elements.
 * @return The transformed {@link DataStream}.
 */
@PublicEvolving
public <OUT> SingleOutputStreamOperator<OUT> process(
		final BroadcastProcessFunction<IN1, IN2, OUT> function,
		final TypeInformation<OUT> outTypeInfo) {

	Preconditions.checkNotNull(function);
	Preconditions.checkArgument(!(inputStream1 instanceof KeyedStream),
			"A BroadcastProcessFunction can only be used on a non-keyed stream.");

	TwoInputStreamOperator<IN1, IN2, OUT> operator =
			new CoBroadcastWithNonKeyedOperator<>(clean(function), broadcastStateDescriptors);
	return transform("Co-Process-Broadcast", outTypeInfo, operator);
}
 

private void processRecord1(
		StreamRecord<IN1> record,
		TwoInputStreamOperator<IN1, IN2, ?> streamOperator,
		Counter numRecordsIn) throws Exception {

	streamOperator.setKeyContextElement1(record);
	streamOperator.processElement1(record);
	postProcessRecord(numRecordsIn);
}
 

private void processRecord2(
		StreamRecord<IN2> record,
		TwoInputStreamOperator<IN1, IN2, ?> streamOperator,
		Counter numRecordsIn) throws Exception {

	streamOperator.setKeyContextElement2(record);
	streamOperator.processElement2(record);
	postProcessRecord(numRecordsIn);
}
 

private StreamTaskNetworkOutput(
		TwoInputStreamOperator<IN1, IN2, ?> operator,
		ThrowingConsumer<StreamRecord<T>, Exception> recordConsumer,
		StreamStatusMaintainer streamStatusMaintainer,
		WatermarkGauge inputWatermarkGauge,
		int inputIndex) {
	super(streamStatusMaintainer);

	this.operator = checkNotNull(operator);
	this.recordConsumer = checkNotNull(recordConsumer);
	this.inputWatermarkGauge = checkNotNull(inputWatermarkGauge);
	this.inputIndex = inputIndex;
}
 

TestHarness(
	TwoInputStreamOperator<TestElem, TestElem, Tuple2<TestElem, TestElem>> operator,
	KeySelector<TestElem, String> keySelector1,
	KeySelector<TestElem, String> keySelector2,
	TypeInformation<String> keyType) throws Exception {
	super(operator, keySelector1, keySelector2, keyType);
}
 

public TwoInputStreamOperatorTestHarness(
		TwoInputStreamOperator<IN1, IN2, OUT> operator,
		int maxParallelism,
		int numSubtasks,
		int subtaskIndex) throws Exception {
	super(operator, maxParallelism, numSubtasks, subtaskIndex);

	this.twoInputOperator = operator;
}
 

public KeyedTwoInputStreamOperatorTestHarness(
		TwoInputStreamOperator<IN1, IN2, OUT> operator,
		final KeySelector<IN1, K> keySelector1,
		final KeySelector<IN2, K> keySelector2,
		TypeInformation<K> keyType) throws Exception {
	this(operator, keySelector1, keySelector2, keyType, 1, 1, 0);
}
 

public <IN1, IN2, OUT> void addCoOperator(
		Integer vertexID,
		String slotSharingGroup,
		@Nullable String coLocationGroup,
		TwoInputStreamOperator<IN1, IN2, OUT> taskOperatorObject,
		TypeInformation<IN1> in1TypeInfo,
		TypeInformation<IN2> in2TypeInfo,
		TypeInformation<OUT> outTypeInfo,
		String operatorName) {

	addNode(vertexID, slotSharingGroup, coLocationGroup, TwoInputStreamTask.class, taskOperatorObject, operatorName);

	TypeSerializer<OUT> outSerializer = (outTypeInfo != null) && !(outTypeInfo instanceof MissingTypeInfo) ?
			outTypeInfo.createSerializer(executionConfig) : null;

	setSerializers(vertexID, in1TypeInfo.createSerializer(executionConfig), in2TypeInfo.createSerializer(executionConfig), outSerializer);

	if (taskOperatorObject instanceof OutputTypeConfigurable) {
		@SuppressWarnings("unchecked")
		OutputTypeConfigurable<OUT> outputTypeConfigurable = (OutputTypeConfigurable<OUT>) taskOperatorObject;
		// sets the output type which must be know at StreamGraph creation time
		outputTypeConfigurable.setOutputType(outTypeInfo, executionConfig);
	}

	if (LOG.isDebugEnabled()) {
		LOG.debug("CO-TASK: {}", vertexID);
	}
}
 

TestHarness(
	TwoInputStreamOperator<TestElem, TestElem, Tuple2<TestElem, TestElem>> operator,
	KeySelector<TestElem, String> keySelector1,
	KeySelector<TestElem, String> keySelector2,
	TypeInformation<String> keyType) throws Exception {
	super(operator, keySelector1, keySelector2, keyType);
}
 

/**
 * Assumes as inputs a {@link BroadcastStream} and a non-keyed {@link DataStream} and applies the given
 * {@link BroadcastProcessFunction} on them, thereby creating a transformed output stream.
 *
 * @param function The {@link BroadcastProcessFunction} that is called for each element in the stream.
 * @param outTypeInfo The type of the output elements.
 * @param <OUT> The type of the output elements.
 * @return The transformed {@link DataStream}.
 */
@PublicEvolving
public <OUT> SingleOutputStreamOperator<OUT> process(
		final BroadcastProcessFunction<IN1, IN2, OUT> function,
		final TypeInformation<OUT> outTypeInfo) {

	Preconditions.checkNotNull(function);
	Preconditions.checkArgument(!(inputStream1 instanceof KeyedStream),
			"A BroadcastProcessFunction can only be used on a non-keyed stream.");

	TwoInputStreamOperator<IN1, IN2, OUT> operator =
			new CoBroadcastWithNonKeyedOperator<>(clean(function), broadcastStateDescriptors);
	return transform("Co-Process-Broadcast", outTypeInfo, operator);
}
 

public TwoInputStreamOperatorTestHarness(
		TwoInputStreamOperator<IN1, IN2, OUT> operator,
		int maxParallelism,
		int numSubtasks,
		int subtaskIndex) throws Exception {
	super(operator, maxParallelism, numSubtasks, subtaskIndex);

	this.twoInputOperator = operator;
}
 

public KeyedTwoInputStreamOperatorTestHarness(
		TwoInputStreamOperator<IN1, IN2, OUT> operator,
		final KeySelector<IN1, K> keySelector1,
		final KeySelector<IN2, K> keySelector2,
		TypeInformation<K> keyType) throws Exception {
	this(operator, keySelector1, keySelector2, keyType, 1, 1, 0);
}
 

/**
 * Creates a new {@code TwoInputTransformation} from the given inputs and operator.
 *
 * @param input1 The first input {@code StreamTransformation}
 * @param input2 The second input {@code StreamTransformation}
 * @param name The name of the {@code StreamTransformation}, this will be shown in Visualizations and the Log
 * @param operator The {@code TwoInputStreamOperator}
 * @param outputType The type of the elements produced by this Transformation
 * @param parallelism The parallelism of this Transformation
 */
public TwoInputTransformation(
		StreamTransformation<IN1> input1,
		StreamTransformation<IN2> input2,
		String name,
		TwoInputStreamOperator<IN1, IN2, OUT> operator,
		TypeInformation<OUT> outputType,
		int parallelism) {
	super(name, outputType, parallelism);
	this.input1 = input1;
	this.input2 = input2;
	this.operator = operator;
}
 

/**
 * Assumes as inputs a {@link BroadcastStream} and a {@link KeyedStream} and applies the given
 * {@link KeyedBroadcastProcessFunction} on them, thereby creating a transformed output stream.
 *
 * @param function The {@link KeyedBroadcastProcessFunction} that is called for each element in the stream.
 * @param outTypeInfo The type of the output elements.
 * @param <KS> The type of the keys in the keyed stream.
 * @param <OUT> The type of the output elements.
 * @return The transformed {@link DataStream}.
 */
@PublicEvolving
public <KS, OUT> SingleOutputStreamOperator<OUT> process(
		final KeyedBroadcastProcessFunction<KS, IN1, IN2, OUT> function,
		final TypeInformation<OUT> outTypeInfo) {

	Preconditions.checkNotNull(function);
	Preconditions.checkArgument(inputStream1 instanceof KeyedStream,
			"A KeyedBroadcastProcessFunction can only be used on a keyed stream.");

	TwoInputStreamOperator<IN1, IN2, OUT> operator =
			new CoBroadcastWithKeyedOperator<>(clean(function), broadcastStateDescriptors);
	return transform("Co-Process-Broadcast-Keyed", outTypeInfo, operator);
}
 

/**
 * Assumes as inputs a {@link BroadcastStream} and a non-keyed {@link DataStream} and applies the given
 * {@link BroadcastProcessFunction} on them, thereby creating a transformed output stream.
 *
 * @param function The {@link BroadcastProcessFunction} that is called for each element in the stream.
 * @param outTypeInfo The type of the output elements.
 * @param <OUT> The type of the output elements.
 * @return The transformed {@link DataStream}.
 */
@PublicEvolving
public <OUT> SingleOutputStreamOperator<OUT> process(
		final BroadcastProcessFunction<IN1, IN2, OUT> function,
		final TypeInformation<OUT> outTypeInfo) {

	Preconditions.checkNotNull(function);
	Preconditions.checkArgument(!(inputStream1 instanceof KeyedStream),
			"A BroadcastProcessFunction can only be used on a non-keyed stream.");

	TwoInputStreamOperator<IN1, IN2, OUT> operator =
			new CoBroadcastWithNonKeyedOperator<>(clean(function), broadcastStateDescriptors);
	return transform("Co-Process-Broadcast", outTypeInfo, operator);
}
 

public TwoInputStreamOperatorTestHarness(TwoInputStreamOperator<IN1, IN2, OUT> operator) throws Exception {
	this(operator, 1, 1, 0);
}
 

private void testBase(
	TwoInputStreamOperator<String, Integer, String> streamOperator,
	boolean prepareDataBeforeProcessing,
	ConcurrentLinkedQueue<Object> expectedOutput,
	boolean orderedCheck) throws Exception {

	final TwoInputStreamTaskTestHarness<String, Integer, String> testHarness = new TwoInputStreamTaskTestHarness<>(
		TestSelectiveReadingTask::new,
		BasicTypeInfo.STRING_TYPE_INFO, BasicTypeInfo.INT_TYPE_INFO, BasicTypeInfo.STRING_TYPE_INFO);

	testHarness.setupOutputForSingletonOperatorChain();
	StreamConfig streamConfig = testHarness.getStreamConfig();
	streamConfig.setStreamOperator(streamOperator);
	streamConfig.setOperatorID(new OperatorID());

	testHarness.invoke();
	testHarness.waitForTaskRunning();

	boolean isProcessing = false;
	if (!prepareDataBeforeProcessing) {
		((TestSelectiveReadingTask) testHarness.getTask()).startProcessing();
		isProcessing = true;
	}

	testHarness.processElement(new StreamRecord<>("Hello-1"), 0, 0);

	// wait until the input is processed to test the listening and blocking logic
	if (!prepareDataBeforeProcessing) {
		testHarness.waitForInputProcessing();
	}

	testHarness.processElement(new StreamRecord<>("Hello-2"), 0, 0);
	testHarness.processElement(new StreamRecord<>("Hello-3"), 0, 0);

	testHarness.processElement(new StreamRecord<>(1), 1, 0);
	testHarness.processElement(new StreamRecord<>(2), 1, 0);
	testHarness.processElement(new StreamRecord<>(3), 1, 0);
	testHarness.processElement(new StreamRecord<>(4), 1, 0);

	testHarness.endInput();

	if (!isProcessing) {
		((TestSelectiveReadingTask) testHarness.getTask()).startProcessing();
	}
	testHarness.waitForTaskCompletion(10_000L);

	LinkedBlockingQueue<Object> output = testHarness.getOutput();
	if (orderedCheck) {
		TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, output);
	} else {
		String[] expectedResult = expectedOutput.stream()
			.map(record -> ((StreamRecord) record).getValue().toString())
			.toArray(String[]::new);
		Arrays.sort(expectedResult);

		String[] result = output.stream()
			.map(record -> ((StreamRecord) record).getValue().toString())
			.toArray(String[]::new);
		Arrays.sort(result);

		assertArrayEquals("Output was not correct.", expectedResult, result);
	}
}
 

/**
 * Returns the {@code TwoInputStreamOperator} of this Transformation.
 */
public TwoInputStreamOperator<IN1, IN2, OUT> getOperator() {
	return operator;
}