org.apache.flink.streaming.runtime.tasks.TwoInputStreamTask Java Examples

private TwoInputStreamTaskTestHarness<BinaryRow, BinaryRow, JoinedRow> buildSortMergeJoin(StreamOperator operator) throws Exception {
	final TwoInputStreamTaskTestHarness<BinaryRow, BinaryRow, JoinedRow> testHarness =
			new TwoInputStreamTaskTestHarness<>(TwoInputStreamTask::new, 2, 2,
				new int[]{1, 2}, typeInfo, (TypeInformation) typeInfo, joinedInfo);

	testHarness.memorySize = 36 * 1024 * 1024;
	testHarness.setupOutputForSingletonOperatorChain();
	testHarness.getStreamConfig().setStreamOperator(operator);
	testHarness.getStreamConfig().setOperatorID(new OperatorID());

	long initialTime = 0L;

	testHarness.invoke();
	testHarness.waitForTaskRunning();

	testHarness.processElement(new StreamRecord<>(newRow("a", "0"), initialTime), 0, 0);
	testHarness.processElement(new StreamRecord<>(newRow("d", "0"), initialTime), 0, 0);
	testHarness.processElement(new StreamRecord<>(newRow("a", "2"), initialTime), 1, 1);
	testHarness.processElement(new StreamRecord<>(newRow("b", "1"), initialTime), 0, 1);
	testHarness.processElement(new StreamRecord<>(newRow("c", "2"), initialTime), 1, 1);
	testHarness.processElement(new StreamRecord<>(newRow("b", "4"), initialTime), 1, 0);
	testHarness.waitForInputProcessing();

	testHarness.endInput();
	return testHarness;
}
 

public <IN1, IN2, OUT> void addCoOperator(
		Integer vertexID,
		String slotSharingGroup,
		@Nullable String coLocationGroup,
		TwoInputStreamOperator<IN1, IN2, OUT> taskOperatorObject,
		TypeInformation<IN1> in1TypeInfo,
		TypeInformation<IN2> in2TypeInfo,
		TypeInformation<OUT> outTypeInfo,
		String operatorName) {

	addNode(vertexID, slotSharingGroup, coLocationGroup, TwoInputStreamTask.class, taskOperatorObject, operatorName);

	TypeSerializer<OUT> outSerializer = (outTypeInfo != null) && !(outTypeInfo instanceof MissingTypeInfo) ?
			outTypeInfo.createSerializer(executionConfig) : null;

	setSerializers(vertexID, in1TypeInfo.createSerializer(executionConfig), in2TypeInfo.createSerializer(executionConfig), outSerializer);

	if (taskOperatorObject instanceof OutputTypeConfigurable) {
		@SuppressWarnings("unchecked")
		OutputTypeConfigurable<OUT> outputTypeConfigurable = (OutputTypeConfigurable<OUT>) taskOperatorObject;
		// sets the output type which must be know at StreamGraph creation time
		outputTypeConfigurable.setOutputType(outTypeInfo, executionConfig);
	}

	if (LOG.isDebugEnabled()) {
		LOG.debug("CO-TASK: {}", vertexID);
	}
}
 

private void init(boolean leftOut, boolean rightOut, boolean buildLeft) throws Exception {
	HashJoinType type = HashJoinType.of(buildLeft, leftOut, rightOut);
	HashJoinOperator operator = newOperator(33 * 32 * 1024, type, !buildLeft);
	testHarness = new TwoInputStreamTaskTestHarness<>(
			TwoInputStreamTask::new, 2, 2, new int[]{1, 2}, typeInfo, (TypeInformation) typeInfo, joinedInfo);
	testHarness.memorySize = 36 * 1024 * 1024;
	testHarness.getExecutionConfig().enableObjectReuse();
	testHarness.setupOutputForSingletonOperatorChain();
	testHarness.getStreamConfig().setStreamOperator(operator);
	testHarness.getStreamConfig().setOperatorID(new OperatorID());

	testHarness.invoke();
	testHarness.waitForTaskRunning();
}
 

public <IN1, IN2, OUT> void addCoOperator(
		Integer vertexID,
		String slotSharingGroup,
		@Nullable String coLocationGroup,
		StreamOperatorFactory<OUT> taskOperatorFactory,
		TypeInformation<IN1> in1TypeInfo,
		TypeInformation<IN2> in2TypeInfo,
		TypeInformation<OUT> outTypeInfo,
		String operatorName) {

	Class<? extends AbstractInvokable> vertexClass = taskOperatorFactory.isOperatorSelectiveReading() ?
		TwoInputSelectableStreamTask.class : TwoInputStreamTask.class;

	addNode(vertexID, slotSharingGroup, coLocationGroup, vertexClass, taskOperatorFactory, operatorName);

	TypeSerializer<OUT> outSerializer = (outTypeInfo != null) && !(outTypeInfo instanceof MissingTypeInfo) ?
			outTypeInfo.createSerializer(executionConfig) : null;

	setSerializers(vertexID, in1TypeInfo.createSerializer(executionConfig), in2TypeInfo.createSerializer(executionConfig), outSerializer);

	if (taskOperatorFactory.isOutputTypeConfigurable()) {
		// sets the output type which must be know at StreamGraph creation time
		taskOperatorFactory.setOutputType(outTypeInfo, executionConfig);
	}

	if (LOG.isDebugEnabled()) {
		LOG.debug("CO-TASK: {}", vertexID);
	}
}
 

private void init(boolean leftOut, boolean rightOut, boolean buildLeft) throws Exception {
	HashJoinType type = HashJoinType.of(buildLeft, leftOut, rightOut);
	HashJoinOperator operator = newOperator(33 * 32 * 1024, type, !buildLeft);
	testHarness = new TwoInputStreamTaskTestHarness<>(
			TwoInputStreamTask::new, 2, 2, new int[]{1, 2}, typeInfo, (TypeInformation) typeInfo, joinedInfo);
	testHarness.memorySize = 36 * 1024 * 1024;
	testHarness.getExecutionConfig().enableObjectReuse();
	testHarness.setupOutputForSingletonOperatorChain();
	testHarness.getStreamConfig().setStreamOperator(operator);
	testHarness.getStreamConfig().setOperatorID(new OperatorID());
	testHarness.getStreamConfig().setManagedMemoryFraction(0.99);

	testHarness.invoke();
	testHarness.waitForTaskRunning();
}
 

private TwoInputStreamTaskTestHarness<BinaryRowData, BinaryRowData, JoinedRowData> buildSortMergeJoin(StreamOperator operator) throws Exception {
	final TwoInputStreamTaskTestHarness<BinaryRowData, BinaryRowData, JoinedRowData> testHarness =
			new TwoInputStreamTaskTestHarness<>(TwoInputStreamTask::new, 2, 2,
				new int[]{1, 2}, typeInfo, (TypeInformation) typeInfo, joinedInfo);

	testHarness.memorySize = 36 * 1024 * 1024;
	testHarness.setupOutputForSingletonOperatorChain();
	testHarness.getStreamConfig().setStreamOperator(operator);
	testHarness.getStreamConfig().setOperatorID(new OperatorID());
	testHarness.getStreamConfig().setManagedMemoryFraction(0.99);

	long initialTime = 0L;

	testHarness.invoke();
	testHarness.waitForTaskRunning();

	testHarness.processElement(new StreamRecord<>(newRow("a", "0"), initialTime), 0, 0);
	testHarness.processElement(new StreamRecord<>(newRow("d", "0"), initialTime), 0, 0);
	testHarness.processElement(new StreamRecord<>(newRow("a", "2"), initialTime), 1, 1);
	testHarness.processElement(new StreamRecord<>(newRow("b", "1"), initialTime), 0, 1);
	testHarness.processElement(new StreamRecord<>(newRow("c", "2"), initialTime), 1, 1);
	testHarness.processElement(new StreamRecord<>(newRow("b", "4"), initialTime), 1, 0);
	testHarness.waitForInputProcessing();

	testHarness.endInput();
	return testHarness;
}
 

public <IN1, IN2, OUT> void addCoOperator(
		Integer vertexID,
		String slotSharingGroup,
		@Nullable String coLocationGroup,
		StreamOperatorFactory<OUT> taskOperatorFactory,
		TypeInformation<IN1> in1TypeInfo,
		TypeInformation<IN2> in2TypeInfo,
		TypeInformation<OUT> outTypeInfo,
		String operatorName) {

	Class<? extends AbstractInvokable> vertexClass = TwoInputStreamTask.class;

	addNode(vertexID, slotSharingGroup, coLocationGroup, vertexClass, taskOperatorFactory, operatorName);

	TypeSerializer<OUT> outSerializer = createSerializer(outTypeInfo);

	setSerializers(vertexID, in1TypeInfo.createSerializer(executionConfig), in2TypeInfo.createSerializer(executionConfig), outSerializer);

	if (taskOperatorFactory.isOutputTypeConfigurable()) {
		// sets the output type which must be know at StreamGraph creation time
		taskOperatorFactory.setOutputType(outTypeInfo, executionConfig);
	}

	if (LOG.isDebugEnabled()) {
		LOG.debug("CO-TASK: {}", vertexID);
	}
}
 

@SuppressWarnings("unchecked")
public StreamTwoInputProcessor(
		Collection<InputGate> inputGates1,
		Collection<InputGate> inputGates2,
		TypeSerializer<IN1> inputSerializer1,
		TypeSerializer<IN2> inputSerializer2,
		TwoInputStreamTask<IN1, IN2, ?> checkpointedTask,
		CheckpointingMode checkpointMode,
		Object lock,
		IOManager ioManager,
		Configuration taskManagerConfig,
		StreamStatusMaintainer streamStatusMaintainer,
		TwoInputStreamOperator<IN1, IN2, ?> streamOperator,
		TaskIOMetricGroup metrics,
		WatermarkGauge input1WatermarkGauge,
		WatermarkGauge input2WatermarkGauge) throws IOException {

	final InputGate inputGate = InputGateUtil.createInputGate(inputGates1, inputGates2);

	this.barrierHandler = InputProcessorUtil.createCheckpointBarrierHandler(
		checkpointedTask, checkpointMode, ioManager, inputGate, taskManagerConfig);

	this.lock = checkNotNull(lock);

	StreamElementSerializer<IN1> ser1 = new StreamElementSerializer<>(inputSerializer1);
	this.deserializationDelegate1 = new NonReusingDeserializationDelegate<>(ser1);

	StreamElementSerializer<IN2> ser2 = new StreamElementSerializer<>(inputSerializer2);
	this.deserializationDelegate2 = new NonReusingDeserializationDelegate<>(ser2);

	// Initialize one deserializer per input channel
	this.recordDeserializers = new SpillingAdaptiveSpanningRecordDeserializer[inputGate.getNumberOfInputChannels()];

	for (int i = 0; i < recordDeserializers.length; i++) {
		recordDeserializers[i] = new SpillingAdaptiveSpanningRecordDeserializer<>(
			ioManager.getSpillingDirectoriesPaths());
	}

	// determine which unioned channels belong to input 1 and which belong to input 2
	int numInputChannels1 = 0;
	for (InputGate gate: inputGates1) {
		numInputChannels1 += gate.getNumberOfInputChannels();
	}

	this.numInputChannels1 = numInputChannels1;
	this.numInputChannels2 = inputGate.getNumberOfInputChannels() - numInputChannels1;

	this.firstStatus = StreamStatus.ACTIVE;
	this.secondStatus = StreamStatus.ACTIVE;

	this.streamStatusMaintainer = checkNotNull(streamStatusMaintainer);
	this.streamOperator = checkNotNull(streamOperator);

	this.statusWatermarkValve1 = new StatusWatermarkValve(numInputChannels1, new ForwardingValveOutputHandler1(streamOperator, lock));
	this.statusWatermarkValve2 = new StatusWatermarkValve(numInputChannels2, new ForwardingValveOutputHandler2(streamOperator, lock));

	this.input1WatermarkGauge = input1WatermarkGauge;
	this.input2WatermarkGauge = input2WatermarkGauge;
	metrics.gauge("checkpointAlignmentTime", barrierHandler::getAlignmentDurationNanos);
}
 

@SuppressWarnings("unchecked")
public StreamTwoInputProcessor(
		Collection<InputGate> inputGates1,
		Collection<InputGate> inputGates2,
		TypeSerializer<IN1> inputSerializer1,
		TypeSerializer<IN2> inputSerializer2,
		TwoInputStreamTask<IN1, IN2, ?> checkpointedTask,
		CheckpointingMode checkpointMode,
		Object lock,
		IOManager ioManager,
		Configuration taskManagerConfig,
		StreamStatusMaintainer streamStatusMaintainer,
		TwoInputStreamOperator<IN1, IN2, ?> streamOperator,
		TaskIOMetricGroup metrics,
		WatermarkGauge input1WatermarkGauge,
		WatermarkGauge input2WatermarkGauge,
		String taskName,
		OperatorChain<?, ?> operatorChain) throws IOException {

	final InputGate inputGate = InputGateUtil.createInputGate(inputGates1, inputGates2);

	this.barrierHandler = InputProcessorUtil.createCheckpointedInputGate(
		checkpointedTask,
		checkpointMode,
		ioManager,
		inputGate,
		taskManagerConfig,
		taskName);

	this.lock = checkNotNull(lock);

	StreamElementSerializer<IN1> ser1 = new StreamElementSerializer<>(inputSerializer1);
	this.deserializationDelegate1 = new NonReusingDeserializationDelegate<>(ser1);

	StreamElementSerializer<IN2> ser2 = new StreamElementSerializer<>(inputSerializer2);
	this.deserializationDelegate2 = new NonReusingDeserializationDelegate<>(ser2);

	// Initialize one deserializer per input channel
	this.recordDeserializers = new SpillingAdaptiveSpanningRecordDeserializer[inputGate.getNumberOfInputChannels()];

	for (int i = 0; i < recordDeserializers.length; i++) {
		recordDeserializers[i] = new SpillingAdaptiveSpanningRecordDeserializer<>(
			ioManager.getSpillingDirectoriesPaths());
	}

	// determine which unioned channels belong to input 1 and which belong to input 2
	int numInputChannels1 = 0;
	for (InputGate gate: inputGates1) {
		numInputChannels1 += gate.getNumberOfInputChannels();
	}

	this.numInputChannels1 = numInputChannels1;
	this.numInputChannels2 = inputGate.getNumberOfInputChannels() - numInputChannels1;

	this.firstStatus = StreamStatus.ACTIVE;
	this.secondStatus = StreamStatus.ACTIVE;

	this.streamStatusMaintainer = checkNotNull(streamStatusMaintainer);
	this.streamOperator = checkNotNull(streamOperator);

	this.statusWatermarkValve1 = new StatusWatermarkValve(numInputChannels1, new ForwardingValveOutputHandler1(streamOperator, lock));
	this.statusWatermarkValve2 = new StatusWatermarkValve(numInputChannels2, new ForwardingValveOutputHandler2(streamOperator, lock));

	this.input1WatermarkGauge = input1WatermarkGauge;
	this.input2WatermarkGauge = input2WatermarkGauge;
	metrics.gauge("checkpointAlignmentTime", barrierHandler::getAlignmentDurationNanos);

	this.operatorChain = checkNotNull(operatorChain);

	this.finishedChannels1 = new BitSet();
	this.finishedChannels2 = new BitSet();
}