org.apache.flink.streaming.runtime.tasks.OperatorChain Java Examples

@Test
public void testEmitMaxWatermarkForFiniteSource() throws Exception {

	// regular stream source operator
	StreamSource<String, FiniteSource<String>> operator =
			new StreamSource<>(new FiniteSource<String>());

	final List<StreamElement> output = new ArrayList<>();

	setupSourceOperator(operator, TimeCharacteristic.EventTime, 0);
	OperatorChain<?, ?> operatorChain = createOperatorChain(operator);
	try {
		operator.run(new Object(), mock(StreamStatusMaintainer.class), new CollectorOutput<String>(output), operatorChain);
	} finally {
		operatorChain.releaseOutputs();
	}

	assertEquals(1, output.size());
	assertEquals(Watermark.MAX_WATERMARK, output.get(0));
}
 

@Test
public void testNoMaxWatermarkOnImmediateCancel() throws Exception {

	final List<StreamElement> output = new ArrayList<>();

	// regular stream source operator
	final StreamSource<String, InfiniteSource<String>> operator =
			new StreamSource<>(new InfiniteSource<String>());

	setupSourceOperator(operator, TimeCharacteristic.EventTime, 0);
	operator.cancel();

	// run and exit
	OperatorChain<?, ?> operatorChain = createOperatorChain(operator);
	try {
		operator.run(new Object(), mock(StreamStatusMaintainer.class), new CollectorOutput<String>(output), operatorChain);
	} finally {
		operatorChain.releaseOutputs();
	}

	assertTrue(output.isEmpty());
}
 

@Override
public void run(Object lockingObject,
				StreamStatusMaintainer streamStatusMaintainer,
				Output<StreamRecord<OUT>> collector,
				OperatorChain<?, ?> operatorChain) throws Exception {
	ACTUAL_ORDER_TRACKING.add("OPERATOR::run");
	super.run(lockingObject, streamStatusMaintainer, collector, operatorChain);
	runStarted.trigger();
	runFinish.await();
}
 

public StreamMultipleInputProcessor(
		CheckpointedInputGate[] checkpointedInputGates,
		TypeSerializer<?>[] inputSerializers,
		IOManager ioManager,
		StreamStatusMaintainer streamStatusMaintainer,
		MultipleInputStreamOperator<?> streamOperator,
		MultipleInputSelectionHandler inputSelectionHandler,
		WatermarkGauge[] inputWatermarkGauges,
		OperatorChain<?, ?> operatorChain,
		Counter numRecordsIn) {

	this.inputSelectionHandler = checkNotNull(inputSelectionHandler);

	List<Input> inputs = streamOperator.getInputs();
	int inputsCount = inputs.size();

	this.inputProcessors = new InputProcessor[inputsCount];
	this.streamStatuses = new StreamStatus[inputsCount];
	this.numRecordsIn = numRecordsIn;

	for (int i = 0; i < inputsCount; i++) {
		streamStatuses[i] = StreamStatus.ACTIVE;
		StreamTaskNetworkOutput dataOutput = new StreamTaskNetworkOutput<>(
			inputs.get(i),
			streamStatusMaintainer,
			inputWatermarkGauges[i],
			i);

		inputProcessors[i] = new InputProcessor(
			dataOutput,
			new StreamTaskNetworkInput<>(
				checkpointedInputGates[i],
				inputSerializers[i],
				ioManager,
				new StatusWatermarkValve(checkpointedInputGates[i].getNumberOfInputChannels(), dataOutput),
				i));
	}

	this.operatorChain = checkNotNull(operatorChain);
}
 

@Override
public void run(Object lockingObject,
				StreamStatusMaintainer streamStatusMaintainer,
				Output<StreamRecord<OUT>> collector,
				OperatorChain<?, ?> operatorChain) throws Exception {
	ACTUAL_ORDER_TRACKING.add("OPERATOR::run");
	super.run(lockingObject, streamStatusMaintainer, collector, operatorChain);
	runStarted.trigger();
	runFinish.await();
}
 

public StreamOneInputProcessor(
		StreamTaskInput<IN> input,
		DataOutput<IN> output,
		OperatorChain<?, ?> operatorChain) {

	this.input = checkNotNull(input);
	this.output = checkNotNull(output);
	this.operatorChain = checkNotNull(operatorChain);
}
 

private <IN, OT extends StreamOperator<IN>> OperatorChain<IN, OT> createOperatorChain(
		StreamConfig streamConfig,
		Environment environment,
		StreamTask<IN, OT> task) {
	return new OperatorChain<>(task, StreamTask.createRecordWriters(streamConfig, environment));
}
 

private <IN, OT extends StreamOperator<IN>> OperatorChain<IN, OT> createOperatorChain(
		StreamConfig streamConfig,
		Environment environment,
		StreamTask<IN, OT> task) {
	return new OperatorChain<>(task, StreamTask.createRecordWriterDelegate(streamConfig, environment));
}
 

private void testLatencyMarkEmission(int numberLatencyMarkers, OperatorSetupOperation operatorSetup) throws Exception {
	final List<StreamElement> output = new ArrayList<>();

	final TestProcessingTimeService testProcessingTimeService = new TestProcessingTimeService();
	testProcessingTimeService.setCurrentTime(0L);
	final List<Long> processingTimes = Arrays.asList(1L, 10L, 11L, 21L, maxProcessingTime);

	// regular stream source operator
	final StreamSource<Long, ProcessingTimeServiceSource> operator =
		new StreamSource<>(new ProcessingTimeServiceSource(testProcessingTimeService, processingTimes));

	operatorSetup.setupSourceOperator(operator, testProcessingTimeService);

	// run and wait to be stopped
	OperatorChain<?, ?> operatorChain = new OperatorChain<>(
		operator.getContainingTask(),
		StreamTask.createRecordWriterDelegate(operator.getOperatorConfig(), new MockEnvironmentBuilder().build()));
	try {
		operator.run(new Object(), mock(StreamStatusMaintainer.class), new CollectorOutput<>(output), operatorChain);
		operator.close();
	} finally {
		operatorChain.releaseOutputs();
	}

	assertEquals(
		numberLatencyMarkers + 1, // + 1 is the final watermark element
		output.size());

	long timestamp = 0L;
	int expectedLatencyIndex = 0;

	int i = 0;
	// verify that its only latency markers + a final watermark
	for (; i < numberLatencyMarkers; i++) {
		StreamElement se = output.get(i);
		Assert.assertTrue(se.isLatencyMarker());
		Assert.assertEquals(operator.getOperatorID(), se.asLatencyMarker().getOperatorId());
		Assert.assertEquals(0, se.asLatencyMarker().getSubtaskIndex());

		// determines the next latency mark that should've been emitted
		// latency marks are emitted once per latencyMarkInterval,
		// as a result of which we never emit both 10 and 11
		while (timestamp > processingTimes.get(expectedLatencyIndex)) {
			expectedLatencyIndex++;
		}
		Assert.assertEquals(processingTimes.get(expectedLatencyIndex).longValue(), se.asLatencyMarker().getMarkedTime());

		timestamp += latencyMarkInterval;
	}

	Assert.assertTrue(output.get(i).isWatermark());
}
 

public StreamTwoInputProcessor(
		CheckpointedInputGate[] checkpointedInputGates,
		TypeSerializer<IN1> inputSerializer1,
		TypeSerializer<IN2> inputSerializer2,
		IOManager ioManager,
		StreamStatusMaintainer streamStatusMaintainer,
		TwoInputStreamOperator<IN1, IN2, ?> streamOperator,
		TwoInputSelectionHandler inputSelectionHandler,
		WatermarkGauge input1WatermarkGauge,
		WatermarkGauge input2WatermarkGauge,
		OperatorChain<?, ?> operatorChain,
		Counter numRecordsIn) {

	this.inputSelectionHandler = checkNotNull(inputSelectionHandler);

	this.output1 = new StreamTaskNetworkOutput<>(
		streamOperator,
		record -> processRecord1(record, streamOperator, numRecordsIn),
		streamStatusMaintainer,
		input1WatermarkGauge,
		0);
	this.output2 = new StreamTaskNetworkOutput<>(
		streamOperator,
		record -> processRecord2(record, streamOperator, numRecordsIn),
		streamStatusMaintainer,
		input2WatermarkGauge,
		1);

	this.input1 = new StreamTaskNetworkInput<>(
		checkpointedInputGates[0],
		inputSerializer1,
		ioManager,
		new StatusWatermarkValve(checkpointedInputGates[0].getNumberOfInputChannels(), output1),
		0);
	this.input2 = new StreamTaskNetworkInput<>(
		checkpointedInputGates[1],
		inputSerializer2,
		ioManager,
		new StatusWatermarkValve(checkpointedInputGates[1].getNumberOfInputChannels(), output2),
		1);

	this.operatorChain = checkNotNull(operatorChain);
}
 

public void run(final Object lockingObject,
		final StreamStatusMaintainer streamStatusMaintainer,
		final Output<StreamRecord<OUT>> collector,
		final OperatorChain<?, ?> operatorChain) throws Exception {

	final TimeCharacteristic timeCharacteristic = getOperatorConfig().getTimeCharacteristic();

	final Configuration configuration = this.getContainingTask().getEnvironment().getTaskManagerInfo().getConfiguration();
	final long latencyTrackingInterval = getExecutionConfig().isLatencyTrackingConfigured()
		? getExecutionConfig().getLatencyTrackingInterval()
		: configuration.getLong(MetricOptions.LATENCY_INTERVAL);

	LatencyMarksEmitter<OUT> latencyEmitter = null;
	if (latencyTrackingInterval > 0) {
		latencyEmitter = new LatencyMarksEmitter<>(
			getProcessingTimeService(),
			collector,
			latencyTrackingInterval,
			this.getOperatorID(),
			getRuntimeContext().getIndexOfThisSubtask());
	}

	final long watermarkInterval = getRuntimeContext().getExecutionConfig().getAutoWatermarkInterval();

	this.ctx = StreamSourceContexts.getSourceContext(
		timeCharacteristic,
		getProcessingTimeService(),
		lockingObject,
		streamStatusMaintainer,
		collector,
		watermarkInterval,
		-1);

	try {
		userFunction.run(ctx);

		// if we get here, then the user function either exited after being done (finite source)
		// or the function was canceled or stopped. For the finite source case, we should emit
		// a final watermark that indicates that we reached the end of event-time, and end inputs
		// of the operator chain
		if (!isCanceledOrStopped()) {
			// in theory, the subclasses of StreamSource may implement the BoundedOneInput interface,
			// so we still need the following call to end the input
			synchronized (lockingObject) {
				operatorChain.endHeadOperatorInput(1);
			}
		}
	} finally {
		if (latencyEmitter != null) {
			latencyEmitter.close();
		}
	}
}
 

public void run(final Object lockingObject,
		final StreamStatusMaintainer streamStatusMaintainer,
		final OperatorChain<?, ?> operatorChain) throws Exception {

	run(lockingObject, streamStatusMaintainer, output, operatorChain);
}
 

private static OperatorChain<?, ?> createOperatorChain(AbstractStreamOperator<?> operator) {
  return new OperatorChain<>(
      operator.getContainingTask(),
      StreamTask.createRecordWriters(
          operator.getOperatorConfig(), new MockEnvironmentBuilder().build()));
}
 

private static OperatorChain<?, ?> createOperatorChain(AbstractStreamOperator<?> operator) {
  return new OperatorChain<>(
      operator.getContainingTask(),
      StreamTask.createRecordWriterDelegate(
          operator.getOperatorConfig(), new MockEnvironmentBuilder().build()));
}
 

public StreamSourceOperatorTestHarness(StreamSource<T, F> operator, int maxParallelism, int parallelism, int subtaskIndex) throws Exception {
    super(operator, maxParallelism, parallelism, subtaskIndex);
    this.sourceOperator = operator;
    this.triggeredCheckpoints = new ConcurrentLinkedQueue<>();
    this.operatorChain = new OperatorChain<>(this.mockTask, StreamTask.createRecordWriterDelegate(this.config, this.getEnvironment()));
}
 

private <IN, OT extends StreamOperator<IN>> OperatorChain<IN, OT> createOperatorChain(
		StreamConfig streamConfig,
		Environment environment,
		StreamTask<IN, OT> task) {
	return new OperatorChain<>(task, StreamTask.createRecordWriters(streamConfig, environment));
}
 

private static OperatorChain<?, ?> createOperatorChain(AbstractStreamOperator<?> operator) {
	return new OperatorChain<>(
		operator.getContainingTask(),
		StreamTask.createRecordWriters(operator.getOperatorConfig(), new MockEnvironmentBuilder().build()));
}
 

private void testLatencyMarkEmission(int numberLatencyMarkers, OperatorSetupOperation operatorSetup) throws Exception {
	final List<StreamElement> output = new ArrayList<>();

	final TestProcessingTimeService testProcessingTimeService = new TestProcessingTimeService();
	testProcessingTimeService.setCurrentTime(0L);
	final List<Long> processingTimes = Arrays.asList(1L, 10L, 11L, 21L, maxProcessingTime);

	// regular stream source operator
	final StreamSource<Long, ProcessingTimeServiceSource> operator =
		new StreamSource<>(new ProcessingTimeServiceSource(testProcessingTimeService, processingTimes));

	operatorSetup.setupSourceOperator(operator, testProcessingTimeService);

	// run and wait to be stopped
	OperatorChain<?, ?> operatorChain = new OperatorChain<>(
		operator.getContainingTask(),
		StreamTask.createRecordWriters(operator.getOperatorConfig(), new MockEnvironmentBuilder().build()));
	try {
		operator.run(new Object(), mock(StreamStatusMaintainer.class), new CollectorOutput<Long>(output), operatorChain);
	} finally {
		operatorChain.releaseOutputs();
	}

	assertEquals(
		numberLatencyMarkers + 1, // + 1 is the final watermark element
		output.size());

	long timestamp = 0L;
	int expectedLatencyIndex = 0;

	int i = 0;
	// verify that its only latency markers + a final watermark
	for (; i < numberLatencyMarkers; i++) {
		StreamElement se = output.get(i);
		Assert.assertTrue(se.isLatencyMarker());
		Assert.assertEquals(operator.getOperatorID(), se.asLatencyMarker().getOperatorId());
		Assert.assertEquals(0, se.asLatencyMarker().getSubtaskIndex());

		// determines the next latency mark that should've been emitted
		// latency marks are emitted once per latencyMarkInterval,
		// as a result of which we never emit both 10 and 11
		while (timestamp > processingTimes.get(expectedLatencyIndex)) {
			expectedLatencyIndex++;
		}
		Assert.assertEquals(processingTimes.get(expectedLatencyIndex).longValue(), se.asLatencyMarker().getMarkedTime());

		timestamp += latencyMarkInterval;
	}

	Assert.assertTrue(output.get(i).isWatermark());
}
 

@SuppressWarnings("unchecked")
public StreamTwoInputProcessor(
		Collection<InputGate> inputGates1,
		Collection<InputGate> inputGates2,
		TypeSerializer<IN1> inputSerializer1,
		TypeSerializer<IN2> inputSerializer2,
		TwoInputStreamTask<IN1, IN2, ?> checkpointedTask,
		CheckpointingMode checkpointMode,
		Object lock,
		IOManager ioManager,
		Configuration taskManagerConfig,
		StreamStatusMaintainer streamStatusMaintainer,
		TwoInputStreamOperator<IN1, IN2, ?> streamOperator,
		TaskIOMetricGroup metrics,
		WatermarkGauge input1WatermarkGauge,
		WatermarkGauge input2WatermarkGauge,
		String taskName,
		OperatorChain<?, ?> operatorChain) throws IOException {

	final InputGate inputGate = InputGateUtil.createInputGate(inputGates1, inputGates2);

	this.barrierHandler = InputProcessorUtil.createCheckpointedInputGate(
		checkpointedTask,
		checkpointMode,
		ioManager,
		inputGate,
		taskManagerConfig,
		taskName);

	this.lock = checkNotNull(lock);

	StreamElementSerializer<IN1> ser1 = new StreamElementSerializer<>(inputSerializer1);
	this.deserializationDelegate1 = new NonReusingDeserializationDelegate<>(ser1);

	StreamElementSerializer<IN2> ser2 = new StreamElementSerializer<>(inputSerializer2);
	this.deserializationDelegate2 = new NonReusingDeserializationDelegate<>(ser2);

	// Initialize one deserializer per input channel
	this.recordDeserializers = new SpillingAdaptiveSpanningRecordDeserializer[inputGate.getNumberOfInputChannels()];

	for (int i = 0; i < recordDeserializers.length; i++) {
		recordDeserializers[i] = new SpillingAdaptiveSpanningRecordDeserializer<>(
			ioManager.getSpillingDirectoriesPaths());
	}

	// determine which unioned channels belong to input 1 and which belong to input 2
	int numInputChannels1 = 0;
	for (InputGate gate: inputGates1) {
		numInputChannels1 += gate.getNumberOfInputChannels();
	}

	this.numInputChannels1 = numInputChannels1;
	this.numInputChannels2 = inputGate.getNumberOfInputChannels() - numInputChannels1;

	this.firstStatus = StreamStatus.ACTIVE;
	this.secondStatus = StreamStatus.ACTIVE;

	this.streamStatusMaintainer = checkNotNull(streamStatusMaintainer);
	this.streamOperator = checkNotNull(streamOperator);

	this.statusWatermarkValve1 = new StatusWatermarkValve(numInputChannels1, new ForwardingValveOutputHandler1(streamOperator, lock));
	this.statusWatermarkValve2 = new StatusWatermarkValve(numInputChannels2, new ForwardingValveOutputHandler2(streamOperator, lock));

	this.input1WatermarkGauge = input1WatermarkGauge;
	this.input2WatermarkGauge = input2WatermarkGauge;
	metrics.gauge("checkpointAlignmentTime", barrierHandler::getAlignmentDurationNanos);

	this.operatorChain = checkNotNull(operatorChain);

	this.finishedChannels1 = new BitSet();
	this.finishedChannels2 = new BitSet();
}
 

public StreamTwoInputSelectableProcessor(
	Collection<InputGate> inputGates1,
	Collection<InputGate> inputGates2,
	TypeSerializer<IN1> inputSerializer1,
	TypeSerializer<IN2> inputSerializer2,
	StreamTask<?, ?> streamTask,
	CheckpointingMode checkpointingMode,
	Object lock,
	IOManager ioManager,
	Configuration taskManagerConfig,
	StreamStatusMaintainer streamStatusMaintainer,
	TwoInputStreamOperator<IN1, IN2, ?> streamOperator,
	WatermarkGauge input1WatermarkGauge,
	WatermarkGauge input2WatermarkGauge,
	String taskName,
	OperatorChain<?, ?> operatorChain) throws IOException {

	checkState(streamOperator instanceof InputSelectable);

	this.streamOperator = checkNotNull(streamOperator);
	this.inputSelector = (InputSelectable) streamOperator;

	this.lock = checkNotNull(lock);

	InputGate unionedInputGate1 = InputGateUtil.createInputGate(inputGates1.toArray(new InputGate[0]));
	InputGate unionedInputGate2 = InputGateUtil.createInputGate(inputGates2.toArray(new InputGate[0]));

	// create a Input instance for each input
	CheckpointedInputGate[] checkpointedInputGates = InputProcessorUtil.createCheckpointedInputGatePair(
		streamTask,
		checkpointingMode,
		ioManager,
		unionedInputGate1,
		unionedInputGate2,
		taskManagerConfig,
		taskName);
	checkState(checkpointedInputGates.length == 2);
	this.input1 = new StreamTaskNetworkInput(checkpointedInputGates[0], inputSerializer1, ioManager, 0);
	this.input2 = new StreamTaskNetworkInput(checkpointedInputGates[1], inputSerializer2, ioManager, 1);

	this.statusWatermarkValve1 = new StatusWatermarkValve(
		unionedInputGate1.getNumberOfInputChannels(),
		new ForwardingValveOutputHandler(streamOperator, lock, streamStatusMaintainer, input1WatermarkGauge, 0));
	this.statusWatermarkValve2 = new StatusWatermarkValve(
		unionedInputGate2.getNumberOfInputChannels(),
		new ForwardingValveOutputHandler(streamOperator, lock, streamStatusMaintainer, input2WatermarkGauge, 1));

	this.operatorChain = checkNotNull(operatorChain);

	this.firstStatus = StreamStatus.ACTIVE;
	this.secondStatus = StreamStatus.ACTIVE;

	this.availableInputsMask = (int) new InputSelection.Builder().select(1).select(2).build().getInputMask();

	this.lastReadInputIndex = 1; // always try to read from the first input

	this.isPrepared = false;
}
 

@SuppressWarnings("unchecked")
public StreamOneInputProcessor(
		InputGate[] inputGates,
		TypeSerializer<IN> inputSerializer,
		StreamTask<?, ?> checkpointedTask,
		CheckpointingMode checkpointMode,
		Object lock,
		IOManager ioManager,
		Configuration taskManagerConfig,
		StreamStatusMaintainer streamStatusMaintainer,
		OneInputStreamOperator<IN, ?> streamOperator,
		TaskIOMetricGroup metrics,
		WatermarkGauge watermarkGauge,
		String taskName,
		OperatorChain<?, ?> operatorChain) throws IOException {

	InputGate inputGate = InputGateUtil.createInputGate(inputGates);

	CheckpointedInputGate barrierHandler = InputProcessorUtil.createCheckpointedInputGate(
		checkpointedTask,
		checkpointMode,
		ioManager,
		inputGate,
		taskManagerConfig,
		taskName);
	this.input = new StreamTaskNetworkInput(barrierHandler, inputSerializer, ioManager, 0);

	this.lock = checkNotNull(lock);

	this.streamStatusMaintainer = checkNotNull(streamStatusMaintainer);
	this.streamOperator = checkNotNull(streamOperator);

	this.statusWatermarkValve = new StatusWatermarkValve(
		inputGate.getNumberOfInputChannels(),
		new ForwardingValveOutputHandler(streamOperator, lock));

	this.watermarkGauge = watermarkGauge;
	metrics.gauge("checkpointAlignmentTime", barrierHandler::getAlignmentDurationNanos);

	this.operatorChain = checkNotNull(operatorChain);
}
 

public void run(final Object lockingObject,
		final StreamStatusMaintainer streamStatusMaintainer,
		final Output<StreamRecord<OUT>> collector,
		final OperatorChain<?, ?> operatorChain) throws Exception {

	final TimeCharacteristic timeCharacteristic = getOperatorConfig().getTimeCharacteristic();

	final Configuration configuration = this.getContainingTask().getEnvironment().getTaskManagerInfo().getConfiguration();
	final long latencyTrackingInterval = getExecutionConfig().isLatencyTrackingConfigured()
		? getExecutionConfig().getLatencyTrackingInterval()
		: configuration.getLong(MetricOptions.LATENCY_INTERVAL);

	LatencyMarksEmitter<OUT> latencyEmitter = null;
	if (latencyTrackingInterval > 0) {
		latencyEmitter = new LatencyMarksEmitter<>(
			getProcessingTimeService(),
			collector,
			latencyTrackingInterval,
			this.getOperatorID(),
			getRuntimeContext().getIndexOfThisSubtask());
	}

	final long watermarkInterval = getRuntimeContext().getExecutionConfig().getAutoWatermarkInterval();

	this.ctx = StreamSourceContexts.getSourceContext(
		timeCharacteristic,
		getProcessingTimeService(),
		lockingObject,
		streamStatusMaintainer,
		collector,
		watermarkInterval,
		-1);

	try {
		userFunction.run(ctx);

		// if we get here, then the user function either exited after being done (finite source)
		// or the function was canceled or stopped. For the finite source case, we should emit
		// a final watermark that indicates that we reached the end of event-time, and end inputs
		// of the operator chain
		if (!isCanceledOrStopped()) {
			advanceToEndOfEventTime();

			synchronized (lockingObject) {
				operatorChain.endInput(1);
			}
		}
	} finally {
		// make sure that the context is closed in any case
		ctx.close();
		if (latencyEmitter != null) {
			latencyEmitter.close();
		}
	}
}
 

public void run(final Object lockingObject,
		final StreamStatusMaintainer streamStatusMaintainer,
		final OperatorChain<?, ?> operatorChain) throws Exception {

	run(lockingObject, streamStatusMaintainer, output, operatorChain);
}