org.apache.flink.streaming.runtime.streamstatus.StatusWatermarkValve Java Examples

@SuppressWarnings("unchecked")
public StreamTaskNetworkInput(
		CheckpointedInputGate checkpointedInputGate,
		TypeSerializer<?> inputSerializer,
		IOManager ioManager,
		StatusWatermarkValve statusWatermarkValve,
		int inputIndex) {
	this.checkpointedInputGate = checkpointedInputGate;
	this.deserializationDelegate = new NonReusingDeserializationDelegate<>(
		new StreamElementSerializer<>(inputSerializer));

	// Initialize one deserializer per input channel
	this.recordDeserializers = new SpillingAdaptiveSpanningRecordDeserializer[checkpointedInputGate.getNumberOfInputChannels()];
	for (int i = 0; i < recordDeserializers.length; i++) {
		recordDeserializers[i] = new SpillingAdaptiveSpanningRecordDeserializer<>(
			ioManager.getSpillingDirectoriesPaths());
	}

	this.statusWatermarkValve = checkNotNull(statusWatermarkValve);
	this.inputIndex = inputIndex;
	this.channelIndexes = getChannelIndexes(checkpointedInputGate);
}
 

@VisibleForTesting
StreamTaskNetworkInput(
	CheckpointedInputGate checkpointedInputGate,
	TypeSerializer<?> inputSerializer,
	StatusWatermarkValve statusWatermarkValve,
	int inputIndex,
	RecordDeserializer<DeserializationDelegate<StreamElement>>[] recordDeserializers) {

	this.checkpointedInputGate = checkpointedInputGate;
	this.deserializationDelegate = new NonReusingDeserializationDelegate<>(
		new StreamElementSerializer<>(inputSerializer));
	this.recordDeserializers = recordDeserializers;
	this.statusWatermarkValve = statusWatermarkValve;
	this.inputIndex = inputIndex;
	this.channelIndexes = getChannelIndexes(checkpointedInputGate);
}
 

private StreamTaskInput<IN> createTaskInput(CheckpointedInputGate inputGate, DataOutput<IN> output) {
	int numberOfInputChannels = inputGate.getNumberOfInputChannels();
	StatusWatermarkValve statusWatermarkValve = new StatusWatermarkValve(numberOfInputChannels, output);

	TypeSerializer<IN> inSerializer = configuration.getTypeSerializerIn1(getUserCodeClassLoader());
	return new StreamTaskNetworkInput<>(
		inputGate,
		inSerializer,
		getEnvironment().getIOManager(),
		statusWatermarkValve,
		0);
}
 

public StreamMultipleInputProcessor(
		CheckpointedInputGate[] checkpointedInputGates,
		TypeSerializer<?>[] inputSerializers,
		IOManager ioManager,
		StreamStatusMaintainer streamStatusMaintainer,
		MultipleInputStreamOperator<?> streamOperator,
		MultipleInputSelectionHandler inputSelectionHandler,
		WatermarkGauge[] inputWatermarkGauges,
		OperatorChain<?, ?> operatorChain,
		Counter numRecordsIn) {

	this.inputSelectionHandler = checkNotNull(inputSelectionHandler);

	List<Input> inputs = streamOperator.getInputs();
	int inputsCount = inputs.size();

	this.inputProcessors = new InputProcessor[inputsCount];
	this.streamStatuses = new StreamStatus[inputsCount];
	this.numRecordsIn = numRecordsIn;

	for (int i = 0; i < inputsCount; i++) {
		streamStatuses[i] = StreamStatus.ACTIVE;
		StreamTaskNetworkOutput dataOutput = new StreamTaskNetworkOutput<>(
			inputs.get(i),
			streamStatusMaintainer,
			inputWatermarkGauges[i],
			i);

		inputProcessors[i] = new InputProcessor(
			dataOutput,
			new StreamTaskNetworkInput<>(
				checkpointedInputGates[i],
				inputSerializers[i],
				ioManager,
				new StatusWatermarkValve(checkpointedInputGates[i].getNumberOfInputChannels(), dataOutput),
				i));
	}

	this.operatorChain = checkNotNull(operatorChain);
}
 

private StreamTaskNetworkInput<Long> createInput(CheckpointBarrierHandler handler, SingleInputGate inputGate) {
	return new StreamTaskNetworkInput<>(
		new CheckpointedInputGate(inputGate, handler),
		LongSerializer.INSTANCE,
		new StatusWatermarkValve(inputGate.getNumberOfInputChannels(), new NoOpDataOutput<>()),
		inputGate.getGateIndex(),
		createDeserializers(inputGate.getNumberOfInputChannels()));
}
 

@Test
public void testReleasingDeserializerTimely()
	throws Exception {

	int numInputChannels = 2;
	LongSerializer inSerializer = LongSerializer.INSTANCE;
	StreamTestSingleInputGate inputGate = new StreamTestSingleInputGate<>(numInputChannels, 0, inSerializer, 1024);

	TestRecordDeserializer[] deserializers = new TestRecordDeserializer[numInputChannels];
	for (int i = 0; i < deserializers.length; i++) {
		deserializers[i] = new TestRecordDeserializer(ioManager.getSpillingDirectoriesPaths());
	}

	TestRecordDeserializer[] copiedDeserializers = Arrays.copyOf(deserializers, deserializers.length);
	DataOutput output = new NoOpDataOutput<>();
	StreamTaskNetworkInput input = new StreamTaskNetworkInput<>(
		new CheckpointedInputGate(
			inputGate.getInputGate(),
			new CheckpointBarrierTracker(1, new DummyCheckpointInvokable())),
		inSerializer,
		new StatusWatermarkValve(1, output),
		0,
		deserializers);

	for (int i = 0; i < numInputChannels; i++) {
		assertNotNull(deserializers[i]);
		inputGate.sendEvent(EndOfPartitionEvent.INSTANCE, i);
		input.emitNext(output);
		assertNull(deserializers[i]);
		assertTrue(copiedDeserializers[i].isCleared());
	}
}
 

private StreamTaskNetworkInput createStreamTaskNetworkInput(List<BufferOrEvent> buffers, DataOutput output) {
	return new StreamTaskNetworkInput<>(
		new CheckpointedInputGate(
			new MockInputGate(1, buffers, false),
			new CheckpointBarrierTracker(1, new DummyCheckpointInvokable())),
		LongSerializer.INSTANCE,
		ioManager,
		new StatusWatermarkValve(1, output),
		0);
}
 

@SuppressWarnings("unchecked")
public StreamInputProcessor(
		InputGate[] inputGates,
		TypeSerializer<IN> inputSerializer,
		StreamTask<?, ?> checkpointedTask,
		CheckpointingMode checkpointMode,
		Object lock,
		IOManager ioManager,
		Configuration taskManagerConfig,
		StreamStatusMaintainer streamStatusMaintainer,
		OneInputStreamOperator<IN, ?> streamOperator,
		TaskIOMetricGroup metrics,
		WatermarkGauge watermarkGauge) throws IOException {

	InputGate inputGate = InputGateUtil.createInputGate(inputGates);

	this.barrierHandler = InputProcessorUtil.createCheckpointBarrierHandler(
		checkpointedTask, checkpointMode, ioManager, inputGate, taskManagerConfig);

	this.lock = checkNotNull(lock);

	StreamElementSerializer<IN> ser = new StreamElementSerializer<>(inputSerializer);
	this.deserializationDelegate = new NonReusingDeserializationDelegate<>(ser);

	// Initialize one deserializer per input channel
	this.recordDeserializers = new SpillingAdaptiveSpanningRecordDeserializer[inputGate.getNumberOfInputChannels()];

	for (int i = 0; i < recordDeserializers.length; i++) {
		recordDeserializers[i] = new SpillingAdaptiveSpanningRecordDeserializer<>(
			ioManager.getSpillingDirectoriesPaths());
	}

	this.numInputChannels = inputGate.getNumberOfInputChannels();

	this.streamStatusMaintainer = checkNotNull(streamStatusMaintainer);
	this.streamOperator = checkNotNull(streamOperator);

	this.statusWatermarkValve = new StatusWatermarkValve(
			numInputChannels,
			new ForwardingValveOutputHandler(streamOperator, lock));

	this.watermarkGauge = watermarkGauge;
	metrics.gauge("checkpointAlignmentTime", barrierHandler::getAlignmentDurationNanos);
}
 

@SuppressWarnings("unchecked")
public StreamTwoInputProcessor(
		Collection<InputGate> inputGates1,
		Collection<InputGate> inputGates2,
		TypeSerializer<IN1> inputSerializer1,
		TypeSerializer<IN2> inputSerializer2,
		TwoInputStreamTask<IN1, IN2, ?> checkpointedTask,
		CheckpointingMode checkpointMode,
		Object lock,
		IOManager ioManager,
		Configuration taskManagerConfig,
		StreamStatusMaintainer streamStatusMaintainer,
		TwoInputStreamOperator<IN1, IN2, ?> streamOperator,
		TaskIOMetricGroup metrics,
		WatermarkGauge input1WatermarkGauge,
		WatermarkGauge input2WatermarkGauge) throws IOException {

	final InputGate inputGate = InputGateUtil.createInputGate(inputGates1, inputGates2);

	this.barrierHandler = InputProcessorUtil.createCheckpointBarrierHandler(
		checkpointedTask, checkpointMode, ioManager, inputGate, taskManagerConfig);

	this.lock = checkNotNull(lock);

	StreamElementSerializer<IN1> ser1 = new StreamElementSerializer<>(inputSerializer1);
	this.deserializationDelegate1 = new NonReusingDeserializationDelegate<>(ser1);

	StreamElementSerializer<IN2> ser2 = new StreamElementSerializer<>(inputSerializer2);
	this.deserializationDelegate2 = new NonReusingDeserializationDelegate<>(ser2);

	// Initialize one deserializer per input channel
	this.recordDeserializers = new SpillingAdaptiveSpanningRecordDeserializer[inputGate.getNumberOfInputChannels()];

	for (int i = 0; i < recordDeserializers.length; i++) {
		recordDeserializers[i] = new SpillingAdaptiveSpanningRecordDeserializer<>(
			ioManager.getSpillingDirectoriesPaths());
	}

	// determine which unioned channels belong to input 1 and which belong to input 2
	int numInputChannels1 = 0;
	for (InputGate gate: inputGates1) {
		numInputChannels1 += gate.getNumberOfInputChannels();
	}

	this.numInputChannels1 = numInputChannels1;
	this.numInputChannels2 = inputGate.getNumberOfInputChannels() - numInputChannels1;

	this.firstStatus = StreamStatus.ACTIVE;
	this.secondStatus = StreamStatus.ACTIVE;

	this.streamStatusMaintainer = checkNotNull(streamStatusMaintainer);
	this.streamOperator = checkNotNull(streamOperator);

	this.statusWatermarkValve1 = new StatusWatermarkValve(numInputChannels1, new ForwardingValveOutputHandler1(streamOperator, lock));
	this.statusWatermarkValve2 = new StatusWatermarkValve(numInputChannels2, new ForwardingValveOutputHandler2(streamOperator, lock));

	this.input1WatermarkGauge = input1WatermarkGauge;
	this.input2WatermarkGauge = input2WatermarkGauge;
	metrics.gauge("checkpointAlignmentTime", barrierHandler::getAlignmentDurationNanos);
}
 

@SuppressWarnings("unchecked")
public StreamOneInputProcessor(
		InputGate[] inputGates,
		TypeSerializer<IN> inputSerializer,
		StreamTask<?, ?> checkpointedTask,
		CheckpointingMode checkpointMode,
		Object lock,
		IOManager ioManager,
		Configuration taskManagerConfig,
		StreamStatusMaintainer streamStatusMaintainer,
		OneInputStreamOperator<IN, ?> streamOperator,
		TaskIOMetricGroup metrics,
		WatermarkGauge watermarkGauge,
		String taskName,
		OperatorChain<?, ?> operatorChain) throws IOException {

	InputGate inputGate = InputGateUtil.createInputGate(inputGates);

	CheckpointedInputGate barrierHandler = InputProcessorUtil.createCheckpointedInputGate(
		checkpointedTask,
		checkpointMode,
		ioManager,
		inputGate,
		taskManagerConfig,
		taskName);
	this.input = new StreamTaskNetworkInput(barrierHandler, inputSerializer, ioManager, 0);

	this.lock = checkNotNull(lock);

	this.streamStatusMaintainer = checkNotNull(streamStatusMaintainer);
	this.streamOperator = checkNotNull(streamOperator);

	this.statusWatermarkValve = new StatusWatermarkValve(
		inputGate.getNumberOfInputChannels(),
		new ForwardingValveOutputHandler(streamOperator, lock));

	this.watermarkGauge = watermarkGauge;
	metrics.gauge("checkpointAlignmentTime", barrierHandler::getAlignmentDurationNanos);

	this.operatorChain = checkNotNull(operatorChain);
}
 

public StreamTwoInputSelectableProcessor(
	Collection<InputGate> inputGates1,
	Collection<InputGate> inputGates2,
	TypeSerializer<IN1> inputSerializer1,
	TypeSerializer<IN2> inputSerializer2,
	StreamTask<?, ?> streamTask,
	CheckpointingMode checkpointingMode,
	Object lock,
	IOManager ioManager,
	Configuration taskManagerConfig,
	StreamStatusMaintainer streamStatusMaintainer,
	TwoInputStreamOperator<IN1, IN2, ?> streamOperator,
	WatermarkGauge input1WatermarkGauge,
	WatermarkGauge input2WatermarkGauge,
	String taskName,
	OperatorChain<?, ?> operatorChain) throws IOException {

	checkState(streamOperator instanceof InputSelectable);

	this.streamOperator = checkNotNull(streamOperator);
	this.inputSelector = (InputSelectable) streamOperator;

	this.lock = checkNotNull(lock);

	InputGate unionedInputGate1 = InputGateUtil.createInputGate(inputGates1.toArray(new InputGate[0]));
	InputGate unionedInputGate2 = InputGateUtil.createInputGate(inputGates2.toArray(new InputGate[0]));

	// create a Input instance for each input
	CheckpointedInputGate[] checkpointedInputGates = InputProcessorUtil.createCheckpointedInputGatePair(
		streamTask,
		checkpointingMode,
		ioManager,
		unionedInputGate1,
		unionedInputGate2,
		taskManagerConfig,
		taskName);
	checkState(checkpointedInputGates.length == 2);
	this.input1 = new StreamTaskNetworkInput(checkpointedInputGates[0], inputSerializer1, ioManager, 0);
	this.input2 = new StreamTaskNetworkInput(checkpointedInputGates[1], inputSerializer2, ioManager, 1);

	this.statusWatermarkValve1 = new StatusWatermarkValve(
		unionedInputGate1.getNumberOfInputChannels(),
		new ForwardingValveOutputHandler(streamOperator, lock, streamStatusMaintainer, input1WatermarkGauge, 0));
	this.statusWatermarkValve2 = new StatusWatermarkValve(
		unionedInputGate2.getNumberOfInputChannels(),
		new ForwardingValveOutputHandler(streamOperator, lock, streamStatusMaintainer, input2WatermarkGauge, 1));

	this.operatorChain = checkNotNull(operatorChain);

	this.firstStatus = StreamStatus.ACTIVE;
	this.secondStatus = StreamStatus.ACTIVE;

	this.availableInputsMask = (int) new InputSelection.Builder().select(1).select(2).build().getInputMask();

	this.lastReadInputIndex = 1; // always try to read from the first input

	this.isPrepared = false;
}
 

@SuppressWarnings("unchecked")
public StreamTwoInputProcessor(
		Collection<InputGate> inputGates1,
		Collection<InputGate> inputGates2,
		TypeSerializer<IN1> inputSerializer1,
		TypeSerializer<IN2> inputSerializer2,
		TwoInputStreamTask<IN1, IN2, ?> checkpointedTask,
		CheckpointingMode checkpointMode,
		Object lock,
		IOManager ioManager,
		Configuration taskManagerConfig,
		StreamStatusMaintainer streamStatusMaintainer,
		TwoInputStreamOperator<IN1, IN2, ?> streamOperator,
		TaskIOMetricGroup metrics,
		WatermarkGauge input1WatermarkGauge,
		WatermarkGauge input2WatermarkGauge,
		String taskName,
		OperatorChain<?, ?> operatorChain) throws IOException {

	final InputGate inputGate = InputGateUtil.createInputGate(inputGates1, inputGates2);

	this.barrierHandler = InputProcessorUtil.createCheckpointedInputGate(
		checkpointedTask,
		checkpointMode,
		ioManager,
		inputGate,
		taskManagerConfig,
		taskName);

	this.lock = checkNotNull(lock);

	StreamElementSerializer<IN1> ser1 = new StreamElementSerializer<>(inputSerializer1);
	this.deserializationDelegate1 = new NonReusingDeserializationDelegate<>(ser1);

	StreamElementSerializer<IN2> ser2 = new StreamElementSerializer<>(inputSerializer2);
	this.deserializationDelegate2 = new NonReusingDeserializationDelegate<>(ser2);

	// Initialize one deserializer per input channel
	this.recordDeserializers = new SpillingAdaptiveSpanningRecordDeserializer[inputGate.getNumberOfInputChannels()];

	for (int i = 0; i < recordDeserializers.length; i++) {
		recordDeserializers[i] = new SpillingAdaptiveSpanningRecordDeserializer<>(
			ioManager.getSpillingDirectoriesPaths());
	}

	// determine which unioned channels belong to input 1 and which belong to input 2
	int numInputChannels1 = 0;
	for (InputGate gate: inputGates1) {
		numInputChannels1 += gate.getNumberOfInputChannels();
	}

	this.numInputChannels1 = numInputChannels1;
	this.numInputChannels2 = inputGate.getNumberOfInputChannels() - numInputChannels1;

	this.firstStatus = StreamStatus.ACTIVE;
	this.secondStatus = StreamStatus.ACTIVE;

	this.streamStatusMaintainer = checkNotNull(streamStatusMaintainer);
	this.streamOperator = checkNotNull(streamOperator);

	this.statusWatermarkValve1 = new StatusWatermarkValve(numInputChannels1, new ForwardingValveOutputHandler1(streamOperator, lock));
	this.statusWatermarkValve2 = new StatusWatermarkValve(numInputChannels2, new ForwardingValveOutputHandler2(streamOperator, lock));

	this.input1WatermarkGauge = input1WatermarkGauge;
	this.input2WatermarkGauge = input2WatermarkGauge;
	metrics.gauge("checkpointAlignmentTime", barrierHandler::getAlignmentDurationNanos);

	this.operatorChain = checkNotNull(operatorChain);

	this.finishedChannels1 = new BitSet();
	this.finishedChannels2 = new BitSet();
}
 

public StreamTwoInputProcessor(
		CheckpointedInputGate[] checkpointedInputGates,
		TypeSerializer<IN1> inputSerializer1,
		TypeSerializer<IN2> inputSerializer2,
		IOManager ioManager,
		StreamStatusMaintainer streamStatusMaintainer,
		TwoInputStreamOperator<IN1, IN2, ?> streamOperator,
		TwoInputSelectionHandler inputSelectionHandler,
		WatermarkGauge input1WatermarkGauge,
		WatermarkGauge input2WatermarkGauge,
		OperatorChain<?, ?> operatorChain,
		Counter numRecordsIn) {

	this.inputSelectionHandler = checkNotNull(inputSelectionHandler);

	this.output1 = new StreamTaskNetworkOutput<>(
		streamOperator,
		record -> processRecord1(record, streamOperator, numRecordsIn),
		streamStatusMaintainer,
		input1WatermarkGauge,
		0);
	this.output2 = new StreamTaskNetworkOutput<>(
		streamOperator,
		record -> processRecord2(record, streamOperator, numRecordsIn),
		streamStatusMaintainer,
		input2WatermarkGauge,
		1);

	this.input1 = new StreamTaskNetworkInput<>(
		checkpointedInputGates[0],
		inputSerializer1,
		ioManager,
		new StatusWatermarkValve(checkpointedInputGates[0].getNumberOfInputChannels(), output1),
		0);
	this.input2 = new StreamTaskNetworkInput<>(
		checkpointedInputGates[1],
		inputSerializer2,
		ioManager,
		new StatusWatermarkValve(checkpointedInputGates[1].getNumberOfInputChannels(), output2),
		1);

	this.operatorChain = checkNotNull(operatorChain);
}
 

@Test
public void testSnapshotAfterEndOfPartition() throws Exception {
	int numInputChannels = 1;
	int channelId = 0;
	int checkpointId = 0;

	VerifyRecordsDataOutput<Long> output = new VerifyRecordsDataOutput<>();
	LongSerializer inSerializer = LongSerializer.INSTANCE;
	StreamTestSingleInputGate<Long> inputGate = new StreamTestSingleInputGate<>(numInputChannels, 0, inSerializer, 1024);
	TestRecordDeserializer[] deserializers = createDeserializers(numInputChannels);
	StreamTaskNetworkInput<Long> input = new StreamTaskNetworkInput<>(
		new CheckpointedInputGate(
			inputGate.getInputGate(),
			new CheckpointBarrierUnaligner(
				TestSubtaskCheckpointCoordinator.INSTANCE,
				"test",
				new DummyCheckpointInvokable(),
				inputGate.getInputGate())),
		inSerializer,
		new StatusWatermarkValve(numInputChannels, output),
		0,
		deserializers);

	inputGate.sendEvent(
		new CheckpointBarrier(checkpointId, 0L, CheckpointOptions.forCheckpointWithDefaultLocation()),
		channelId);
	inputGate.sendElement(new StreamRecord<>(42L), channelId);

	assertHasNextElement(input, output);
	assertHasNextElement(input, output);
	assertEquals(1, output.getNumberOfEmittedRecords());

	// send EndOfPartitionEvent and ensure that deserializer has been released
	inputGate.sendEvent(EndOfPartitionEvent.INSTANCE, channelId);
	input.emitNext(output);
	assertNull(deserializers[channelId]);

	// now snapshot all inflight buffers
	CompletableFuture<Void> completableFuture = input.prepareSnapshot(ChannelStateWriter.NO_OP, checkpointId);
	completableFuture.join();
}