org.apache.flink.streaming.runtime.metrics.WatermarkGauge Java Examples

private ForwardingValveOutputHandler(
	TwoInputStreamOperator<IN1, IN2, ?> operator,
	Object lock,
	StreamStatusMaintainer streamStatusMaintainer,
	WatermarkGauge inputWatermarkGauge,
	int inputIndex) {

	this.operator = checkNotNull(operator);
	this.lock = checkNotNull(lock);

	this.streamStatusMaintainer = checkNotNull(streamStatusMaintainer);

	this.inputWatermarkGauge = inputWatermarkGauge;

	this.inputIndex = inputIndex;
}
 

/**
 * Constructor for initialization, possibly with initial state (recovery / savepoint / etc).
 *
 * @param env The task environment for this task.
 */
public TwoInputStreamTask(Environment env) {
	super(env);
	input1WatermarkGauge = new WatermarkGauge();
	input2WatermarkGauge = new WatermarkGauge();
	minInputWatermarkGauge = new MinWatermarkGauge(input1WatermarkGauge, input2WatermarkGauge);
}
 

/**
 * Constructor for initialization, possibly with initial state (recovery / savepoint / etc).
 *
 * @param env The task environment for this task.
 */
public AbstractTwoInputStreamTask(Environment env) {
	super(env);

	input1WatermarkGauge = new WatermarkGauge();
	input2WatermarkGauge = new WatermarkGauge();
	minInputWatermarkGauge = new MinWatermarkGauge(input1WatermarkGauge, input2WatermarkGauge);
}
 

private StreamTaskNetworkOutput(
		OneInputStreamOperator<IN, ?> operator,
		StreamStatusMaintainer streamStatusMaintainer,
		WatermarkGauge watermarkGauge,
		Counter numRecordsIn) {
	super(streamStatusMaintainer);

	this.operator = checkNotNull(operator);
	this.watermarkGauge = checkNotNull(watermarkGauge);
	this.numRecordsIn = checkNotNull(numRecordsIn);
}
 

/**
 * Constructor for initialization, possibly with initial state (recovery / savepoint / etc).
 *
 * @param env The task environment for this task.
 */
public AbstractTwoInputStreamTask(Environment env) throws Exception {
	super(env);

	input1WatermarkGauge = new WatermarkGauge();
	input2WatermarkGauge = new WatermarkGauge();
	minInputWatermarkGauge = new MinWatermarkGauge(input1WatermarkGauge, input2WatermarkGauge);
}
 

@Override
public void init() throws Exception {
	StreamConfig configuration = getConfiguration();
	ClassLoader userClassLoader = getUserCodeClassLoader();

	TypeSerializer<?>[] inputDeserializers = configuration.getTypeSerializersIn(userClassLoader);

	ArrayList<IndexedInputGate>[] inputLists = new ArrayList[inputDeserializers.length];
	WatermarkGauge[] watermarkGauges = new WatermarkGauge[inputDeserializers.length];

	for (int i = 0; i < inputDeserializers.length; i++) {
		inputLists[i] = new ArrayList<>();
		watermarkGauges[i] = new WatermarkGauge();
		headOperator.getMetricGroup().gauge(MetricNames.currentInputWatermarkName(i + 1), watermarkGauges[i]);
	}

	MinWatermarkGauge minInputWatermarkGauge = new MinWatermarkGauge(watermarkGauges);
	headOperator.getMetricGroup().gauge(MetricNames.IO_CURRENT_INPUT_WATERMARK, minInputWatermarkGauge);

	List<StreamEdge> inEdges = configuration.getInPhysicalEdges(userClassLoader);
	int numberOfInputs = configuration.getNumberOfInputs();

	for (int i = 0; i < numberOfInputs; i++) {
		int inputType = inEdges.get(i).getTypeNumber();
		IndexedInputGate reader = getEnvironment().getInputGate(i);
		inputLists[inputType - 1].add(reader);
	}

	createInputProcessor(inputLists, inputDeserializers, watermarkGauges);

	// wrap watermark gauge since registered metrics must be unique
	getEnvironment().getMetricGroup().gauge(MetricNames.IO_CURRENT_INPUT_WATERMARK, minInputWatermarkGauge::getValue);
}
 

protected void createInputProcessor(
		List<IndexedInputGate>[] inputGates,
		TypeSerializer<?>[] inputDeserializers,
		WatermarkGauge[] inputWatermarkGauges) {
	MultipleInputSelectionHandler selectionHandler = new MultipleInputSelectionHandler(
		headOperator instanceof InputSelectable ? (InputSelectable) headOperator : null,
		inputGates.length);

	CheckpointedInputGate[] checkpointedInputGates = InputProcessorUtil.createCheckpointedMultipleInputGate(
		this,
		getConfiguration(),
		getCheckpointCoordinator(),
		getEnvironment().getMetricGroup().getIOMetricGroup(),
		getTaskNameWithSubtaskAndId(),
		inputGates);
	checkState(checkpointedInputGates.length == inputGates.length);

	inputProcessor = new StreamMultipleInputProcessor(
		checkpointedInputGates,
		inputDeserializers,
		getEnvironment().getIOManager(),
		getStreamStatusMaintainer(),
		headOperator,
		selectionHandler,
		inputWatermarkGauges,
		operatorChain,
		setupNumRecordsInCounter(headOperator));
}
 

private StreamTaskNetworkOutput(
		TwoInputStreamOperator<IN1, IN2, ?> operator,
		ThrowingConsumer<StreamRecord<T>, Exception> recordConsumer,
		StreamStatusMaintainer streamStatusMaintainer,
		WatermarkGauge inputWatermarkGauge,
		int inputIndex) {
	super(streamStatusMaintainer);

	this.operator = checkNotNull(operator);
	this.recordConsumer = checkNotNull(recordConsumer);
	this.inputWatermarkGauge = checkNotNull(inputWatermarkGauge);
	this.inputIndex = inputIndex;
}
 

public StreamMultipleInputProcessor(
		CheckpointedInputGate[] checkpointedInputGates,
		TypeSerializer<?>[] inputSerializers,
		IOManager ioManager,
		StreamStatusMaintainer streamStatusMaintainer,
		MultipleInputStreamOperator<?> streamOperator,
		MultipleInputSelectionHandler inputSelectionHandler,
		WatermarkGauge[] inputWatermarkGauges,
		OperatorChain<?, ?> operatorChain,
		Counter numRecordsIn) {

	this.inputSelectionHandler = checkNotNull(inputSelectionHandler);

	List<Input> inputs = streamOperator.getInputs();
	int inputsCount = inputs.size();

	this.inputProcessors = new InputProcessor[inputsCount];
	this.streamStatuses = new StreamStatus[inputsCount];
	this.numRecordsIn = numRecordsIn;

	for (int i = 0; i < inputsCount; i++) {
		streamStatuses[i] = StreamStatus.ACTIVE;
		StreamTaskNetworkOutput dataOutput = new StreamTaskNetworkOutput<>(
			inputs.get(i),
			streamStatusMaintainer,
			inputWatermarkGauges[i],
			i);

		inputProcessors[i] = new InputProcessor(
			dataOutput,
			new StreamTaskNetworkInput<>(
				checkpointedInputGates[i],
				inputSerializers[i],
				ioManager,
				new StatusWatermarkValve(checkpointedInputGates[i].getNumberOfInputChannels(), dataOutput),
				i));
	}

	this.operatorChain = checkNotNull(operatorChain);
}
 

private StreamTaskNetworkOutput(
		Input<T> input,
		StreamStatusMaintainer streamStatusMaintainer,
		WatermarkGauge inputWatermarkGauge,
		int inputIndex) {
	super(streamStatusMaintainer);

	this.input = checkNotNull(input);
	this.inputWatermarkGauge = checkNotNull(inputWatermarkGauge);
	this.inputIndex = inputIndex;
}
 

@SuppressWarnings("unchecked")
public StreamInputProcessor(
		InputGate[] inputGates,
		TypeSerializer<IN> inputSerializer,
		StreamTask<?, ?> checkpointedTask,
		CheckpointingMode checkpointMode,
		Object lock,
		IOManager ioManager,
		Configuration taskManagerConfig,
		StreamStatusMaintainer streamStatusMaintainer,
		OneInputStreamOperator<IN, ?> streamOperator,
		TaskIOMetricGroup metrics,
		WatermarkGauge watermarkGauge) throws IOException {

	InputGate inputGate = InputGateUtil.createInputGate(inputGates);

	this.barrierHandler = InputProcessorUtil.createCheckpointBarrierHandler(
		checkpointedTask, checkpointMode, ioManager, inputGate, taskManagerConfig);

	this.lock = checkNotNull(lock);

	StreamElementSerializer<IN> ser = new StreamElementSerializer<>(inputSerializer);
	this.deserializationDelegate = new NonReusingDeserializationDelegate<>(ser);

	// Initialize one deserializer per input channel
	this.recordDeserializers = new SpillingAdaptiveSpanningRecordDeserializer[inputGate.getNumberOfInputChannels()];

	for (int i = 0; i < recordDeserializers.length; i++) {
		recordDeserializers[i] = new SpillingAdaptiveSpanningRecordDeserializer<>(
			ioManager.getSpillingDirectoriesPaths());
	}

	this.numInputChannels = inputGate.getNumberOfInputChannels();

	this.streamStatusMaintainer = checkNotNull(streamStatusMaintainer);
	this.streamOperator = checkNotNull(streamOperator);

	this.statusWatermarkValve = new StatusWatermarkValve(
			numInputChannels,
			new ForwardingValveOutputHandler(streamOperator, lock));

	this.watermarkGauge = watermarkGauge;
	metrics.gauge("checkpointAlignmentTime", barrierHandler::getAlignmentDurationNanos);
}
 

@SuppressWarnings("unchecked")
public StreamTwoInputProcessor(
		Collection<InputGate> inputGates1,
		Collection<InputGate> inputGates2,
		TypeSerializer<IN1> inputSerializer1,
		TypeSerializer<IN2> inputSerializer2,
		TwoInputStreamTask<IN1, IN2, ?> checkpointedTask,
		CheckpointingMode checkpointMode,
		Object lock,
		IOManager ioManager,
		Configuration taskManagerConfig,
		StreamStatusMaintainer streamStatusMaintainer,
		TwoInputStreamOperator<IN1, IN2, ?> streamOperator,
		TaskIOMetricGroup metrics,
		WatermarkGauge input1WatermarkGauge,
		WatermarkGauge input2WatermarkGauge) throws IOException {

	final InputGate inputGate = InputGateUtil.createInputGate(inputGates1, inputGates2);

	this.barrierHandler = InputProcessorUtil.createCheckpointBarrierHandler(
		checkpointedTask, checkpointMode, ioManager, inputGate, taskManagerConfig);

	this.lock = checkNotNull(lock);

	StreamElementSerializer<IN1> ser1 = new StreamElementSerializer<>(inputSerializer1);
	this.deserializationDelegate1 = new NonReusingDeserializationDelegate<>(ser1);

	StreamElementSerializer<IN2> ser2 = new StreamElementSerializer<>(inputSerializer2);
	this.deserializationDelegate2 = new NonReusingDeserializationDelegate<>(ser2);

	// Initialize one deserializer per input channel
	this.recordDeserializers = new SpillingAdaptiveSpanningRecordDeserializer[inputGate.getNumberOfInputChannels()];

	for (int i = 0; i < recordDeserializers.length; i++) {
		recordDeserializers[i] = new SpillingAdaptiveSpanningRecordDeserializer<>(
			ioManager.getSpillingDirectoriesPaths());
	}

	// determine which unioned channels belong to input 1 and which belong to input 2
	int numInputChannels1 = 0;
	for (InputGate gate: inputGates1) {
		numInputChannels1 += gate.getNumberOfInputChannels();
	}

	this.numInputChannels1 = numInputChannels1;
	this.numInputChannels2 = inputGate.getNumberOfInputChannels() - numInputChannels1;

	this.firstStatus = StreamStatus.ACTIVE;
	this.secondStatus = StreamStatus.ACTIVE;

	this.streamStatusMaintainer = checkNotNull(streamStatusMaintainer);
	this.streamOperator = checkNotNull(streamOperator);

	this.statusWatermarkValve1 = new StatusWatermarkValve(numInputChannels1, new ForwardingValveOutputHandler1(streamOperator, lock));
	this.statusWatermarkValve2 = new StatusWatermarkValve(numInputChannels2, new ForwardingValveOutputHandler2(streamOperator, lock));

	this.input1WatermarkGauge = input1WatermarkGauge;
	this.input2WatermarkGauge = input2WatermarkGauge;
	metrics.gauge("checkpointAlignmentTime", barrierHandler::getAlignmentDurationNanos);
}
 

@SuppressWarnings("unchecked")
public StreamOneInputProcessor(
		InputGate[] inputGates,
		TypeSerializer<IN> inputSerializer,
		StreamTask<?, ?> checkpointedTask,
		CheckpointingMode checkpointMode,
		Object lock,
		IOManager ioManager,
		Configuration taskManagerConfig,
		StreamStatusMaintainer streamStatusMaintainer,
		OneInputStreamOperator<IN, ?> streamOperator,
		TaskIOMetricGroup metrics,
		WatermarkGauge watermarkGauge,
		String taskName,
		OperatorChain<?, ?> operatorChain) throws IOException {

	InputGate inputGate = InputGateUtil.createInputGate(inputGates);

	CheckpointedInputGate barrierHandler = InputProcessorUtil.createCheckpointedInputGate(
		checkpointedTask,
		checkpointMode,
		ioManager,
		inputGate,
		taskManagerConfig,
		taskName);
	this.input = new StreamTaskNetworkInput(barrierHandler, inputSerializer, ioManager, 0);

	this.lock = checkNotNull(lock);

	this.streamStatusMaintainer = checkNotNull(streamStatusMaintainer);
	this.streamOperator = checkNotNull(streamOperator);

	this.statusWatermarkValve = new StatusWatermarkValve(
		inputGate.getNumberOfInputChannels(),
		new ForwardingValveOutputHandler(streamOperator, lock));

	this.watermarkGauge = watermarkGauge;
	metrics.gauge("checkpointAlignmentTime", barrierHandler::getAlignmentDurationNanos);

	this.operatorChain = checkNotNull(operatorChain);
}
 

public StreamTwoInputSelectableProcessor(
	Collection<InputGate> inputGates1,
	Collection<InputGate> inputGates2,
	TypeSerializer<IN1> inputSerializer1,
	TypeSerializer<IN2> inputSerializer2,
	StreamTask<?, ?> streamTask,
	CheckpointingMode checkpointingMode,
	Object lock,
	IOManager ioManager,
	Configuration taskManagerConfig,
	StreamStatusMaintainer streamStatusMaintainer,
	TwoInputStreamOperator<IN1, IN2, ?> streamOperator,
	WatermarkGauge input1WatermarkGauge,
	WatermarkGauge input2WatermarkGauge,
	String taskName,
	OperatorChain<?, ?> operatorChain) throws IOException {

	checkState(streamOperator instanceof InputSelectable);

	this.streamOperator = checkNotNull(streamOperator);
	this.inputSelector = (InputSelectable) streamOperator;

	this.lock = checkNotNull(lock);

	InputGate unionedInputGate1 = InputGateUtil.createInputGate(inputGates1.toArray(new InputGate[0]));
	InputGate unionedInputGate2 = InputGateUtil.createInputGate(inputGates2.toArray(new InputGate[0]));

	// create a Input instance for each input
	CheckpointedInputGate[] checkpointedInputGates = InputProcessorUtil.createCheckpointedInputGatePair(
		streamTask,
		checkpointingMode,
		ioManager,
		unionedInputGate1,
		unionedInputGate2,
		taskManagerConfig,
		taskName);
	checkState(checkpointedInputGates.length == 2);
	this.input1 = new StreamTaskNetworkInput(checkpointedInputGates[0], inputSerializer1, ioManager, 0);
	this.input2 = new StreamTaskNetworkInput(checkpointedInputGates[1], inputSerializer2, ioManager, 1);

	this.statusWatermarkValve1 = new StatusWatermarkValve(
		unionedInputGate1.getNumberOfInputChannels(),
		new ForwardingValveOutputHandler(streamOperator, lock, streamStatusMaintainer, input1WatermarkGauge, 0));
	this.statusWatermarkValve2 = new StatusWatermarkValve(
		unionedInputGate2.getNumberOfInputChannels(),
		new ForwardingValveOutputHandler(streamOperator, lock, streamStatusMaintainer, input2WatermarkGauge, 1));

	this.operatorChain = checkNotNull(operatorChain);

	this.firstStatus = StreamStatus.ACTIVE;
	this.secondStatus = StreamStatus.ACTIVE;

	this.availableInputsMask = (int) new InputSelection.Builder().select(1).select(2).build().getInputMask();

	this.lastReadInputIndex = 1; // always try to read from the first input

	this.isPrepared = false;
}
 

@SuppressWarnings("unchecked")
public StreamTwoInputProcessor(
		Collection<InputGate> inputGates1,
		Collection<InputGate> inputGates2,
		TypeSerializer<IN1> inputSerializer1,
		TypeSerializer<IN2> inputSerializer2,
		TwoInputStreamTask<IN1, IN2, ?> checkpointedTask,
		CheckpointingMode checkpointMode,
		Object lock,
		IOManager ioManager,
		Configuration taskManagerConfig,
		StreamStatusMaintainer streamStatusMaintainer,
		TwoInputStreamOperator<IN1, IN2, ?> streamOperator,
		TaskIOMetricGroup metrics,
		WatermarkGauge input1WatermarkGauge,
		WatermarkGauge input2WatermarkGauge,
		String taskName,
		OperatorChain<?, ?> operatorChain) throws IOException {

	final InputGate inputGate = InputGateUtil.createInputGate(inputGates1, inputGates2);

	this.barrierHandler = InputProcessorUtil.createCheckpointedInputGate(
		checkpointedTask,
		checkpointMode,
		ioManager,
		inputGate,
		taskManagerConfig,
		taskName);

	this.lock = checkNotNull(lock);

	StreamElementSerializer<IN1> ser1 = new StreamElementSerializer<>(inputSerializer1);
	this.deserializationDelegate1 = new NonReusingDeserializationDelegate<>(ser1);

	StreamElementSerializer<IN2> ser2 = new StreamElementSerializer<>(inputSerializer2);
	this.deserializationDelegate2 = new NonReusingDeserializationDelegate<>(ser2);

	// Initialize one deserializer per input channel
	this.recordDeserializers = new SpillingAdaptiveSpanningRecordDeserializer[inputGate.getNumberOfInputChannels()];

	for (int i = 0; i < recordDeserializers.length; i++) {
		recordDeserializers[i] = new SpillingAdaptiveSpanningRecordDeserializer<>(
			ioManager.getSpillingDirectoriesPaths());
	}

	// determine which unioned channels belong to input 1 and which belong to input 2
	int numInputChannels1 = 0;
	for (InputGate gate: inputGates1) {
		numInputChannels1 += gate.getNumberOfInputChannels();
	}

	this.numInputChannels1 = numInputChannels1;
	this.numInputChannels2 = inputGate.getNumberOfInputChannels() - numInputChannels1;

	this.firstStatus = StreamStatus.ACTIVE;
	this.secondStatus = StreamStatus.ACTIVE;

	this.streamStatusMaintainer = checkNotNull(streamStatusMaintainer);
	this.streamOperator = checkNotNull(streamOperator);

	this.statusWatermarkValve1 = new StatusWatermarkValve(numInputChannels1, new ForwardingValveOutputHandler1(streamOperator, lock));
	this.statusWatermarkValve2 = new StatusWatermarkValve(numInputChannels2, new ForwardingValveOutputHandler2(streamOperator, lock));

	this.input1WatermarkGauge = input1WatermarkGauge;
	this.input2WatermarkGauge = input2WatermarkGauge;
	metrics.gauge("checkpointAlignmentTime", barrierHandler::getAlignmentDurationNanos);

	this.operatorChain = checkNotNull(operatorChain);

	this.finishedChannels1 = new BitSet();
	this.finishedChannels2 = new BitSet();
}
 

public StreamTwoInputProcessor(
		CheckpointedInputGate[] checkpointedInputGates,
		TypeSerializer<IN1> inputSerializer1,
		TypeSerializer<IN2> inputSerializer2,
		IOManager ioManager,
		StreamStatusMaintainer streamStatusMaintainer,
		TwoInputStreamOperator<IN1, IN2, ?> streamOperator,
		TwoInputSelectionHandler inputSelectionHandler,
		WatermarkGauge input1WatermarkGauge,
		WatermarkGauge input2WatermarkGauge,
		OperatorChain<?, ?> operatorChain,
		Counter numRecordsIn) {

	this.inputSelectionHandler = checkNotNull(inputSelectionHandler);

	this.output1 = new StreamTaskNetworkOutput<>(
		streamOperator,
		record -> processRecord1(record, streamOperator, numRecordsIn),
		streamStatusMaintainer,
		input1WatermarkGauge,
		0);
	this.output2 = new StreamTaskNetworkOutput<>(
		streamOperator,
		record -> processRecord2(record, streamOperator, numRecordsIn),
		streamStatusMaintainer,
		input2WatermarkGauge,
		1);

	this.input1 = new StreamTaskNetworkInput<>(
		checkpointedInputGates[0],
		inputSerializer1,
		ioManager,
		new StatusWatermarkValve(checkpointedInputGates[0].getNumberOfInputChannels(), output1),
		0);
	this.input2 = new StreamTaskNetworkInput<>(
		checkpointedInputGates[1],
		inputSerializer2,
		ioManager,
		new StatusWatermarkValve(checkpointedInputGates[1].getNumberOfInputChannels(), output2),
		1);

	this.operatorChain = checkNotNull(operatorChain);
}