org.apache.flink.streaming.runtime.streamstatus.StreamStatus Java Examples

@Override
public StreamElement deserialize(DataInputView source) throws IOException {
	int tag = source.readByte();
	if (tag == TAG_REC_WITH_TIMESTAMP) {
		long timestamp = source.readLong();
		return new StreamRecord<T>(typeSerializer.deserialize(source), timestamp);
	}
	else if (tag == TAG_REC_WITHOUT_TIMESTAMP) {
		return new StreamRecord<T>(typeSerializer.deserialize(source));
	}
	else if (tag == TAG_WATERMARK) {
		return new Watermark(source.readLong());
	}
	else if (tag == TAG_STREAM_STATUS) {
		return new StreamStatus(source.readInt());
	}
	else if (tag == TAG_LATENCY_MARKER) {
		return new LatencyMarker(source.readLong(), new OperatorID(source.readLong(), source.readLong()), source.readInt());
	}
	else {
		throw new IOException("Corrupt stream, found tag: " + tag);
	}
}
 

@Override
public void onProcessingTime(long timestamp) throws Exception {
	advanceWatermark();

	if (idleTimeout > 0) {
		final long currentTime = getProcessingTimeService().getCurrentProcessingTime();
		if (currentTime - lastRecordTime > idleTimeout) {
			// mark the channel as idle to ignore watermarks from this channel
			streamStatusMaintainer.toggleStreamStatus(StreamStatus.IDLE);
		}
	}

	// register next timer
	long now = getProcessingTimeService().getCurrentProcessingTime();
	getProcessingTimeService().registerTimer(now + watermarkInterval, this);
}
 

@Override
public void processElement(StreamRecord<RowData> element) throws Exception {
	if (idleTimeout > 0) {
		// mark the channel active
		streamStatusMaintainer.toggleStreamStatus(StreamStatus.ACTIVE);
		lastRecordTime = getProcessingTimeService().getCurrentProcessingTime();
	}
	RowData row = element.getValue();
	if (row.isNullAt(rowtimeFieldIndex)) {
		throw new RuntimeException("RowTime field should not be null," +
				" please convert it to a non-null long value.");
	}
	Long watermark = watermarkGenerator.currentWatermark(row);
	if (watermark != null) {
		currentWatermark = Math.max(currentWatermark, watermark);
	}
	// forward element
	output.collect(element);

	// eagerly emit watermark to avoid period timer not called (this often happens when cpu load is high)
	// current_wm - last_wm > interval
	if (currentWatermark - lastWatermark > watermarkInterval) {
		advanceWatermark();
	}
}
 

@Override
public void processElement(StreamRecord<BaseRow> element) throws Exception {
	if (idleTimeout > 0) {
		// mark the channel active
		streamStatusMaintainer.toggleStreamStatus(StreamStatus.ACTIVE);
		lastRecordTime = getProcessingTimeService().getCurrentProcessingTime();
	}
	BaseRow row = element.getValue();
	if (row.isNullAt(rowtimeFieldIndex)) {
		throw new RuntimeException("RowTime field should not be null," +
			" please convert it to a non-null long value.");
	}
	long wm = row.getLong(rowtimeFieldIndex) - watermarkDelay;
	currentWatermark = Math.max(currentWatermark, wm);
	// forward element
	output.collect(element);

	if (currentWatermark >= expectedWatermark) {
		output.emitWatermark(new Watermark(currentWatermark));
		long start = getMiniBatchStart(currentWatermark, tzOffset, watermarkInterval);
		long end = start + watermarkInterval - 1;
		expectedWatermark = end > currentWatermark ? end : end + watermarkInterval;
	}
}
 

@Override
public void onProcessingTime(long timestamp) throws Exception {
	advanceWatermark();

	if (idleTimeout > 0) {
		final long currentTime = getProcessingTimeService().getCurrentProcessingTime();
		if (currentTime - lastRecordTime > idleTimeout) {
			// mark the channel as idle to ignore watermarks from this channel
			streamStatusMaintainer.toggleStreamStatus(StreamStatus.IDLE);
		}
	}

	// register next timer
	long now = getProcessingTimeService().getCurrentProcessingTime();
	getProcessingTimeService().registerTimer(now + watermarkInterval, this);
}
 

@Override
public void processElement(StreamRecord<BaseRow> element) throws Exception {
	if (idleTimeout > 0) {
		// mark the channel active
		streamStatusMaintainer.toggleStreamStatus(StreamStatus.ACTIVE);
		lastRecordTime = getProcessingTimeService().getCurrentProcessingTime();
	}
	BaseRow row = element.getValue();
	if (row.isNullAt(rowtimeFieldIndex)) {
		throw new RuntimeException("RowTime field should not be null," +
				" please convert it to a non-null long value.");
	}
	long ts = row.getLong(rowtimeFieldIndex);
	currentMaxTimestamp = Math.max(currentMaxTimestamp, ts);
	// forward element
	output.collect(element);

	// eagerly emit watermark to avoid period timer not called
	// current_ts - last_ts > interval
	if (currentMaxTimestamp - (currentWatermark + watermarkDelay) > watermarkInterval) {
		advanceWatermark();
	}
}
 

@Override
public void handleStreamStatus(StreamStatus streamStatus) {
	try {
		synchronized (lock) {
			secondStatus = streamStatus;

			// check if we need to toggle the task's stream status
			if (!streamStatus.equals(streamStatusMaintainer.getStreamStatus())) {
				if (streamStatus.isActive()) {
					// we're no longer idle if at least one input has become active
					streamStatusMaintainer.toggleStreamStatus(StreamStatus.ACTIVE);
				} else if (firstStatus.isIdle()) {
					// we're idle once both inputs are idle
					streamStatusMaintainer.toggleStreamStatus(StreamStatus.IDLE);
				}
			}
		}
	} catch (Exception e) {
		throw new RuntimeException("Exception occurred while processing valve output stream status: ", e);
	}
}
 

@Override
public void emitWatermark(Watermark mark) {
	if (allowWatermark(mark)) {
		synchronized (checkpointLock) {
			streamStatusMaintainer.toggleStreamStatus(StreamStatus.ACTIVE);

			if (nextCheck != null) {
				this.failOnNextCheck = false;
			} else {
				scheduleNextIdleDetectionTask();
			}

			processAndEmitWatermark(mark);
		}
	}
}
 

@Override
public void emitWatermark(Watermark mark) {
	if (allowWatermark(mark)) {
		synchronized (checkpointLock) {
			streamStatusMaintainer.toggleStreamStatus(StreamStatus.ACTIVE);

			if (nextCheck != null) {
				this.failOnNextCheck = false;
			} else {
				scheduleNextIdleDetectionTask();
			}

			processAndEmitWatermark(mark);
		}
	}
}
 

@Override
public void handleStreamStatus(StreamStatus streamStatus) {
	try {
		synchronized (lock) {
			secondStatus = streamStatus;

			// check if we need to toggle the task's stream status
			if (!streamStatus.equals(streamStatusMaintainer.getStreamStatus())) {
				if (streamStatus.isActive()) {
					// we're no longer idle if at least one input has become active
					streamStatusMaintainer.toggleStreamStatus(StreamStatus.ACTIVE);
				} else if (firstStatus.isIdle()) {
					// we're idle once both inputs are idle
					streamStatusMaintainer.toggleStreamStatus(StreamStatus.IDLE);
				}
			}
		}
	} catch (Exception e) {
		throw new RuntimeException("Exception occurred while processing valve output stream status: ", e);
	}
}
 

@Override
public void handleStreamStatus(StreamStatus streamStatus) {
	try {
		synchronized (lock) {
			firstStatus = streamStatus;

			// check if we need to toggle the task's stream status
			if (!streamStatus.equals(streamStatusMaintainer.getStreamStatus())) {
				if (streamStatus.isActive()) {
					// we're no longer idle if at least one input has become active
					streamStatusMaintainer.toggleStreamStatus(StreamStatus.ACTIVE);
				} else if (secondStatus.isIdle()) {
					// we're idle once both inputs are idle
					streamStatusMaintainer.toggleStreamStatus(StreamStatus.IDLE);
				}
			}
		}
	} catch (Exception e) {
		throw new RuntimeException("Exception occurred while processing valve output stream status: ", e);
	}
}
 

@Override
public void emitWatermark(Watermark watermark) {
	final long ts = watermark.getTimestamp();

	if (ts <= currentWatermark) {
		return;
	}

	currentWatermark = ts;

	if (idle) {
		idle = false;
		statusMaintainer.toggleStreamStatus(StreamStatus.ACTIVE);
	}

	output.emitWatermark(new org.apache.flink.streaming.api.watermark.Watermark(ts));
}
 

@Override
public void handleStreamStatus(StreamStatus streamStatus) {
	try {
		synchronized (lock) {
			firstStatus = streamStatus;

			// check if we need to toggle the task's stream status
			if (!streamStatus.equals(streamStatusMaintainer.getStreamStatus())) {
				if (streamStatus.isActive()) {
					// we're no longer idle if at least one input has become active
					streamStatusMaintainer.toggleStreamStatus(StreamStatus.ACTIVE);
				} else if (secondStatus.isIdle()) {
					// we're idle once both inputs are idle
					streamStatusMaintainer.toggleStreamStatus(StreamStatus.IDLE);
				}
			}
		}
	} catch (Exception e) {
		throw new RuntimeException("Exception occurred while processing valve output stream status: ", e);
	}
}
 

@Override
public StreamElement deserialize(DataInputView source) throws IOException {
	int tag = source.readByte();
	if (tag == TAG_REC_WITH_TIMESTAMP) {
		long timestamp = source.readLong();
		return new StreamRecord<T>(typeSerializer.deserialize(source), timestamp);
	}
	else if (tag == TAG_REC_WITHOUT_TIMESTAMP) {
		return new StreamRecord<T>(typeSerializer.deserialize(source));
	}
	else if (tag == TAG_WATERMARK) {
		return new Watermark(source.readLong());
	}
	else if (tag == TAG_STREAM_STATUS) {
		return new StreamStatus(source.readInt());
	}
	else if (tag == TAG_LATENCY_MARKER) {
		return new LatencyMarker(source.readLong(), new OperatorID(source.readLong(), source.readLong()), source.readInt());
	}
	else {
		throw new IOException("Corrupt stream, found tag: " + tag);
	}
}
 

@Override
public void emitStreamStatus(StreamStatus streamStatus) {
	final StreamStatus anotherStreamStatus;
	if (inputIndex == 0) {
		firstStatus = streamStatus;
		anotherStreamStatus = secondStatus;
	} else {
		secondStatus = streamStatus;
		anotherStreamStatus = firstStatus;
	}

	// check if we need to toggle the task's stream status
	if (!streamStatus.equals(streamStatusMaintainer.getStreamStatus())) {
		if (streamStatus.isActive()) {
			// we're no longer idle if at least one input has become active
			streamStatusMaintainer.toggleStreamStatus(StreamStatus.ACTIVE);
		} else if (anotherStreamStatus.isIdle()) {
			// we're idle once both inputs are idle
			streamStatusMaintainer.toggleStreamStatus(StreamStatus.IDLE);
		}
	}
}
 

@Override
public StreamElement deserialize(DataInputView source) throws IOException {
	int tag = source.readByte();
	if (tag == TAG_REC_WITH_TIMESTAMP) {
		long timestamp = source.readLong();
		return new StreamRecord<T>(typeSerializer.deserialize(source), timestamp);
	}
	else if (tag == TAG_REC_WITHOUT_TIMESTAMP) {
		return new StreamRecord<T>(typeSerializer.deserialize(source));
	}
	else if (tag == TAG_WATERMARK) {
		return new Watermark(source.readLong());
	}
	else if (tag == TAG_STREAM_STATUS) {
		return new StreamStatus(source.readInt());
	}
	else if (tag == TAG_LATENCY_MARKER) {
		return new LatencyMarker(source.readLong(), new OperatorID(source.readLong(), source.readLong()), source.readInt());
	}
	else {
		throw new IOException("Corrupt stream, found tag: " + tag);
	}
}
 

@Override
public void emitStreamStatus(StreamStatus streamStatus) {
	final StreamStatus anotherStreamStatus;

	streamStatuses[inputIndex] = streamStatus;

	// check if we need to toggle the task's stream status
	if (!streamStatus.equals(streamStatusMaintainer.getStreamStatus())) {
		if (streamStatus.isActive()) {
			// we're no longer idle if at least one input has become active
			streamStatusMaintainer.toggleStreamStatus(StreamStatus.ACTIVE);
		} else if (allStreamStatusesAreIdle()) {
			streamStatusMaintainer.toggleStreamStatus(StreamStatus.IDLE);
		}
	}
}
 

@Override
public void emitWatermark(Watermark mark) {
	if (allowWatermark(mark)) {
		synchronized (checkpointLock) {
			streamStatusMaintainer.toggleStreamStatus(StreamStatus.ACTIVE);

			if (nextCheck != null) {
				this.failOnNextCheck = false;
			} else {
				scheduleNextIdleDetectionTask();
			}

			processAndEmitWatermark(mark);
		}
	}
}
 

@Test
public void testWatermarkAssignerWithIdleSource() throws Exception {
	// with timeout 1000 ms
	OneInputStreamOperatorTestHarness<RowData, RowData> testHarness = createTestHarness(0, WATERMARK_GENERATOR, 1000);
	testHarness.getExecutionConfig().setAutoWatermarkInterval(50);
	testHarness.open();

	testHarness.processElement(new StreamRecord<>(GenericRowData.of(1L)));
	testHarness.processElement(new StreamRecord<>(GenericRowData.of(2L)));
	testHarness.processWatermark(new Watermark(2)); // this watermark should be ignored
	testHarness.processElement(new StreamRecord<>(GenericRowData.of(3L)));
	testHarness.processElement(new StreamRecord<>(GenericRowData.of(4L)));

	// trigger watermark emit
	testHarness.setProcessingTime(51);
	ConcurrentLinkedQueue<Object> output = testHarness.getOutput();
	List<Watermark> watermarks = extractWatermarks(output);
	assertEquals(1, watermarks.size());
	assertEquals(new Watermark(3), watermarks.get(0));
	assertEquals(StreamStatus.ACTIVE, testHarness.getStreamStatus());
	output.clear();

	testHarness.setProcessingTime(1001);
	assertEquals(StreamStatus.IDLE, testHarness.getStreamStatus());

	testHarness.processElement(new StreamRecord<>(GenericRowData.of(4L)));
	testHarness.processElement(new StreamRecord<>(GenericRowData.of(5L)));
	testHarness.processElement(new StreamRecord<>(GenericRowData.of(6L)));
	testHarness.processElement(new StreamRecord<>(GenericRowData.of(7L)));
	testHarness.processElement(new StreamRecord<>(GenericRowData.of(8L)));

	assertEquals(StreamStatus.ACTIVE, testHarness.getStreamStatus());
	testHarness.setProcessingTime(1060);
	output = testHarness.getOutput();
	watermarks = extractWatermarks(output);
	assertEquals(1, watermarks.size());
	assertEquals(new Watermark(7), watermarks.get(0));
}
 

@Override
public void collectWithTimestamp(T element, long timestamp) {
	synchronized (checkpointLock) {
		streamStatusMaintainer.toggleStreamStatus(StreamStatus.ACTIVE);

		if (nextCheck != null) {
			this.failOnNextCheck = false;
		} else {
			scheduleNextIdleDetectionTask();
		}

		processAndCollectWithTimestamp(element, timestamp);
	}
}
 

public StreamMultipleInputProcessor(
		CheckpointedInputGate[] checkpointedInputGates,
		TypeSerializer<?>[] inputSerializers,
		IOManager ioManager,
		StreamStatusMaintainer streamStatusMaintainer,
		MultipleInputStreamOperator<?> streamOperator,
		MultipleInputSelectionHandler inputSelectionHandler,
		WatermarkGauge[] inputWatermarkGauges,
		OperatorChain<?, ?> operatorChain,
		Counter numRecordsIn) {

	this.inputSelectionHandler = checkNotNull(inputSelectionHandler);

	List<Input> inputs = streamOperator.getInputs();
	int inputsCount = inputs.size();

	this.inputProcessors = new InputProcessor[inputsCount];
	this.streamStatuses = new StreamStatus[inputsCount];
	this.numRecordsIn = numRecordsIn;

	for (int i = 0; i < inputsCount; i++) {
		streamStatuses[i] = StreamStatus.ACTIVE;
		StreamTaskNetworkOutput dataOutput = new StreamTaskNetworkOutput<>(
			inputs.get(i),
			streamStatusMaintainer,
			inputWatermarkGauges[i],
			i);

		inputProcessors[i] = new InputProcessor(
			dataOutput,
			new StreamTaskNetworkInput<>(
				checkpointedInputGates[i],
				inputSerializers[i],
				ioManager,
				new StatusWatermarkValve(checkpointedInputGates[i].getNumberOfInputChannels(), dataOutput),
				i));
	}

	this.operatorChain = checkNotNull(operatorChain);
}
 

private boolean allStreamStatusesAreIdle() {
	for (StreamStatus streamStatus : streamStatuses) {
		if (streamStatus.isActive()) {
			return false;
		}
	}
	return true;
}
 

@Test
public void becomingActiveEmitsStatus() {
	final CollectingDataOutput<Object> testingOutput = new CollectingDataOutput<>();
	final WatermarkToDataOutput wmOutput = new WatermarkToDataOutput(testingOutput);

	wmOutput.markIdle();
	wmOutput.emitWatermark(new org.apache.flink.api.common.eventtime.Watermark(100L));

	assertThat(testingOutput.events, contains(
		StreamStatus.IDLE,
		StreamStatus.ACTIVE,
		new Watermark(100L)
	));
}
 

@SuppressWarnings("unchecked")
private static <T> MockStreamTask setupSourceOperator(
		StreamSource<T, ?> operator,
		TimeCharacteristic timeChar,
		long watermarkInterval,
		final TimerService timeProvider) throws Exception {

	ExecutionConfig executionConfig = new ExecutionConfig();
	executionConfig.setAutoWatermarkInterval(watermarkInterval);

	StreamConfig cfg = new StreamConfig(new Configuration());
	cfg.setStateBackend(new MemoryStateBackend());

	cfg.setTimeCharacteristic(timeChar);
	cfg.setOperatorID(new OperatorID());

	Environment env = new DummyEnvironment("MockTwoInputTask", 1, 0);

	StreamStatusMaintainer streamStatusMaintainer = mock(StreamStatusMaintainer.class);
	when(streamStatusMaintainer.getStreamStatus()).thenReturn(StreamStatus.ACTIVE);

	MockStreamTask mockTask = new MockStreamTaskBuilder(env)
		.setConfig(cfg)
		.setExecutionConfig(executionConfig)
		.setStreamStatusMaintainer(streamStatusMaintainer)
		.setTimerService(timeProvider)
		.build();

	operator.setup(mockTask, cfg, (Output<StreamRecord<T>>) mock(Output.class));
	return mockTask;
}
 

public void emitStreamStatus(StreamStatus streamStatus) {
	serializationDelegate.setInstance(streamStatus);

	try {
		recordWriter.broadcastEmit(serializationDelegate);
	}
	catch (Exception e) {
		throw new RuntimeException(e.getMessage(), e);
	}
}
 

@Override
public void collectWithTimestamp(T element, long timestamp) {
	synchronized (checkpointLock) {
		streamStatusMaintainer.toggleStreamStatus(StreamStatus.ACTIVE);

		if (nextCheck != null) {
			this.failOnNextCheck = false;
		} else {
			scheduleNextIdleDetectionTask();
		}

		processAndCollectWithTimestamp(element, timestamp);
	}
}
 

@SuppressWarnings("unchecked")
@Override
public void handleStreamStatus(StreamStatus streamStatus) {
	try {
		synchronized (lock) {
			streamStatusMaintainer.toggleStreamStatus(streamStatus);
		}
	} catch (Exception e) {
		throw new RuntimeException("Exception occurred while processing valve output stream status: ", e);
	}
}
 

@SuppressWarnings("unchecked")
private static <T> void setupSourceOperator(StreamSource<T, ?> operator,
											TimeCharacteristic timeChar,
											long watermarkInterval,
											final ProcessingTimeService timeProvider) throws Exception {

	ExecutionConfig executionConfig = new ExecutionConfig();
	executionConfig.setAutoWatermarkInterval(watermarkInterval);

	StreamConfig cfg = new StreamConfig(new Configuration());
	cfg.setStateBackend(new MemoryStateBackend());

	cfg.setTimeCharacteristic(timeChar);
	cfg.setOperatorID(new OperatorID());

	Environment env = new DummyEnvironment("MockTwoInputTask", 1, 0);

	StreamStatusMaintainer streamStatusMaintainer = mock(StreamStatusMaintainer.class);
	when(streamStatusMaintainer.getStreamStatus()).thenReturn(StreamStatus.ACTIVE);

	MockStreamTask mockTask = new MockStreamTaskBuilder(env)
		.setConfig(cfg)
		.setExecutionConfig(executionConfig)
		.setStreamStatusMaintainer(streamStatusMaintainer)
		.setProcessingTimeService(timeProvider)
		.build();

	operator.setup(mockTask, cfg, (Output<StreamRecord<T>>) mock(Output.class));
}
 

public void emitStreamStatus(StreamStatus streamStatus) {
	serializationDelegate.setInstance(streamStatus);

	try {
		recordWriter.broadcastEmit(serializationDelegate);
	}
	catch (Exception e) {
		throw new RuntimeException(e.getMessage(), e);
	}
}
 

@Override
public void handleStreamStatus(StreamStatus streamStatus) {
	try {
		synchronized (lock) {
			final StreamStatus anotherStreamStatus;
			if (inputIndex == 0) {
				firstStatus = streamStatus;
				anotherStreamStatus = secondStatus;
			} else {
				secondStatus = streamStatus;
				anotherStreamStatus = firstStatus;
			}

			// check if we need to toggle the task's stream status
			if (!streamStatus.equals(streamStatusMaintainer.getStreamStatus())) {
				if (streamStatus.isActive()) {
					// we're no longer idle if at least one input has become active
					streamStatusMaintainer.toggleStreamStatus(StreamStatus.ACTIVE);
				} else if (anotherStreamStatus.isIdle()) {
					// we're idle once both inputs are idle
					streamStatusMaintainer.toggleStreamStatus(StreamStatus.IDLE);
				}
			}
		}
	} catch (Exception e) {
		throw new RuntimeException("Exception occurred while processing valve output stream status of input"
			+ (inputIndex + 1) + ": ", e);
	}
}