org.apache.flink.streaming.runtime.streamstatus.StreamStatusMaintainer Java Examples

private AutomaticWatermarkContext(
		final Output<StreamRecord<T>> output,
		final long watermarkInterval,
		final ProcessingTimeService timeService,
		final Object checkpointLock,
		final StreamStatusMaintainer streamStatusMaintainer,
		final long idleTimeout) {

	super(timeService, checkpointLock, streamStatusMaintainer, idleTimeout);

	this.output = Preconditions.checkNotNull(output, "The output cannot be null.");

	Preconditions.checkArgument(watermarkInterval >= 1L, "The watermark interval cannot be smaller than 1 ms.");
	this.watermarkInterval = watermarkInterval;

	this.reuse = new StreamRecord<>(null);

	this.lastRecordTime = Long.MIN_VALUE;

	long now = this.timeService.getCurrentProcessingTime();
	this.nextWatermarkTimer = this.timeService.registerTimer(now + watermarkInterval,
		new WatermarkEmittingTask(this.timeService, checkpointLock, output));
}
 

/**
 * Create a watermark context.
 *
 * @param timeService the time service to schedule idleness detection tasks
 * @param checkpointLock the checkpoint lock
 * @param streamStatusMaintainer the stream status maintainer to toggle and retrieve current status
 * @param idleTimeout (-1 if idleness checking is disabled)
 */
public WatermarkContext(
		final ProcessingTimeService timeService,
		final Object checkpointLock,
		final StreamStatusMaintainer streamStatusMaintainer,
		final long idleTimeout) {

	this.timeService = Preconditions.checkNotNull(timeService, "Time Service cannot be null.");
	this.checkpointLock = Preconditions.checkNotNull(checkpointLock, "Checkpoint Lock cannot be null.");
	this.streamStatusMaintainer = Preconditions.checkNotNull(streamStatusMaintainer, "Stream Status Maintainer cannot be null.");

	if (idleTimeout != -1) {
		Preconditions.checkArgument(idleTimeout >= 1, "The idle timeout cannot be smaller than 1 ms.");
	}
	this.idleTimeout = idleTimeout;

	scheduleNextIdleDetectionTask();
}
 

public MockStreamTask(
	Environment environment,
	Object checkpointLock,
	StreamConfig config,
	ExecutionConfig executionConfig,
	StreamTaskStateInitializer streamTaskStateInitializer,
	StreamStatusMaintainer streamStatusMaintainer,
	CheckpointStorageWorkerView checkpointStorage,
	TimerService timerService,
	BiConsumer<String, Throwable> handleAsyncException,
	TaskMailbox taskMailbox,
	StreamTaskActionExecutor.SynchronizedStreamTaskActionExecutor taskActionExecutor,
	StreamInputProcessor inputProcessor) throws Exception {

	super(environment, timerService, FatalExitExceptionHandler.INSTANCE, taskActionExecutor, taskMailbox);
	this.checkpointLock = checkpointLock;
	this.config = config;
	this.executionConfig = executionConfig;
	this.streamTaskStateInitializer = streamTaskStateInitializer;
	this.streamStatusMaintainer = streamStatusMaintainer;
	this.checkpointStorage = checkpointStorage;
	this.processingTimeService = timerService;
	this.handleAsyncException = handleAsyncException;
	this.inputProcessor = inputProcessor;
}
 

private AutomaticWatermarkContext(
		final Output<StreamRecord<T>> output,
		final long watermarkInterval,
		final ProcessingTimeService timeService,
		final Object checkpointLock,
		final StreamStatusMaintainer streamStatusMaintainer,
		final long idleTimeout) {

	super(timeService, checkpointLock, streamStatusMaintainer, idleTimeout);

	this.output = Preconditions.checkNotNull(output, "The output cannot be null.");

	Preconditions.checkArgument(watermarkInterval >= 1L, "The watermark interval cannot be smaller than 1 ms.");
	this.watermarkInterval = watermarkInterval;

	this.reuse = new StreamRecord<>(null);

	this.lastRecordTime = Long.MIN_VALUE;

	long now = this.timeService.getCurrentProcessingTime();
	this.nextWatermarkTimer = this.timeService.registerTimer(now + watermarkInterval,
		new WatermarkEmittingTask(this.timeService, checkpointLock, output));
}
 

private AutomaticWatermarkContext(
		final Output<StreamRecord<T>> output,
		final long watermarkInterval,
		final ProcessingTimeService timeService,
		final Object checkpointLock,
		final StreamStatusMaintainer streamStatusMaintainer,
		final long idleTimeout) {

	super(timeService, checkpointLock, streamStatusMaintainer, idleTimeout);

	this.output = Preconditions.checkNotNull(output, "The output cannot be null.");

	Preconditions.checkArgument(watermarkInterval >= 1L, "The watermark interval cannot be smaller than 1 ms.");
	this.watermarkInterval = watermarkInterval;

	this.reuse = new StreamRecord<>(null);

	this.lastRecordTime = Long.MIN_VALUE;

	long now = this.timeService.getCurrentProcessingTime();
	this.nextWatermarkTimer = this.timeService.registerTimer(now + watermarkInterval,
		new WatermarkEmittingTask(this.timeService, checkpointLock, output));
}
 

@Test
public void testNoMaxWatermarkOnImmediateCancel() throws Exception {

	final List<StreamElement> output = new ArrayList<>();

	// regular stream source operator
	final StreamSource<String, InfiniteSource<String>> operator =
			new StreamSource<>(new InfiniteSource<String>());

	setupSourceOperator(operator, TimeCharacteristic.EventTime, 0);
	operator.cancel();

	// run and exit
	operator.run(new Object(), mock(StreamStatusMaintainer.class), new CollectorOutput<String>(output));

	assertTrue(output.isEmpty());
}
 

@Test
public void testNoMaxWatermarkOnImmediateStop() throws Exception {

	final List<StreamElement> output = new ArrayList<>();

	// regular stream source operator
	final StoppableStreamSource<String, InfiniteSource<String>> operator =
			new StoppableStreamSource<>(new InfiniteSource<String>());

	setupSourceOperator(operator, TimeCharacteristic.EventTime, 0);
	operator.stop();

	// run and stop
	operator.run(new Object(), mock(StreamStatusMaintainer.class), new CollectorOutput<String>(output));

	assertTrue(output.isEmpty());
}
 

/**
 * Create a watermark context.
 *
 * @param timeService the time service to schedule idleness detection tasks
 * @param checkpointLock the checkpoint lock
 * @param streamStatusMaintainer the stream status maintainer to toggle and retrieve current status
 * @param idleTimeout (-1 if idleness checking is disabled)
 */
public WatermarkContext(
		final ProcessingTimeService timeService,
		final Object checkpointLock,
		final StreamStatusMaintainer streamStatusMaintainer,
		final long idleTimeout) {

	this.timeService = Preconditions.checkNotNull(timeService, "Time Service cannot be null.");
	this.checkpointLock = Preconditions.checkNotNull(checkpointLock, "Checkpoint Lock cannot be null.");
	this.streamStatusMaintainer = Preconditions.checkNotNull(streamStatusMaintainer, "Stream Status Maintainer cannot be null.");

	if (idleTimeout != -1) {
		Preconditions.checkArgument(idleTimeout >= 1, "The idle timeout cannot be smaller than 1 ms.");
	}
	this.idleTimeout = idleTimeout;

	scheduleNextIdleDetectionTask();
}
 

private ForwardingValveOutputHandler(
	TwoInputStreamOperator<IN1, IN2, ?> operator,
	Object lock,
	StreamStatusMaintainer streamStatusMaintainer,
	WatermarkGauge inputWatermarkGauge,
	int inputIndex) {

	this.operator = checkNotNull(operator);
	this.lock = checkNotNull(lock);

	this.streamStatusMaintainer = checkNotNull(streamStatusMaintainer);

	this.inputWatermarkGauge = inputWatermarkGauge;

	this.inputIndex = inputIndex;
}
 

@Test
public void testEmitMaxWatermarkForFiniteSource() throws Exception {

	// regular stream source operator
	StreamSource<String, FiniteSource<String>> operator =
			new StreamSource<>(new FiniteSource<String>());

	final List<StreamElement> output = new ArrayList<>();

	setupSourceOperator(operator, TimeCharacteristic.EventTime, 0);
	OperatorChain<?, ?> operatorChain = createOperatorChain(operator);
	try {
		operator.run(new Object(), mock(StreamStatusMaintainer.class), new CollectorOutput<String>(output), operatorChain);
	} finally {
		operatorChain.releaseOutputs();
	}

	assertEquals(1, output.size());
	assertEquals(Watermark.MAX_WATERMARK, output.get(0));
}
 

@Test
public void testNoMaxWatermarkOnImmediateCancel() throws Exception {

	final List<StreamElement> output = new ArrayList<>();

	// regular stream source operator
	final StreamSource<String, InfiniteSource<String>> operator =
			new StreamSource<>(new InfiniteSource<String>());

	setupSourceOperator(operator, TimeCharacteristic.EventTime, 0);
	operator.cancel();

	// run and exit
	OperatorChain<?, ?> operatorChain = createOperatorChain(operator);
	try {
		operator.run(new Object(), mock(StreamStatusMaintainer.class), new CollectorOutput<String>(output), operatorChain);
	} finally {
		operatorChain.releaseOutputs();
	}

	assertTrue(output.isEmpty());
}
 

/**
 * Create a watermark context.
 *
 * @param timeService the time service to schedule idleness detection tasks
 * @param checkpointLock the checkpoint lock
 * @param streamStatusMaintainer the stream status maintainer to toggle and retrieve current status
 * @param idleTimeout (-1 if idleness checking is disabled)
 */
public WatermarkContext(
		final ProcessingTimeService timeService,
		final Object checkpointLock,
		final StreamStatusMaintainer streamStatusMaintainer,
		final long idleTimeout) {

	this.timeService = Preconditions.checkNotNull(timeService, "Time Service cannot be null.");
	this.checkpointLock = Preconditions.checkNotNull(checkpointLock, "Checkpoint Lock cannot be null.");
	this.streamStatusMaintainer = Preconditions.checkNotNull(streamStatusMaintainer, "Stream Status Maintainer cannot be null.");

	if (idleTimeout != -1) {
		Preconditions.checkArgument(idleTimeout >= 1, "The idle timeout cannot be smaller than 1 ms.");
	}
	this.idleTimeout = idleTimeout;

	scheduleNextIdleDetectionTask();
}
 

/**
 * Depending on the {@link TimeCharacteristic}, this method will return the adequate
 * {@link org.apache.flink.streaming.api.functions.source.SourceFunction.SourceContext}. That is:
 * <ul>
 *     <li>{@link TimeCharacteristic#IngestionTime} = {@code AutomaticWatermarkContext}</li>
 *     <li>{@link TimeCharacteristic#ProcessingTime} = {@code NonTimestampContext}</li>
 *     <li>{@link TimeCharacteristic#EventTime} = {@code ManualWatermarkContext}</li>
 * </ul>
 * */
public static <OUT> SourceFunction.SourceContext<OUT> getSourceContext(
		TimeCharacteristic timeCharacteristic,
		ProcessingTimeService processingTimeService,
		Object checkpointLock,
		StreamStatusMaintainer streamStatusMaintainer,
		Output<StreamRecord<OUT>> output,
		long watermarkInterval,
		long idleTimeout) {

	final SourceFunction.SourceContext<OUT> ctx;
	switch (timeCharacteristic) {
		case EventTime:
			ctx = new ManualWatermarkContext<>(
				output,
				processingTimeService,
				checkpointLock,
				streamStatusMaintainer,
				idleTimeout);

			break;
		case IngestionTime:
			ctx = new AutomaticWatermarkContext<>(
				output,
				watermarkInterval,
				processingTimeService,
				checkpointLock,
				streamStatusMaintainer,
				idleTimeout);

			break;
		case ProcessingTime:
			ctx = new NonTimestampContext<>(checkpointLock, output);
			break;
		default:
			throw new IllegalArgumentException(String.valueOf(timeCharacteristic));
	}
	return ctx;
}
 

@Test
public void testNoMaxWatermarkOnAsyncStop() throws Exception {

	final List<StreamElement> output = new ArrayList<>();

	// regular stream source operator
	final StoppableStreamSource<String, InfiniteSource<String>> operator =
			new StoppableStreamSource<>(new InfiniteSource<String>());

	setupSourceOperator(operator, TimeCharacteristic.EventTime, 0);

	// trigger an async cancel in a bit
	new Thread("canceler") {
		@Override
		public void run() {
			try {
				Thread.sleep(200);
			} catch (InterruptedException ignored) {}
			operator.stop();
		}
	}.start();

	// run and wait to be stopped
	operator.run(new Object(), mock(StreamStatusMaintainer.class), new CollectorOutput<String>(output));

	assertTrue(output.isEmpty());
}
 

@SuppressWarnings("unchecked")
private static <T> void setupSourceOperator(StreamSource<T, ?> operator,
											TimeCharacteristic timeChar,
											long watermarkInterval,
											final ProcessingTimeService timeProvider) throws Exception {

	ExecutionConfig executionConfig = new ExecutionConfig();
	executionConfig.setAutoWatermarkInterval(watermarkInterval);

	StreamConfig cfg = new StreamConfig(new Configuration());
	cfg.setStateBackend(new MemoryStateBackend());

	cfg.setTimeCharacteristic(timeChar);
	cfg.setOperatorID(new OperatorID());

	Environment env = new DummyEnvironment("MockTwoInputTask", 1, 0);

	StreamStatusMaintainer streamStatusMaintainer = mock(StreamStatusMaintainer.class);
	when(streamStatusMaintainer.getStreamStatus()).thenReturn(StreamStatus.ACTIVE);

	MockStreamTask mockTask = new MockStreamTaskBuilder(env)
		.setConfig(cfg)
		.setExecutionConfig(executionConfig)
		.setStreamStatusMaintainer(streamStatusMaintainer)
		.setProcessingTimeService(timeProvider)
		.build();

	operator.setup(mockTask, cfg, (Output<StreamRecord<T>>) mock(Output.class));
}
 

@SuppressWarnings("unchecked")
private static <T> MockStreamTask setupSourceOperator(
		StreamSource<T, ?> operator,
		TimeCharacteristic timeChar,
		long watermarkInterval,
		final TimerService timeProvider) throws Exception {

	ExecutionConfig executionConfig = new ExecutionConfig();
	executionConfig.setAutoWatermarkInterval(watermarkInterval);

	StreamConfig cfg = new StreamConfig(new Configuration());
	cfg.setStateBackend(new MemoryStateBackend());

	cfg.setTimeCharacteristic(timeChar);
	cfg.setOperatorID(new OperatorID());

	Environment env = new DummyEnvironment("MockTwoInputTask", 1, 0);

	StreamStatusMaintainer streamStatusMaintainer = mock(StreamStatusMaintainer.class);
	when(streamStatusMaintainer.getStreamStatus()).thenReturn(StreamStatus.ACTIVE);

	MockStreamTask mockTask = new MockStreamTaskBuilder(env)
		.setConfig(cfg)
		.setExecutionConfig(executionConfig)
		.setStreamStatusMaintainer(streamStatusMaintainer)
		.setTimerService(timeProvider)
		.build();

	operator.setup(mockTask, cfg, (Output<StreamRecord<T>>) mock(Output.class));
	return mockTask;
}
 

private StreamTaskNetworkOutput(
		TwoInputStreamOperator<IN1, IN2, ?> operator,
		ThrowingConsumer<StreamRecord<T>, Exception> recordConsumer,
		StreamStatusMaintainer streamStatusMaintainer,
		WatermarkGauge inputWatermarkGauge,
		int inputIndex) {
	super(streamStatusMaintainer);

	this.operator = checkNotNull(operator);
	this.recordConsumer = checkNotNull(recordConsumer);
	this.inputWatermarkGauge = checkNotNull(inputWatermarkGauge);
	this.inputIndex = inputIndex;
}
 

/**
 * Depending on the {@link TimeCharacteristic}, this method will return the adequate
 * {@link org.apache.flink.streaming.api.functions.source.SourceFunction.SourceContext}. That is:
 * <ul>
 *     <li>{@link TimeCharacteristic#IngestionTime} = {@code AutomaticWatermarkContext}</li>
 *     <li>{@link TimeCharacteristic#ProcessingTime} = {@code NonTimestampContext}</li>
 *     <li>{@link TimeCharacteristic#EventTime} = {@code ManualWatermarkContext}</li>
 * </ul>
 * */
public static <OUT> SourceFunction.SourceContext<OUT> getSourceContext(
		TimeCharacteristic timeCharacteristic,
		ProcessingTimeService processingTimeService,
		Object checkpointLock,
		StreamStatusMaintainer streamStatusMaintainer,
		Output<StreamRecord<OUT>> output,
		long watermarkInterval,
		long idleTimeout) {

	final SourceFunction.SourceContext<OUT> ctx;
	switch (timeCharacteristic) {
		case EventTime:
			ctx = new ManualWatermarkContext<>(
				output,
				processingTimeService,
				checkpointLock,
				streamStatusMaintainer,
				idleTimeout);

			break;
		case IngestionTime:
			ctx = new AutomaticWatermarkContext<>(
				output,
				watermarkInterval,
				processingTimeService,
				checkpointLock,
				streamStatusMaintainer,
				idleTimeout);

			break;
		case ProcessingTime:
			ctx = new NonTimestampContext<>(checkpointLock, output);
			break;
		default:
			throw new IllegalArgumentException(String.valueOf(timeCharacteristic));
	}
	return ctx;
}
 

public static <OutT, SrcT extends SourceFunction<OutT>> void run(
    StreamSource<OutT, SrcT> streamSource,
    Object lockingObject,
    StreamStatusMaintainer streamStatusMaintainer,
    Output<StreamRecord<OutT>> collector)
    throws Exception {
  streamSource.run(
      lockingObject, streamStatusMaintainer, collector, createOperatorChain(streamSource));
}
 

@Override
public void run(Object lockingObject,
				StreamStatusMaintainer streamStatusMaintainer,
				Output<StreamRecord<OUT>> collector) throws Exception {
	ACTUAL_ORDER_TRACKING.add("OPERATOR::run");
	super.run(lockingObject, streamStatusMaintainer, collector);
	runStarted.trigger();
	runFinish.await();
}
 

public static <OutT, SrcT extends SourceFunction<OutT>> void run(
    StreamSource<OutT, SrcT> streamSource,
    Object lockingObject,
    StreamStatusMaintainer streamStatusMaintainer,
    Output<StreamRecord<OutT>> collector)
    throws Exception {
  streamSource.run(
      lockingObject, streamStatusMaintainer, collector, createOperatorChain(streamSource));
}
 

@Override
public void run(Object lockingObject,
				StreamStatusMaintainer streamStatusMaintainer,
				Output<StreamRecord<OUT>> collector,
				OperatorChain<?, ?> operatorChain) throws Exception {
	ACTUAL_ORDER_TRACKING.add("OPERATOR::run");
	super.run(lockingObject, streamStatusMaintainer, collector, operatorChain);
	runStarted.trigger();
	runFinish.await();
}
 

@Override
public void run(Object lockingObject,
				StreamStatusMaintainer streamStatusMaintainer,
				Output<StreamRecord<Long>> collector,
				OperatorChain<?, ?> operatorChain) throws Exception {
	while (!canceled) {
		try {
			Thread.sleep(500);
		} catch (InterruptedException ignored) {}
	}
}
 

AsyncDataOutputToOutput(
		Output<StreamRecord<T>> output,
		StreamStatusMaintainer streamStatusMaintainer) {
	super(streamStatusMaintainer);

	this.output = checkNotNull(output);
}
 

private ManualWatermarkContext(
		final Output<StreamRecord<T>> output,
		final ProcessingTimeService timeService,
		final Object checkpointLock,
		final StreamStatusMaintainer streamStatusMaintainer,
		final long idleTimeout) {

	super(timeService, checkpointLock, streamStatusMaintainer, idleTimeout);

	this.output = Preconditions.checkNotNull(output, "The output cannot be null.");
	this.reuse = new StreamRecord<>(null);
}
 

@SuppressWarnings("unchecked")
private static <T> void setupSourceOperator(StreamSource<T, ?> operator,
											TimeCharacteristic timeChar,
											long watermarkInterval,
											final ProcessingTimeService timeProvider) throws Exception {

	ExecutionConfig executionConfig = new ExecutionConfig();
	executionConfig.setAutoWatermarkInterval(watermarkInterval);

	StreamConfig cfg = new StreamConfig(new Configuration());
	cfg.setStateBackend(new MemoryStateBackend());

	cfg.setTimeCharacteristic(timeChar);
	cfg.setOperatorID(new OperatorID());

	Environment env = new DummyEnvironment("MockTwoInputTask", 1, 0);

	StreamStatusMaintainer streamStatusMaintainer = mock(StreamStatusMaintainer.class);
	when(streamStatusMaintainer.getStreamStatus()).thenReturn(StreamStatus.ACTIVE);

	MockStreamTask mockTask = new MockStreamTaskBuilder(env)
		.setConfig(cfg)
		.setExecutionConfig(executionConfig)
		.setStreamStatusMaintainer(streamStatusMaintainer)
		.setProcessingTimeService(timeProvider)
		.build();

	operator.setup(mockTask, cfg, (Output<StreamRecord<T>>) mock(Output.class));
}
 

private StreamTaskNetworkOutput(
		OneInputStreamOperator<IN, ?> operator,
		StreamStatusMaintainer streamStatusMaintainer,
		WatermarkGauge watermarkGauge,
		Counter numRecordsIn) {
	super(streamStatusMaintainer);

	this.operator = checkNotNull(operator);
	this.watermarkGauge = checkNotNull(watermarkGauge);
	this.numRecordsIn = checkNotNull(numRecordsIn);
}
 

public MockStreamTask(
	Environment environment,
	String name,
	Object checkpointLock,
	StreamConfig config,
	ExecutionConfig executionConfig,
	StreamTaskStateInitializer streamTaskStateInitializer,
	CloseableRegistry closableRegistry,
	StreamStatusMaintainer streamStatusMaintainer,
	CheckpointStorageWorkerView checkpointStorage,
	ProcessingTimeService processingTimeService,
	BiConsumer<String, Throwable> handleAsyncException,
	Map<String, Accumulator<?, ?>> accumulatorMap
) {
	super(environment);
	this.name = name;
	this.checkpointLock = checkpointLock;
	this.config = config;
	this.executionConfig = executionConfig;
	this.streamTaskStateInitializer = streamTaskStateInitializer;
	this.closableRegistry = closableRegistry;
	this.streamStatusMaintainer = streamStatusMaintainer;
	this.checkpointStorage = checkpointStorage;
	this.processingTimeService = processingTimeService;
	this.handleAsyncException = handleAsyncException;
	this.accumulatorMap = accumulatorMap;
}
 

public static <OutT, SrcT extends SourceFunction<OutT>> void run(
    StreamSource<OutT, SrcT> streamSource,
    Object lockingObject,
    StreamStatusMaintainer streamStatusMaintainer,
    Output<StreamRecord<OutT>> collector)
    throws Exception {
  streamSource.run(lockingObject, streamStatusMaintainer, collector);
}
 

private ManualWatermarkContext(
		final Output<StreamRecord<T>> output,
		final ProcessingTimeService timeService,
		final Object checkpointLock,
		final StreamStatusMaintainer streamStatusMaintainer,
		final long idleTimeout) {

	super(timeService, checkpointLock, streamStatusMaintainer, idleTimeout);

	this.output = Preconditions.checkNotNull(output, "The output cannot be null.");
	this.reuse = new StreamRecord<>(null);
}