org.apache.flink.streaming.runtime.tasks.ProcessingTimeCallback Java Examples

public LatencyMarksEmitter(
		final ProcessingTimeService processingTimeService,
		final Output<StreamRecord<OUT>> output,
		long latencyTrackingInterval,
		final OperatorID operatorId,
		final int subtaskIndex) {

	latencyMarkTimer = processingTimeService.scheduleAtFixedRate(
		new ProcessingTimeCallback() {
			@Override
			public void onProcessingTime(long timestamp) throws Exception {
				try {
					// ProcessingTimeService callbacks are executed under the checkpointing lock
					output.emitLatencyMarker(new LatencyMarker(processingTimeService.getCurrentProcessingTime(), operatorId, subtaskIndex));
				} catch (Throwable t) {
					// we catch the Throwables here so that we don't trigger the processing
					// timer services async exception handler
					LOG.warn("Error while emitting latency marker.", t);
				}
			}
		},
		0L,
		latencyTrackingInterval);
}
 

public LatencyMarksEmitter(
		final ProcessingTimeService processingTimeService,
		final Output<StreamRecord<OUT>> output,
		long latencyTrackingInterval,
		final OperatorID operatorId,
		final int subtaskIndex) {

	latencyMarkTimer = processingTimeService.scheduleAtFixedRate(
		new ProcessingTimeCallback() {
			@Override
			public void onProcessingTime(long timestamp) throws Exception {
				try {
					// ProcessingTimeService callbacks are executed under the checkpointing lock
					output.emitLatencyMarker(new LatencyMarker(processingTimeService.getCurrentProcessingTime(), operatorId, subtaskIndex));
				} catch (Throwable t) {
					// we catch the Throwables here so that we don't trigger the processing
					// timer services async exception handler
					LOG.warn("Error while emitting latency marker.", t);
				}
			}
		},
		0L,
		latencyTrackingInterval);
}
 

public LatencyMarksEmitter(
		final ProcessingTimeService processingTimeService,
		final Output<StreamRecord<OUT>> output,
		long latencyTrackingInterval,
		final OperatorID operatorId,
		final int subtaskIndex) {

	latencyMarkTimer = processingTimeService.scheduleWithFixedDelay(
		new ProcessingTimeCallback() {
			@Override
			public void onProcessingTime(long timestamp) throws Exception {
				try {
					// ProcessingTimeService callbacks are executed under the checkpointing lock
					output.emitLatencyMarker(new LatencyMarker(processingTimeService.getCurrentProcessingTime(), operatorId, subtaskIndex));
				} catch (Throwable t) {
					// we catch the Throwables here so that we don't trigger the processing
					// timer services async exception handler
					LOG.warn("Error while emitting latency marker.", t);
				}
			}
		},
		0L,
		latencyTrackingInterval);
}
 

@Test
public void testErrorReporting() throws Exception {
	AtomicReference<Throwable> errorRef = new AtomicReference<>();
	OneShotLatch latch = new OneShotLatch();
	testHarness.getEnvironment().setExternalExceptionHandler(ex -> {
		errorRef.set(ex);
		latch.trigger();
	});

	ProcessingTimeCallback callback = timestamp -> {
		throw new Exception("Exception in Timer");
	};

	timeService.registerTimer(System.currentTimeMillis(), callback);
	latch.await();
	assertThat(errorRef.get(), instanceOf(Exception.class));
}
 

@Override
public void processElement(StreamRecord<IN> element) throws Exception {
	final StreamRecordQueueEntry<OUT> streamRecordBufferEntry = new StreamRecordQueueEntry<>(element);

	if (timeout > 0L) {
		// register a timeout for this AsyncStreamRecordBufferEntry
		long timeoutTimestamp = timeout + getProcessingTimeService().getCurrentProcessingTime();

		final ScheduledFuture<?> timerFuture = getProcessingTimeService().registerTimer(
			timeoutTimestamp,
			new ProcessingTimeCallback() {
				@Override
				public void onProcessingTime(long timestamp) throws Exception {
					userFunction.timeout(element.getValue(), streamRecordBufferEntry);
				}
			});

		// Cancel the timer once we've completed the stream record buffer entry. This will remove
		// the register trigger task
		streamRecordBufferEntry.onComplete(
			(StreamElementQueueEntry<Collection<OUT>> value) -> {
				timerFuture.cancel(true);
			},
			executor);
	}

	addAsyncBufferEntry(streamRecordBufferEntry);

	userFunction.asyncInvoke(element.getValue(), streamRecordBufferEntry);
}
 

@Override
public void processElement(StreamRecord<IN> element) throws Exception {
	final StreamRecordQueueEntry<OUT> streamRecordBufferEntry = new StreamRecordQueueEntry<>(element);

	if (timeout > 0L) {
		// register a timeout for this AsyncStreamRecordBufferEntry
		long timeoutTimestamp = timeout + getProcessingTimeService().getCurrentProcessingTime();

		final ScheduledFuture<?> timerFuture = getProcessingTimeService().registerTimer(
			timeoutTimestamp,
			new ProcessingTimeCallback() {
				@Override
				public void onProcessingTime(long timestamp) throws Exception {
					userFunction.timeout(element.getValue(), streamRecordBufferEntry);
				}
			});

		// Cancel the timer once we've completed the stream record buffer entry. This will remove
		// the register trigger task
		streamRecordBufferEntry.onComplete(
			(StreamElementQueueEntry<Collection<OUT>> value) -> {
				timerFuture.cancel(true);
			},
			executor);
	}

	addAsyncBufferEntry(streamRecordBufferEntry);

	userFunction.asyncInvoke(element.getValue(), streamRecordBufferEntry);
}
 

@Override
public void onProcessingTime(long timestamp)
        throws Exception
{
    for (RichSinkFunction<T> sink : sinks) {
        if (sink instanceof ProcessingTimeCallback) {
            ((ProcessingTimeCallback) sink).onProcessingTime(timestamp);
        }
    }
}
 

/**
 * FLINK-5652
 * Tests that registered timers are properly canceled upon completion of a
 * {@link StreamRecordQueueEntry} in order to avoid resource leaks because TriggerTasks hold
 * a reference on the StreamRecordQueueEntry.
 */
@Test
public void testTimeoutCleanup() throws Exception {
	final Object lock = new Object();

	final long timeout = 100000L;
	final long timestamp = 1L;

	Environment environment = createMockEnvironment();

	ScheduledFuture<?> scheduledFuture = mock(ScheduledFuture.class);

	ProcessingTimeService processingTimeService = mock(ProcessingTimeService.class);
	when(processingTimeService.getCurrentProcessingTime()).thenReturn(timestamp);
	doReturn(scheduledFuture).when(processingTimeService).registerTimer(anyLong(), any(ProcessingTimeCallback.class));

	StreamTask<?, ?> containingTask = mock(StreamTask.class);
	when(containingTask.getEnvironment()).thenReturn(environment);
	when(containingTask.getCheckpointLock()).thenReturn(lock);
	when(containingTask.getProcessingTimeService()).thenReturn(processingTimeService);

	StreamConfig streamConfig = new MockStreamConfig();
	streamConfig.setTypeSerializerIn1(IntSerializer.INSTANCE);

	Output<StreamRecord<Integer>> output = mock(Output.class);

	AsyncWaitOperator<Integer, Integer> operator = new AsyncWaitOperator<>(
		new AsyncFunction<Integer, Integer>() {
			private static final long serialVersionUID = -3718276118074877073L;

			@Override
			public void asyncInvoke(Integer input, ResultFuture<Integer> resultFuture) throws Exception {
				resultFuture.complete(Collections.singletonList(input));
			}
		},
		timeout,
		1,
		AsyncDataStream.OutputMode.UNORDERED);

	operator.setup(
		containingTask,
		streamConfig,
		output);

	operator.open();

	final StreamRecord<Integer> streamRecord = new StreamRecord<>(42, timestamp);

	synchronized (lock) {
		// processing an element will register a timeout
		operator.processElement(streamRecord);
	}

	synchronized (lock) {
		// closing the operator waits until all inputs have been processed
		operator.close();
	}

	// check that we actually outputted the result of the single input
	verify(output).collect(eq(streamRecord));
	verify(processingTimeService).registerTimer(eq(processingTimeService.getCurrentProcessingTime() + timeout), any(ProcessingTimeCallback.class));

	// check that we have cancelled our registered timeout
	verify(scheduledFuture).cancel(eq(true));
}
 

@Test
public void testOpenCloseAndTimestamps() throws Exception {

	final OneInputStreamTaskTestHarness<String, String> testHarness = new OneInputStreamTaskTestHarness<>(
			OneInputStreamTask::new,
			BasicTypeInfo.STRING_TYPE_INFO,
			BasicTypeInfo.STRING_TYPE_INFO);

	testHarness.setupOutputForSingletonOperatorChain();

	StreamConfig streamConfig = testHarness.getStreamConfig();

	StreamMap<String, String> mapOperator = new StreamMap<>(new DummyMapFunction<String>());
	streamConfig.setStreamOperator(mapOperator);
	streamConfig.setOperatorID(new OperatorID());

	testHarness.invoke();
	testHarness.waitForTaskRunning();

	final OneInputStreamTask<String, String> mapTask = testHarness.getTask();

	// first one spawns thread
	mapTask.getProcessingTimeService().registerTimer(System.currentTimeMillis(), new ProcessingTimeCallback() {
		@Override
		public void onProcessingTime(long timestamp) {
		}
	});

	assertEquals(1, StreamTask.TRIGGER_THREAD_GROUP.activeCount());

	testHarness.endInput();
	testHarness.waitForTaskCompletion();

	// thread needs to die in time
	long deadline = System.currentTimeMillis() + 4000;
	while (StreamTask.TRIGGER_THREAD_GROUP.activeCount() > 0 && System.currentTimeMillis() < deadline) {
		Thread.sleep(10);
	}

	assertEquals("Trigger timer thread did not properly shut down",
			0, StreamTask.TRIGGER_THREAD_GROUP.activeCount());
}
 

@Override
public ScheduledFuture<?> registerTimer(long timestamp, ProcessingTimeCallback target) {
	return FUTURE;
}
 

@Override
public ScheduledFuture<?> scheduleAtFixedRate(
	ProcessingTimeCallback callback, long initialDelay, long period) {
	return FUTURE;
}
 

/**
 * FLINK-5652
 * Tests that registered timers are properly canceled upon completion of a
 * {@link StreamRecordQueueEntry} in order to avoid resource leaks because TriggerTasks hold
 * a reference on the StreamRecordQueueEntry.
 */
@Test
public void testTimeoutCleanup() throws Exception {
	final Object lock = new Object();

	final long timeout = 100000L;
	final long timestamp = 1L;

	Environment environment = createMockEnvironment();

	ScheduledFuture<?> scheduledFuture = mock(ScheduledFuture.class);

	ProcessingTimeService processingTimeService = mock(ProcessingTimeService.class);
	when(processingTimeService.getCurrentProcessingTime()).thenReturn(timestamp);
	doReturn(scheduledFuture).when(processingTimeService).registerTimer(anyLong(), any(ProcessingTimeCallback.class));

	StreamTask<?, ?> containingTask = mock(StreamTask.class);
	when(containingTask.getEnvironment()).thenReturn(environment);
	when(containingTask.getCheckpointLock()).thenReturn(lock);
	when(containingTask.getProcessingTimeService()).thenReturn(processingTimeService);

	StreamConfig streamConfig = new MockStreamConfig();
	streamConfig.setTypeSerializerIn1(IntSerializer.INSTANCE);

	Output<StreamRecord<Integer>> output = mock(Output.class);

	AsyncWaitOperator<Integer, Integer> operator = new AsyncWaitOperator<>(
		new AsyncFunction<Integer, Integer>() {
			private static final long serialVersionUID = -3718276118074877073L;

			@Override
			public void asyncInvoke(Integer input, ResultFuture<Integer> resultFuture) throws Exception {
				resultFuture.complete(Collections.singletonList(input));
			}
		},
		timeout,
		1,
		AsyncDataStream.OutputMode.UNORDERED);

	operator.setup(
		containingTask,
		streamConfig,
		output);

	operator.open();

	final StreamRecord<Integer> streamRecord = new StreamRecord<>(42, timestamp);

	synchronized (lock) {
		// processing an element will register a timeout
		operator.processElement(streamRecord);
	}

	synchronized (lock) {
		// closing the operator waits until all inputs have been processed
		operator.close();
	}

	// check that we actually outputted the result of the single input
	verify(output).collect(eq(streamRecord));
	verify(processingTimeService).registerTimer(eq(processingTimeService.getCurrentProcessingTime() + timeout), any(ProcessingTimeCallback.class));

	// check that we have cancelled our registered timeout
	verify(scheduledFuture).cancel(eq(true));
}
 

@Test
public void testOpenCloseAndTimestamps() throws Exception {

	final OneInputStreamTaskTestHarness<String, String> testHarness = new OneInputStreamTaskTestHarness<>(
			OneInputStreamTask::new,
			BasicTypeInfo.STRING_TYPE_INFO,
			BasicTypeInfo.STRING_TYPE_INFO);

	testHarness.setupOutputForSingletonOperatorChain();

	StreamConfig streamConfig = testHarness.getStreamConfig();

	StreamMap<String, String> mapOperator = new StreamMap<>(new DummyMapFunction<String>());
	streamConfig.setStreamOperator(mapOperator);
	streamConfig.setOperatorID(new OperatorID());

	testHarness.invoke();
	testHarness.waitForTaskRunning();

	final OneInputStreamTask<String, String> mapTask = testHarness.getTask();

	// first one spawns thread
	mapTask.getProcessingTimeService().registerTimer(System.currentTimeMillis(), new ProcessingTimeCallback() {
		@Override
		public void onProcessingTime(long timestamp) {
		}
	});

	assertEquals(1, StreamTask.TRIGGER_THREAD_GROUP.activeCount());

	testHarness.endInput();
	testHarness.waitForTaskCompletion();

	// thread needs to die in time
	long deadline = System.currentTimeMillis() + 4000;
	while (StreamTask.TRIGGER_THREAD_GROUP.activeCount() > 0 && System.currentTimeMillis() < deadline) {
		Thread.sleep(10);
	}

	assertEquals("Trigger timer thread did not properly shut down",
			0, StreamTask.TRIGGER_THREAD_GROUP.activeCount());
}
 

@Override
public ScheduledFuture<?> registerTimer(long timestamp, ProcessingTimeCallback target) {
	return FUTURE;
}
 

@Override
public ScheduledFuture<?> scheduleAtFixedRate(
	ProcessingTimeCallback callback, long initialDelay, long period) {
	return FUTURE;
}
 

@Override
public ScheduledFuture<?> scheduleWithFixedDelay(
	ProcessingTimeCallback callback, long initialDelay, long period) {
	return FUTURE;
}