Java Code Examples for org.apache.flink.streaming.runtime.tasks.ProcessingTimeService#registerTimer()

@SuppressWarnings("FutureReturnValueIgnored")
private void finishBundleCallback() {
  minEventTimeTimerTimestampInLastBundle = minEventTimeTimerTimestampInCurrentBundle;
  minEventTimeTimerTimestampInCurrentBundle = Long.MAX_VALUE;
  try {
    if (!closed
        && minEventTimeTimerTimestampInLastBundle < Long.MAX_VALUE
        && minEventTimeTimerTimestampInLastBundle <= getEffectiveInputWatermark()) {
      ProcessingTimeService processingTimeService = getProcessingTimeService();
      // We are scheduling a timer for advancing the watermark, to not delay finishing the bundle
      // and temporarily release the checkpoint lock. Otherwise, we could potentially loop when a
      // timer keeps scheduling a timer for the same timestamp.
      processingTimeService.registerTimer(
          processingTimeService.getCurrentProcessingTime(),
          ts -> processWatermark1(new Watermark(getEffectiveInputWatermark())));
    } else {
      processWatermark1(new Watermark(getEffectiveInputWatermark()));
    }
  } catch (Exception e) {
    throw new RuntimeException(
        "Failed to progress watermark to " + getEffectiveInputWatermark(), e);
  }
}
 

public StreamingFileSinkHelper(
		Buckets<IN, ?> buckets,
		boolean isRestored,
		OperatorStateStore stateStore,
		ProcessingTimeService procTimeService,
		long bucketCheckInterval) throws Exception {
	this.bucketCheckInterval = bucketCheckInterval;
	this.buckets = buckets;
	this.bucketStates = stateStore.getListState(BUCKET_STATE_DESC);
	this.maxPartCountersState = stateStore.getUnionListState(MAX_PART_COUNTER_STATE_DESC);
	this.procTimeService = procTimeService;

	if (isRestored) {
		buckets.initializeState(bucketStates, maxPartCountersState);
	}

	long currentProcessingTime = procTimeService.getCurrentProcessingTime();
	procTimeService.registerTimer(currentProcessingTime + bucketCheckInterval, this);
}
 

@Override
public void processElement(StreamRecord<String> element) throws Exception {
	if (!isTriggerEvent(element)) {
		// Pass through entries that are not triggers as is, so that the test can observe them.
		output.collect(element);
		return;
	}

	// The test operator creates a one-time timer (per input element) and passes the input element further
	// (to the next operator or to the output).
	// The execution is yielded until the operator's timer trigger is confirmed.

	int index = count;
	ProcessingTimeService processingTimeService = getProcessingTimeService();
	processingTimeService
		.registerTimer(
			processingTimeService.getCurrentProcessingTime() + 1000L,
			timestamp -> {
				output.collect(new StreamRecord<>(RESULT_PREFIX + chainIndex + ":" + index));
				--count;
			});

	++count;
	output.collect(element);

	while (count > 0) {
		mailboxExecutor.yield();
	}
}
 

@Test
public void checkScheduledTimestampe() throws Exception {
	final OneInputStreamTaskTestHarness<String, String> testHarness = new OneInputStreamTaskTestHarness<>(
			OneInputStreamTask::new,
			BasicTypeInfo.STRING_TYPE_INFO,
			BasicTypeInfo.STRING_TYPE_INFO);

	testHarness.setupOutputForSingletonOperatorChain();

	StreamConfig streamConfig = testHarness.getStreamConfig();
	StreamMap<String, String> mapOperator = new StreamMap<>(new DummyMapFunction<String>());
	streamConfig.setStreamOperator(mapOperator);

	testHarness.invoke();
	testHarness.waitForTaskRunning();

	final OneInputStreamTask<String, String> mapTask = testHarness.getTask();

	final AtomicReference<Throwable> errorRef = new AtomicReference<>();

	final long t1 = System.currentTimeMillis();
	final long t2 = System.currentTimeMillis() - 200;
	final long t3 = System.currentTimeMillis() + 100;
	final long t4 = System.currentTimeMillis() + 200;

	ProcessingTimeService timeService = mapTask.getProcessingTimeService();
	timeService.registerTimer(t1, new ValidatingProcessingTimeCallback(errorRef, t1, 0));
	timeService.registerTimer(t2, new ValidatingProcessingTimeCallback(errorRef, t2, 1));
	timeService.registerTimer(t3, new ValidatingProcessingTimeCallback(errorRef, t3, 2));
	timeService.registerTimer(t4, new ValidatingProcessingTimeCallback(errorRef, t4, 3));

	long deadline = System.currentTimeMillis() + 20000;
	while (errorRef.get() == null &&
			ValidatingProcessingTimeCallback.numInSequence < 4 &&
			System.currentTimeMillis() < deadline) {
		Thread.sleep(100);
	}

	// handle errors
	if (errorRef.get() != null) {
		errorRef.get().printStackTrace();
		fail(errorRef.get().getMessage());
	}

	assertEquals(4, ValidatingProcessingTimeCallback.numInSequence);

	testHarness.endInput();
	testHarness.waitForTaskCompletion();

	// wait until the trigger thread is shut down. otherwise, the other tests may become unstable
	deadline = System.currentTimeMillis() + 4000;
	while (StreamTask.TRIGGER_THREAD_GROUP.activeCount() > 0 && System.currentTimeMillis() < deadline) {
		Thread.sleep(10);
	}

	assertEquals("Trigger timer thread did not properly shut down",
			0, StreamTask.TRIGGER_THREAD_GROUP.activeCount());
}
 

@Test
public void checkScheduledTimestampe() throws Exception {
	final OneInputStreamTaskTestHarness<String, String> testHarness = new OneInputStreamTaskTestHarness<>(
			OneInputStreamTask::new,
			BasicTypeInfo.STRING_TYPE_INFO,
			BasicTypeInfo.STRING_TYPE_INFO);

	testHarness.setupOutputForSingletonOperatorChain();

	StreamConfig streamConfig = testHarness.getStreamConfig();
	StreamMap<String, String> mapOperator = new StreamMap<>(new DummyMapFunction<String>());
	streamConfig.setStreamOperator(mapOperator);

	testHarness.invoke();
	testHarness.waitForTaskRunning();

	final OneInputStreamTask<String, String> mapTask = testHarness.getTask();

	final AtomicReference<Throwable> errorRef = new AtomicReference<>();

	final long t1 = System.currentTimeMillis();
	final long t2 = System.currentTimeMillis() - 200;
	final long t3 = System.currentTimeMillis() + 100;
	final long t4 = System.currentTimeMillis() + 200;

	ProcessingTimeService timeService = mapTask.getProcessingTimeService();
	timeService.registerTimer(t1, new ValidatingProcessingTimeCallback(errorRef, t1, 0));
	timeService.registerTimer(t2, new ValidatingProcessingTimeCallback(errorRef, t2, 1));
	timeService.registerTimer(t3, new ValidatingProcessingTimeCallback(errorRef, t3, 2));
	timeService.registerTimer(t4, new ValidatingProcessingTimeCallback(errorRef, t4, 3));

	long deadline = System.currentTimeMillis() + 20000;
	while (errorRef.get() == null &&
			ValidatingProcessingTimeCallback.numInSequence < 4 &&
			System.currentTimeMillis() < deadline) {
		Thread.sleep(100);
	}

	// handle errors
	if (errorRef.get() != null) {
		errorRef.get().printStackTrace();
		fail(errorRef.get().getMessage());
	}

	assertEquals(4, ValidatingProcessingTimeCallback.numInSequence);

	testHarness.endInput();
	testHarness.waitForTaskCompletion();

	// wait until the trigger thread is shut down. otherwise, the other tests may become unstable
	deadline = System.currentTimeMillis() + 4000;
	while (StreamTask.TRIGGER_THREAD_GROUP.activeCount() > 0 && System.currentTimeMillis() < deadline) {
		Thread.sleep(10);
	}

	assertEquals("Trigger timer thread did not properly shut down",
			0, StreamTask.TRIGGER_THREAD_GROUP.activeCount());
}
 

@Test
public void testCustomTimeServiceProvider() throws Throwable {
	final TestProcessingTimeService tp = new TestProcessingTimeService();

	final OneInputStreamTaskTestHarness<String, String> testHarness = new OneInputStreamTaskTestHarness<>(
			(env) -> new OneInputStreamTask<>(env, tp),
			BasicTypeInfo.STRING_TYPE_INFO,
			BasicTypeInfo.STRING_TYPE_INFO);

	testHarness.setupOutputForSingletonOperatorChain();

	StreamConfig streamConfig = testHarness.getStreamConfig();

	StreamMap<String, String> mapOperator = new StreamMap<>(new StreamTaskTimerTest.DummyMapFunction<>());
	streamConfig.setStreamOperator(mapOperator);
	streamConfig.setOperatorID(new OperatorID());

	testHarness.invoke();
	testHarness.waitForTaskRunning();

	ProcessingTimeService processingTimeService = ((StreamMap<?, ?>) testHarness.getHeadOperator()).getProcessingTimeService();

	assertEquals(Long.MIN_VALUE, processingTimeService.getCurrentProcessingTime());

	tp.setCurrentTime(11);
	assertEquals(processingTimeService.getCurrentProcessingTime(), 11);

	tp.setCurrentTime(15);
	tp.setCurrentTime(16);
	assertEquals(processingTimeService.getCurrentProcessingTime(), 16);

	// register 2 tasks
	processingTimeService.registerTimer(30, timestamp -> {});

	processingTimeService.registerTimer(40, timestamp -> {});

	assertEquals(2, tp.getNumActiveTimers());

	tp.setCurrentTime(35);
	assertEquals(1, tp.getNumActiveTimers());

	tp.setCurrentTime(40);
	assertEquals(0, tp.getNumActiveTimers());

	tp.shutdownService();
}