org.apache.flink.streaming.api.operators.InternalTimer Java Examples

@Override
public void onProcessingTime(InternalTimer<String, VoidNamespace> timer) throws Exception {
  final long triggerTimestamp = timer.getTimestamp();
  final Reductions reductions = reductionsSupplier.get();

  Iterable<Message> delayedMessages = delayedMessagesBuffer.getForTimestamp(triggerTimestamp);
  if (delayedMessages == null) {
    throw new IllegalStateException(
        "A delayed message timer was triggered with timestamp "
            + triggerTimestamp
            + ", but no messages were buffered for it.");
  }
  for (Message delayedMessage : delayedMessages) {
    if (thisPartition.contains(delayedMessage.target())) {
      reductions.enqueue(delayedMessage);
    } else {
      remoteSink.accept(delayedMessage);
    }
  }
  // we clear the delayedMessageBuffer *before* we process the enqueued local reductions, because
  // processing the envelops might actually trigger a delayed message to be sent with the same
  // @triggerTimestamp
  // so it would be re-enqueued into the delayedMessageBuffer.
  delayedMessagesBuffer.clearForTimestamp(triggerTimestamp);
  reductions.processEnvelopes();
}
 

@Override
public void onProcessingTime(InternalTimer<String, VoidNamespace> timer) throws Exception {
  final long triggerTimestamp = timer.getTimestamp();
  final Reductions reductions = reductionsSupplier.get();

  Iterable<Message> delayedMessages = delayedMessagesBuffer.getForTimestamp(triggerTimestamp);
  if (delayedMessages == null) {
    throw new IllegalStateException(
        "A delayed message timer was triggered with timestamp "
            + triggerTimestamp
            + ", but no messages were buffered for it.");
  }
  for (Message delayedMessage : delayedMessages) {
    if (thisPartition.contains(delayedMessage.target())) {
      reductions.enqueue(delayedMessage);
    } else {
      remoteSink.accept(delayedMessage);
    }
  }
  // we clear the delayedMessageBuffer *before* we process the enqueued local reductions, because
  // processing the envelops might actually trigger a delayed message to be sent with the same
  // @triggerTimestamp
  // so it would be re-enqueued into the delayedMessageBuffer.
  delayedMessagesBuffer.clearForTimestamp(triggerTimestamp);
  reductions.processEnvelopes();
}
 

@Override
public void onEventTime(InternalTimer<Object, VoidNamespace> timer) throws Exception {
	registeredTimer.clear();
	long lastUnprocessedTime = emitResultAndCleanUpState(timerService.currentWatermark());
	if (lastUnprocessedTime < Long.MAX_VALUE) {
		registerTimer(lastUnprocessedTime);
	}

	// if we have more state at any side, then update the timer, else clean it up.
	if (stateCleaningEnabled) {
		if (lastUnprocessedTime < Long.MAX_VALUE || rightState.iterator().hasNext()) {
			registerProcessingCleanupTimer();
		} else {
			cleanupLastTimer();
		}
	}
}
 

@Override
public void onEventTime(InternalTimer<Object, VoidNamespace> timer) throws Exception {
	registeredTimer.clear();
	long lastUnprocessedTime = emitResultAndCleanUpState(timerService.currentWatermark());
	if (lastUnprocessedTime < Long.MAX_VALUE) {
		registerTimer(lastUnprocessedTime);
	}

	// if we have more state at any side, then update the timer, else clean it up.
	if (stateCleaningEnabled) {
		if (lastUnprocessedTime < Long.MAX_VALUE || !rightState.isEmpty()) {
			registerProcessingCleanupTimer();
		} else {
			cleanupLastTimer();
		}
	}
}
 

@Override
public void onEventTime(InternalTimer<RowData, VoidNamespace> timer) throws Exception {
	long timestamp = timer.getTimestamp();

	// gets all rows for the triggering timestamps
	List<RowData> inputs = dataState.get(timestamp);
	if (inputs != null) {
		// sort rows on secondary fields if necessary
		if (comparator != null) {
			inputs.sort(comparator);
		}

		// emit rows in order
		inputs.forEach((RowData row) -> collector.collect(row));

		// remove emitted rows from state
		dataState.remove(timestamp);
		lastTriggeringTsState.update(timestamp);
	}
}
 

@Override
public void onProcessingTime(InternalTimer<RowData, VoidNamespace> timer) throws Exception {

	// gets all rows for the triggering timestamps
	Iterable<RowData> inputs = dataState.get();

	// insert all rows into the sort buffer
	sortBuffer.clear();
	inputs.forEach(sortBuffer::add);

	// sort the rows
	sortBuffer.sort(comparator);

	// Emit the rows in order
	sortBuffer.forEach((RowData row) -> collector.collect(row));

	// remove all buffered rows
	dataState.clear();
}
 

@Override
public void onProcessingTime(InternalTimer<K, W> timer) throws Exception {
	if (functionsClosed) {
		return;
	}

	setCurrentKey(timer.getKey());

	triggerContext.window = timer.getNamespace();
	if (triggerContext.onProcessingTime(timer.getTimestamp())) {
		// fire
		emitWindowResult(triggerContext.window);
	}

	if (!windowAssigner.isEventTime()) {
		windowFunction.cleanWindowIfNeeded(triggerContext.window, timer.getTimestamp());
	}
}
 

@Override
public void onProcessingTime(InternalTimer<BaseRow, VoidNamespace> timer) throws Exception {

	// gets all rows for the triggering timestamps
	Iterable<BaseRow> inputs = dataState.get();

	// insert all rows into the sort buffer
	sortBuffer.clear();
	inputs.forEach(sortBuffer::add);

	// sort the rows
	sortBuffer.sort(comparator);

	// Emit the rows in order
	sortBuffer.forEach((BaseRow row) -> collector.collect(row));

	// remove all buffered rows
	dataState.clear();
}
 

/**
 * Occurs when an event-time timer fires due to watermark progression.
 *
 * @param timer the timer details.
 */
@Override
public void onEventTime(InternalTimer<K, VoidNamespace> timer) throws Exception {

    long currentWatermark = internalTimerService.currentWatermark();

    PriorityQueue<Long> sortedTimestamps = getSortedTimestamps();
    while (!sortedTimestamps.isEmpty() && sortedTimestamps.peek() <= currentWatermark) {
        long timestamp = sortedTimestamps.poll();
        for (T event : elementQueueState.get(timestamp)) {
            output.collect(new StreamRecord<>(event, timestamp));
        }
        elementQueueState.remove(timestamp);
    }

    if (sortedTimestamps.isEmpty()) {
        elementQueueState.clear();
    }

    if (!sortedTimestamps.isEmpty()) {
        saveRegisterWatermarkTimer();
    }
}
 

@Override
public void onEventTime(InternalTimer<K, VoidNamespace> timer) throws Exception {
	collector.setAbsoluteTimestamp(timer.getTimestamp());
	onTimerContext.timeDomain = TimeDomain.EVENT_TIME;
	onTimerContext.timer = timer;
	userFunction.onTimer(timer.getTimestamp(), onTimerContext, collector);
	onTimerContext.timeDomain = null;
	onTimerContext.timer = null;
}
 

@Override
public void onProcessingTime(InternalTimer<Long, Long> timer) throws Exception {
	ValueState<Long> state = getKeyedStateBackend().getPartitionedState(
		timer.getNamespace(),
		LongSerializer.INSTANCE,
		stateDescriptor);

	assertEquals(state.value(), timer.getNamespace());
	getRuntimeContext().getAccumulator(SUCCESSFUL_PROCESSING_TIME_CHECK_ACCUMULATOR).add(1);
}
 

@Override
public void onEventTime(InternalTimer<KS, VoidNamespace> timer) throws Exception {
	collector.setAbsoluteTimestamp(timer.getTimestamp());
	onTimerContext.timeDomain = TimeDomain.EVENT_TIME;
	onTimerContext.timer = timer;
	userFunction.onTimer(timer.getTimestamp(), onTimerContext, collector);
	onTimerContext.timeDomain = null;
	onTimerContext.timer = null;
}
 

@Override
public void onProcessingTime(InternalTimer<K, VoidNamespace> timer) throws Exception {
	collector.eraseTimestamp();
	onTimerContext.timeDomain = TimeDomain.PROCESSING_TIME;
	onTimerContext.timer = timer;
	userFunction.onTimer(timer.getTimestamp(), onTimerContext, collector);
	onTimerContext.timeDomain = null;
	onTimerContext.timer = null;
}
 

@Override
public void onProcessingTime(InternalTimer<KEY, VoidNamespace> timer) throws Exception {
	// 1) get the queue of pending elements for the key and the corresponding NFA,
	// 2) process the pending elements in process time order and custom comparator if exists
	//		by feeding them in the NFA
	// 3) update the stored state for the key, by only storing the new NFA and MapState iff they
	//		have state to be used later.

	// STEP 1
	PriorityQueue<Long> sortedTimestamps = getSortedTimestamps();
	NFAState nfa = getNFAState();

	// STEP 2
	while (!sortedTimestamps.isEmpty()) {
		long timestamp = sortedTimestamps.poll();
		advanceTime(nfa, timestamp);
		try (Stream<IN> elements = sort(elementQueueState.get(timestamp))) {
			elements.forEachOrdered(
				event -> {
					try {
						processEvent(nfa, event, timestamp);
					} catch (Exception e) {
						throw new RuntimeException(e);
					}
				}
			);
		}
		elementQueueState.remove(timestamp);
	}

	// STEP 3
	updateNFA(nfa);
}
 

@Override
public void onProcessingTime(InternalTimer<Long, Long> timer) throws Exception {
	ValueState<Long> state = getKeyedStateBackend().getPartitionedState(
		timer.getNamespace(),
		LongSerializer.INSTANCE,
		stateDescriptor);

	assertEquals(state.value(), timer.getNamespace());
	getRuntimeContext().getAccumulator(SUCCESSFUL_PROCESSING_TIME_CHECK_ACCUMULATOR).add(1);
}
 

@Override
public void onEventTime(InternalTimer<Long, Long> timer) throws Exception {
	ValueState<Long> state = getKeyedStateBackend().getPartitionedState(
		timer.getNamespace(),
		LongSerializer.INSTANCE,
		stateDescriptor);

	assertEquals(state.value(), timer.getNamespace());
	getRuntimeContext().getAccumulator(SUCCESSFUL_EVENT_TIME_CHECK_ACCUMULATOR).add(1);
}
 

@Override
public void onEventTime(InternalTimer<Long, Long> timer) throws Exception {
	ValueState<Long> state = getKeyedStateBackend().getPartitionedState(
		timer.getNamespace(),
		LongSerializer.INSTANCE,
		stateDescriptor);

	assertEquals(state.value(), timer.getNamespace());
	getRuntimeContext().getAccumulator(SUCCESSFUL_EVENT_TIME_CHECK_ACCUMULATOR).add(1);
}
 

@Override
public void onProcessingTime(InternalTimer<K, VoidNamespace> timer) throws Exception {
	collector.eraseTimestamp();
	onTimerContext.timeDomain = TimeDomain.PROCESSING_TIME;
	onTimerContext.timer = timer;
	userFunction.onTimer(timer.getTimestamp(), onTimerContext, collector);
	onTimerContext.timeDomain = null;
	onTimerContext.timer = null;
}
 

@Override
public void onEventTime(InternalTimer<K, VoidNamespace> timer) throws Exception {
	collector.setAbsoluteTimestamp(timer.getTimestamp());
	onTimerContext.timeDomain = TimeDomain.EVENT_TIME;
	onTimerContext.timer = timer;
	userFunction.onTimer(timer.getTimestamp(), onTimerContext, collector);
	onTimerContext.timeDomain = null;
	onTimerContext.timer = null;
}
 

@Override
public void onEventTime(InternalTimer<K, VoidNamespace> timer) throws Exception {
	collector.setAbsoluteTimestamp(timer.getTimestamp());
	onTimerContext.timeDomain = TimeDomain.EVENT_TIME;
	onTimerContext.timer = timer;
	userFunction.onTimer(timer.getTimestamp(), onTimerContext, collector);
	onTimerContext.timeDomain = null;
	onTimerContext.timer = null;
}
 

@Override
public void onProcessingTime(InternalTimer<K, VoidNamespace> timer) throws Exception {
	collector.eraseTimestamp();
	onTimerContext.timeDomain = TimeDomain.PROCESSING_TIME;
	onTimerContext.timer = timer;
	userFunction.onTimer(timer.getTimestamp(), onTimerContext, collector);
	onTimerContext.timeDomain = null;
	onTimerContext.timer = null;
}
 

@Override
public void onProcessingTime(InternalTimer<KEY, VoidNamespace> timer) throws Exception {
	// 1) get the queue of pending elements for the key and the corresponding NFA,
	// 2) process the pending elements in process time order and custom comparator if exists
	//		by feeding them in the NFA
	// 3) update the stored state for the key, by only storing the new NFA and MapState iff they
	//		have state to be used later.

	// STEP 1
	PriorityQueue<Long> sortedTimestamps = getSortedTimestamps();
	NFAState nfa = getNFAState();

	// STEP 2
	while (!sortedTimestamps.isEmpty()) {
		long timestamp = sortedTimestamps.poll();
		advanceTime(nfa, timestamp);
		try (Stream<IN> elements = sort(elementQueueState.get(timestamp))) {
			elements.forEachOrdered(
				event -> {
					try {
						processEvent(nfa, event, timestamp);
					} catch (Exception e) {
						throw new RuntimeException(e);
					}
				}
			);
		}
		elementQueueState.remove(timestamp);
	}

	// STEP 3
	updateNFA(nfa);
}
 

/**
 * Processes all pending processing timers. This is intended for use during shutdown. From Flink
 * 1.10 on, processing timer execution is stopped when the operator is closed. This leads to
 * problems for applications which assume all pending timers will be completed. Although Flink
 * does drain the remaining timers after close(), this is not sufficient because no new timers
 * are allowed to be scheduled anymore. This breaks Beam pipelines which rely on all processing
 * timers to be scheduled and executed.
 */
void processPendingProcessingTimeTimers() {
  final KeyedStateBackend<Object> keyedStateBackend = getKeyedStateBackend();
  final InternalPriorityQueue<InternalTimer<Object, TimerData>> processingTimeTimersQueue =
      Workarounds.retrieveInternalProcessingTimerQueue(timerService);

  InternalTimer<Object, TimerData> internalTimer;
  while ((internalTimer = processingTimeTimersQueue.poll()) != null) {
    keyedStateBackend.setCurrentKey(internalTimer.getKey());
    TimerData timer = internalTimer.getNamespace();
    checkInvokeStartBundle();
    fireTimer(timer);
  }
}
 

@Override
public void onProcessingTime(InternalTimer<ByteBuffer, VoidNamespace> internalTimer)
    throws Exception {
  ValueState<Long> dedupingState = getPartitionedState(dedupingStateDescriptor);

  Long lastSeenTimestamp = dedupingState.value();
  if (lastSeenTimestamp != null
      && lastSeenTimestamp.equals(internalTimer.getTimestamp() - MAX_RETENTION_SINCE_ACCESS)) {
    dedupingState.clear();
  }
}
 

@Override
public void onEventTime(InternalTimer<K, W> timer) throws Exception {
	setCurrentKey(timer.getKey());

	triggerContext.window = timer.getNamespace();
	if (triggerContext.onEventTime(timer.getTimestamp())) {
		// fire
		emitWindowResult(triggerContext.window);
	}

	if (windowAssigner.isEventTime()) {
		windowFunction.cleanWindowIfNeeded(triggerContext.window, timer.getTimestamp());
	}
}
 

/**
 * The users of this class are not allowed to use processing time timers.
 * See class javadoc.
 */
@Override
public final void onProcessingTime(InternalTimer<Object, VoidNamespace> timer) throws Exception {
	if (stateCleaningEnabled) {
		long timerTime = timer.getTimestamp();
		Long cleanupTime = latestRegisteredCleanupTimer.value();

		if (cleanupTime != null && cleanupTime == timerTime) {
			cleanupState(cleanupTime);
			latestRegisteredCleanupTimer.clear();
		}
	}
}
 

@Override
public void onEventTime(InternalTimer<Long, Long> timer) throws Exception {
	ValueState<Long> state = getKeyedStateBackend().getPartitionedState(
		timer.getNamespace(),
		LongSerializer.INSTANCE,
		stateDescriptor);

	assertEquals(state.value(), timer.getNamespace());
	getRuntimeContext().getAccumulator(SUCCESSFUL_EVENT_TIME_CHECK_ACCUMULATOR).add(1);
}
 

@Override
public void onProcessingTime(InternalTimer<Long, Long> timer) throws Exception {
	ValueState<Long> state = getKeyedStateBackend().getPartitionedState(
		timer.getNamespace(),
		LongSerializer.INSTANCE,
		stateDescriptor);

	assertEquals(state.value(), timer.getNamespace());
	getRuntimeContext().getAccumulator(SUCCESSFUL_PROCESSING_TIME_CHECK_ACCUMULATOR).add(1);
}
 

@Override
public void onEventTime(InternalTimer<Long, Long> timer) throws Exception {
	ValueState<Long> state = getKeyedStateBackend().getPartitionedState(
		timer.getNamespace(),
		LongSerializer.INSTANCE,
		stateDescriptor);

	assertEquals(state.value(), timer.getNamespace());
	getRuntimeContext().getAccumulator(SUCCESSFUL_EVENT_TIME_CHECK_ACCUMULATOR).add(1);
}
 

@Override
public void onProcessingTime(InternalTimer<KS, VoidNamespace> timer) throws Exception {
	collector.eraseTimestamp();
	onTimerContext.timeDomain = TimeDomain.PROCESSING_TIME;
	onTimerContext.timer = timer;
	userFunction.onTimer(timer.getTimestamp(), onTimerContext, collector);
	onTimerContext.timeDomain = null;
	onTimerContext.timer = null;
}