org.apache.flink.cep.nfa.NFAState Java Examples

@Test
public void testProcessingTimestampisPassedToNFA() throws Exception {

	final NFA<Event> nfa = NFACompiler.compileFactory(Pattern.<Event>begin("begin"), true).createNFA();
	final NFA<Event> spyNFA = spy(nfa);

	try (
		OneInputStreamOperatorTestHarness<Event, Map<String, List<Event>>> harness =
			CepOperatorTestUtilities.getCepTestHarness(createOperatorForNFA(spyNFA).build())) {

		long timestamp = 5;
		harness.open();
		harness.setProcessingTime(timestamp);
		StreamRecord<Event> event = event().withTimestamp(3).asStreamRecord();
		harness.processElement(event);
		verify(spyNFA).process(
			any(SharedBufferAccessor.class),
			any(NFAState.class),
			eq(event.getValue()),
			eq(timestamp),
			any(AfterMatchSkipStrategy.class),
			any(TimerService.class));
	}
}
 

private void migrateOldState() throws Exception {
	getKeyedStateBackend().applyToAllKeys(
		VoidNamespace.INSTANCE,
		VoidNamespaceSerializer.INSTANCE,
		new ValueStateDescriptor<>(
			"nfaOperatorStateName",
			new NFA.NFASerializer<>(inputSerializer)
		),
		new KeyedStateFunction<Object, ValueState<MigratedNFA<IN>>>() {
			@Override
			public void process(Object key, ValueState<MigratedNFA<IN>> state) throws Exception {
				MigratedNFA<IN> oldState = state.value();
				computationStates.update(new NFAState(oldState.getComputationStates()));
				org.apache.flink.cep.nfa.SharedBuffer<IN> sharedBuffer = oldState.getSharedBuffer();
				partialMatches.init(sharedBuffer.getEventsBuffer(), sharedBuffer.getPages());
				state.clear();
			}
		}
	);
}
 

@Test
public void testProcessingTimestampisPassedToNFA() throws Exception {

	final NFA<Event> nfa = NFACompiler.compileFactory(Pattern.<Event>begin("begin"), true).createNFA();
	final NFA<Event> spyNFA = spy(nfa);

	try (
		OneInputStreamOperatorTestHarness<Event, Map<String, List<Event>>> harness =
			CepOperatorTestUtilities.getCepTestHarness(createOperatorForNFA(spyNFA).build())) {

		long timestamp = 5;
		harness.open();
		harness.setProcessingTime(timestamp);
		StreamRecord<Event> event = event().withTimestamp(3).asStreamRecord();
		harness.processElement(event);
		verify(spyNFA).process(
			any(SharedBufferAccessor.class),
			any(NFAState.class),
			eq(event.getValue()),
			eq(timestamp),
			any(AfterMatchSkipStrategy.class),
			any(TimerService.class));
	}
}
 

private void migrateOldState() throws Exception {
	getKeyedStateBackend().applyToAllKeys(
		VoidNamespace.INSTANCE,
		VoidNamespaceSerializer.INSTANCE,
		new ValueStateDescriptor<>(
			"nfaOperatorStateName",
			new NFA.NFASerializer<>(inputSerializer)
		),
		new KeyedStateFunction<Object, ValueState<MigratedNFA<IN>>>() {
			@Override
			public void process(Object key, ValueState<MigratedNFA<IN>> state) throws Exception {
				MigratedNFA<IN> oldState = state.value();
				computationStates.update(new NFAState(oldState.getComputationStates()));
				org.apache.flink.cep.nfa.SharedBuffer<IN> sharedBuffer = oldState.getSharedBuffer();
				partialMatches.init(sharedBuffer.getEventsBuffer(), sharedBuffer.getPages());
				state.clear();
			}
		}
	);
}
 

@Test
public void testProcessingTimestampisPassedToNFA() throws Exception {

	final NFA<Event> nfa = NFACompiler.compileFactory(Pattern.<Event>begin("begin"), true).createNFA();
	final NFA<Event> spyNFA = spy(nfa);

	try (
		OneInputStreamOperatorTestHarness<Event, Map<String, List<Event>>> harness =
			CepOperatorTestUtilities.getCepTestHarness(createOperatorForNFA(spyNFA).build())) {

		long timestamp = 5;
		harness.open();
		harness.setProcessingTime(timestamp);
		StreamRecord<Event> event = event().withTimestamp(3).asStreamRecord();
		harness.processElement(event);
		verify(spyNFA).process(
			any(SharedBufferAccessor.class),
			any(NFAState.class),
			eq(event.getValue()),
			eq(timestamp),
			any(AfterMatchSkipStrategy.class),
			any(TimerService.class));
	}
}
 

/**
 * Advances the time for the given NFA to the given timestamp. This means that no more events with timestamp
 * <b>lower</b> than the given timestamp should be passed to the nfa, This can lead to pruning and timeouts.
 */
private void advanceTime(NFAState nfaState, long timestamp) throws Exception {
	try (SharedBufferAccessor<IN> sharedBufferAccessor = partialMatches.getAccessor()) {
		Collection<Tuple2<Map<String, List<IN>>, Long>> timedOut =
				nfa.advanceTime(sharedBufferAccessor, nfaState, timestamp);
		if (!timedOut.isEmpty()) {
			processTimedOutSequences(timedOut);
		}
	}
}
 

/**
 * Advances the time for the given NFA to the given timestamp. This means that no more events with timestamp
 * <b>lower</b> than the given timestamp should be passed to the nfa, This can lead to pruning and timeouts.
 */
private void advanceTime(NFAState nfaState, long timestamp) throws Exception {
	try (SharedBufferAccessor<IN> sharedBufferAccessor = partialMatches.getAccessor()) {
		Collection<Tuple2<Map<String, List<IN>>, Long>> timedOut =
				nfa.advanceTime(sharedBufferAccessor, nfaState, timestamp);
		if (!timedOut.isEmpty()) {
			processTimedOutSequences(timedOut);
		}
	}
}
 

@Override
public void onProcessingTime(InternalTimer<KEY, VoidNamespace> timer) throws Exception {
	// 1) get the queue of pending elements for the key and the corresponding NFA,
	// 2) process the pending elements in process time order and custom comparator if exists
	//		by feeding them in the NFA
	// 3) update the stored state for the key, by only storing the new NFA and MapState iff they
	//		have state to be used later.

	// STEP 1
	PriorityQueue<Long> sortedTimestamps = getSortedTimestamps();
	NFAState nfa = getNFAState();

	// STEP 2
	while (!sortedTimestamps.isEmpty()) {
		long timestamp = sortedTimestamps.poll();
		advanceTime(nfa, timestamp);
		try (Stream<IN> elements = sort(elementQueueState.get(timestamp))) {
			elements.forEachOrdered(
				event -> {
					try {
						processEvent(nfa, event, timestamp);
					} catch (Exception e) {
						throw new RuntimeException(e);
					}
				}
			);
		}
		elementQueueState.remove(timestamp);
	}

	// STEP 3
	updateNFA(nfa);
}
 

private NFATestHarness(
		SharedBuffer<Event> sharedBuffer,
		NFA<Event> nfa,
		NFAState nfaState,
		AfterMatchSkipStrategy afterMatchSkipStrategy,
		TimerService timerService) {
	this.sharedBuffer = sharedBuffer;
	this.nfa = nfa;
	this.nfaState = nfaState;
	this.afterMatchSkipStrategy = afterMatchSkipStrategy;
	this.timerService = timerService;
}
 

private NFATestHarness(
		SharedBuffer<Event> sharedBuffer,
		NFA<Event> nfa,
		NFAState nfaState,
		AfterMatchSkipStrategy afterMatchSkipStrategy,
		TimerService timerService) {
	this.sharedBuffer = sharedBuffer;
	this.nfa = nfa;
	this.nfaState = nfaState;
	this.afterMatchSkipStrategy = afterMatchSkipStrategy;
	this.timerService = timerService;
}
 

@Override
public void onProcessingTime(InternalTimer<KEY, VoidNamespace> timer) throws Exception {
	// 1) get the queue of pending elements for the key and the corresponding NFA,
	// 2) process the pending elements in process time order and custom comparator if exists
	//		by feeding them in the NFA
	// 3) update the stored state for the key, by only storing the new NFA and MapState iff they
	//		have state to be used later.

	// STEP 1
	PriorityQueue<Long> sortedTimestamps = getSortedTimestamps();
	NFAState nfa = getNFAState();

	// STEP 2
	while (!sortedTimestamps.isEmpty()) {
		long timestamp = sortedTimestamps.poll();
		advanceTime(nfa, timestamp);
		try (Stream<IN> elements = sort(elementQueueState.get(timestamp))) {
			elements.forEachOrdered(
				event -> {
					try {
						processEvent(nfa, event, timestamp);
					} catch (Exception e) {
						throw new RuntimeException(e);
					}
				}
			);
		}
		elementQueueState.remove(timestamp);
	}

	// STEP 3
	updateNFA(nfa);
}
 

private NFATestHarness(
		SharedBuffer<Event> sharedBuffer,
		NFA<Event> nfa,
		NFAState nfaState,
		AfterMatchSkipStrategy afterMatchSkipStrategy,
		TimerService timerService) {
	this.sharedBuffer = sharedBuffer;
	this.nfa = nfa;
	this.nfaState = nfaState;
	this.afterMatchSkipStrategy = afterMatchSkipStrategy;
	this.timerService = timerService;
}
 

@Override
public Matcher<TypeSerializerSchemaCompatibility<NFAState>> schemaCompatibilityMatcher(MigrationVersion version) {
	return TypeSerializerMatchers.isCompatibleAsIs();
}
 

@Override
public TypeSerializer<NFAState> createPriorSerializer() {
	return new NFAStateSerializer();
}
 

public NFATestHarnessBuilderBase withNFAState(NFAState nfaState) {
	this.nfaState = nfaState;
	return this;
}
 

@Override
public NFAState createTestData() {
	return new NFAState(Collections.emptyList());
}
 

@Override
public TypeSerializer<NFAState> createUpgradedSerializer() {
	return new NFAStateSerializer();
}
 

@Override
public Matcher<NFAState> testDataMatcher() {
	return is(new NFAState(Collections.emptyList()));
}
 

private void updateNFA(NFAState nfaState) throws IOException {
	if (nfaState.isStateChanged()) {
		nfaState.resetStateChanged();
		computationStates.update(nfaState);
	}
}
 

private NFAState getNFAState() throws IOException {
	NFAState nfaState = computationStates.value();
	return nfaState != null ? nfaState : nfa.createInitialNFAState();
}
 

@Override
public void onEventTime(InternalTimer<KEY, VoidNamespace> timer) throws Exception {

	// 1) get the queue of pending elements for the key and the corresponding NFA,
	// 2) process the pending elements in event time order and custom comparator if exists
	//		by feeding them in the NFA
	// 3) advance the time to the current watermark, so that expired patterns are discarded.
	// 4) update the stored state for the key, by only storing the new NFA and MapState iff they
	//		have state to be used later.
	// 5) update the last seen watermark.

	// STEP 1
	PriorityQueue<Long> sortedTimestamps = getSortedTimestamps();
	NFAState nfaState = getNFAState();

	// STEP 2
	while (!sortedTimestamps.isEmpty() && sortedTimestamps.peek() <= timerService.currentWatermark()) {
		long timestamp = sortedTimestamps.poll();
		advanceTime(nfaState, timestamp);
		try (Stream<IN> elements = sort(elementQueueState.get(timestamp))) {
			elements.forEachOrdered(
				event -> {
					try {
						processEvent(nfaState, event, timestamp);
					} catch (Exception e) {
						throw new RuntimeException(e);
					}
				}
			);
		}
		elementQueueState.remove(timestamp);
	}

	// STEP 3
	advanceTime(nfaState, timerService.currentWatermark());

	// STEP 4
	updateNFA(nfaState);

	if (!sortedTimestamps.isEmpty() || !partialMatches.isEmpty()) {
		saveRegisterWatermarkTimer();
	}

	// STEP 5
	updateLastSeenWatermark(timerService.currentWatermark());
}
 

public NFATestHarnessBuilderBase withNFAState(NFAState nfaState) {
	this.nfaState = nfaState;
	return this;
}
 

private void updateNFA(NFAState nfaState) throws IOException {
	if (nfaState.isStateChanged()) {
		nfaState.resetStateChanged();
		computationStates.update(nfaState);
	}
}
 

private NFAState getNFAState() throws IOException {
	NFAState nfaState = computationStates.value();
	return nfaState != null ? nfaState : nfa.createInitialNFAState();
}
 

@Override
public void onEventTime(InternalTimer<KEY, VoidNamespace> timer) throws Exception {

	// 1) get the queue of pending elements for the key and the corresponding NFA,
	// 2) process the pending elements in event time order and custom comparator if exists
	//		by feeding them in the NFA
	// 3) advance the time to the current watermark, so that expired patterns are discarded.
	// 4) update the stored state for the key, by only storing the new NFA and MapState iff they
	//		have state to be used later.
	// 5) update the last seen watermark.

	// STEP 1
	PriorityQueue<Long> sortedTimestamps = getSortedTimestamps();
	NFAState nfaState = getNFAState();

	// STEP 2
	while (!sortedTimestamps.isEmpty() && sortedTimestamps.peek() <= timerService.currentWatermark()) {
		long timestamp = sortedTimestamps.poll();
		advanceTime(nfaState, timestamp);
		try (Stream<IN> elements = sort(elementQueueState.get(timestamp))) {
			elements.forEachOrdered(
				event -> {
					try {
						processEvent(nfaState, event, timestamp);
					} catch (Exception e) {
						throw new RuntimeException(e);
					}
				}
			);
		}
		elementQueueState.remove(timestamp);
	}

	// STEP 3
	advanceTime(nfaState, timerService.currentWatermark());

	// STEP 4
	updateNFA(nfaState);

	if (!sortedTimestamps.isEmpty() || !partialMatches.isEmpty()) {
		saveRegisterWatermarkTimer();
	}

	// STEP 5
	updateLastSeenWatermark(timerService.currentWatermark());
}
 

public NFATestHarnessBuilderBase withNFAState(NFAState nfaState) {
	this.nfaState = nfaState;
	return this;
}
 

/**
 * Process the given event by giving it to the NFA and outputting the produced set of matched
 * event sequences.
 *
 * @param nfaState Our NFAState object
 * @param event The current event to be processed
 * @param timestamp The timestamp of the event
 */
private void processEvent(NFAState nfaState, IN event, long timestamp) throws Exception {
	try (SharedBufferAccessor<IN> sharedBufferAccessor = partialMatches.getAccessor()) {
		Collection<Map<String, List<IN>>> patterns =
			nfa.process(sharedBufferAccessor, nfaState, event, timestamp, afterMatchSkipStrategy, cepTimerService);
		processMatchedSequences(patterns, timestamp);
	}
}
 

/**
 * Process the given event by giving it to the NFA and outputting the produced set of matched
 * event sequences.
 *
 * @param nfaState Our NFAState object
 * @param event The current event to be processed
 * @param timestamp The timestamp of the event
 */
private void processEvent(NFAState nfaState, IN event, long timestamp) throws Exception {
	try (SharedBufferAccessor<IN> sharedBufferAccessor = partialMatches.getAccessor()) {
		Collection<Map<String, List<IN>>> patterns =
			nfa.process(sharedBufferAccessor, nfaState, event, timestamp, afterMatchSkipStrategy, cepTimerService);
		processMatchedSequences(patterns, timestamp);
	}
}