org.apache.flink.cep.Event Java Examples

@Test
public void testSharedBufferClearing() throws Exception {
	Pattern<Event, ?> pattern = Pattern.<Event>begin("start").followedBy("end");

	Event a = new Event(40, "a", 1.0);
	Event b = new Event(41, "b", 2.0);

	NFA<Event> nfa = compile(pattern, false);
	TestTimerService timerService = new TestTimerService();
	try (SharedBufferAccessor<Event> accessor = Mockito.spy(sharedBuffer.getAccessor())) {
		nfa.process(accessor, nfa.createInitialNFAState(), a, 1, AfterMatchSkipStrategy.noSkip(),
			timerService);
		nfa.process(accessor, nfa.createInitialNFAState(), b, 2, AfterMatchSkipStrategy.noSkip(),
			timerService);
		Mockito.verify(accessor, Mockito.never()).advanceTime(anyLong());
		nfa.advanceTime(accessor, nfa.createInitialNFAState(), 2);
		Mockito.verify(accessor, Mockito.times(1)).advanceTime(2);
	}
}
 

@Test
public void testEventTimestamp() throws Exception {
	final Event event = event().withId(1).build();
	final long timestamp = 3;

	final Pattern<Event, ?> pattern = Pattern.<Event>begin("start").where(new IterativeCondition<Event>() {
		@Override
		public boolean filter(Event value, Context<Event> ctx) throws Exception {
			return ctx.timestamp() == timestamp;
		}
	});

	final NFATestHarness testHarness = forPattern(pattern).build();

	final List<List<Event>> resultingPattern = testHarness.feedRecord(new StreamRecord<>(event, timestamp));

	comparePatterns(resultingPattern, Collections.singletonList(
		Collections.singletonList(event)
	));
}
 

@Test
public void testSharedBufferClearing() throws Exception {
	Pattern<Event, ?> pattern = Pattern.<Event>begin("start").followedBy("end");

	Event a = new Event(40, "a", 1.0);
	Event b = new Event(41, "b", 2.0);

	NFA<Event> nfa = compile(pattern, false);
	TestTimerService timerService = new TestTimerService();
	try (SharedBufferAccessor<Event> accessor = Mockito.spy(sharedBuffer.getAccessor())) {
		nfa.process(accessor, nfa.createInitialNFAState(), a, 1, AfterMatchSkipStrategy.noSkip(),
			timerService);
		nfa.process(accessor, nfa.createInitialNFAState(), b, 2, AfterMatchSkipStrategy.noSkip(),
			timerService);
		Mockito.verify(accessor, Mockito.never()).advanceTime(anyLong());
		nfa.advanceTime(accessor, nfa.createInitialNFAState(), 2);
		Mockito.verify(accessor, Mockito.times(1)).advanceTime(2);
	}
}
 

@Test
public void testPatternWithSubtyping() {
	Pattern<Event, ?> pattern = Pattern.<Event>begin("start").next("subevent").subtype(SubEvent.class).followedBy("end");

	Pattern<Event, ?> previous;
	Pattern<Event, ?> previous2;

	assertNotNull(previous = pattern.getPrevious());
	assertNotNull(previous2 = previous.getPrevious());
	assertNull(previous2.getPrevious());

	assertNotNull(previous.getCondition());
	assertTrue(previous.getCondition() instanceof SubtypeCondition);

	assertEquals(pattern.getName(), "end");
	assertEquals(previous.getName(), "subevent");
	assertEquals(previous2.getName(), "start");
}
 

@Override
public int compare(List<Event> o1, List<Event> o2) {
	int sizeComp = Integer.compare(o1.size(), o2.size());
	if (sizeComp == 0) {
		EventComparator comp = new EventComparator();
		for (int i = 0; i < o1.size(); i++) {
			int eventComp = comp.compare(o1.get(i), o2.get(i));
			if (eventComp != 0) {
				return eventComp;
			}
		}
		return 0;
	} else {
		return sizeComp;
	}
}
 

@Test
public void testSharedBufferClearing() throws Exception {
	Pattern<Event, ?> pattern = Pattern.<Event>begin("start").followedBy("end");

	Event a = new Event(40, "a", 1.0);
	Event b = new Event(41, "b", 2.0);

	NFA<Event> nfa = compile(pattern, false);
	TestTimerService timerService = new TestTimerService();
	try (SharedBufferAccessor<Event> accessor = Mockito.spy(sharedBuffer.getAccessor())) {
		nfa.process(accessor, nfa.createInitialNFAState(), a, 1, AfterMatchSkipStrategy.noSkip(),
			timerService);
		nfa.process(accessor, nfa.createInitialNFAState(), b, 2, AfterMatchSkipStrategy.noSkip(),
			timerService);
		Mockito.verify(accessor, Mockito.never()).advanceTime(anyLong());
		nfa.advanceTime(accessor, nfa.createInitialNFAState(), 2);
		Mockito.verify(accessor, Mockito.times(1)).advanceTime(2);
	}
}
 

@Test
public void testSharedBufferIsProperlyCleared() throws Exception {
	List<StreamRecord<Event>> inputEvents = new ArrayList<>();

	for (int i = 0; i < 4; i++) {
		inputEvents.add(new StreamRecord<>(new Event(1, "a", 1.0), i));
	}

	SkipPastLastStrategy matchSkipStrategy = AfterMatchSkipStrategy.skipPastLastEvent();
	Pattern<Event, ?> pattern = Pattern.<Event>begin("start", matchSkipStrategy)
		.where(new SimpleCondition<Event>() {
			private static final long serialVersionUID = 5726188262756267490L;

			@Override
			public boolean filter(Event value) throws Exception {
				return true;
			}
		}).times(2);

	SharedBuffer<Event> sharedBuffer = TestSharedBuffer.createTestBuffer(Event.createTypeSerializer());
	NFATestHarness nfaTestHarness = NFATestHarness.forPattern(pattern).withSharedBuffer(sharedBuffer).build();

	nfaTestHarness.feedRecords(inputEvents);

	assertThat(sharedBuffer.isEmpty(), Matchers.is(true));
}
 

@Override
public int compare(List<Event> o1, List<Event> o2) {
	int sizeComp = Integer.compare(o1.size(), o2.size());
	if (sizeComp == 0) {
		EventComparator comp = new EventComparator();
		for (int i = 0; i < o1.size(); i++) {
			int eventComp = comp.compare(o1.get(i), o2.get(i));
			if (eventComp != 0) {
				return eventComp;
			}
		}
		return 0;
	} else {
		return sizeComp;
	}
}
 

@Test
public void testIterativeWithBranchingPatternCombinations() throws Exception {
	List<List<Event>> actual = testIterativeWithBranchingPattern(false);

	compareMaps(actual,
		Lists.<List<Event>>newArrayList(
			Lists.newArrayList(startEvent1, endEvent, middleEvent1, middleEvent2, middleEvent4),
			Lists.newArrayList(startEvent1, endEvent, middleEvent2, middleEvent1),
			Lists.newArrayList(startEvent1, endEvent, middleEvent3, middleEvent1),
			Lists.newArrayList(startEvent2, endEvent, middleEvent3, middleEvent4),
			Lists.newArrayList(startEvent1, endEvent, middleEvent4, middleEvent1),
			Lists.newArrayList(startEvent1, endEvent, middleEvent1),
			Lists.newArrayList(startEvent2, endEvent, middleEvent3)
		)
	);
}
 

@Test
public void testCurrentProcessingTimeInEventTime() throws Exception {

	try (
		OneInputStreamOperatorTestHarness<Event, String> harness = getCepTestHarness(
			createCepOperator(
				extractCurrentProcessingTimeAndNames(1),
				new NFAForwardingFactory(),
				EVENT_TIME))) {
		harness.open();

		harness.setProcessingTime(10);
		harness.processElement(event().withName("A").withTimestamp(5).asStreamRecord());
		harness.setProcessingTime(100);
		harness.processWatermark(6);

		assertOutput(harness.getOutput())
			.nextElementEquals("100:A")
			.watermarkEquals(6)
			.hasNoMoreElements();
	}
}
 

@Test
public void testCurrentProcessingTimeInProcessingTime() throws Exception {

	try (
		OneInputStreamOperatorTestHarness<Event, String> harness = getCepTestHarness(
			createCepOperator(
				extractCurrentProcessingTimeAndNames(1),
				new NFAForwardingFactory(),
				PROCESSING_TIME))) {
		harness.open();

		harness.setProcessingTime(15);
		harness.processElement(event().withName("A").asStreamRecord());
		harness.setProcessingTime(35);
		harness.processElement(event().withName("B").asStreamRecord());

		assertOutput(harness.getOutput())
			.nextElementEquals("15:A")
			.nextElementEquals("35:B")
			.hasNoMoreElements();
	}
}
 

@Test
public void testSharedBufferIsProperlyCleared() throws Exception {
	List<StreamRecord<Event>> inputEvents = new ArrayList<>();

	for (int i = 0; i < 4; i++) {
		inputEvents.add(new StreamRecord<>(new Event(1, "a", 1.0), i));
	}

	SkipPastLastStrategy matchSkipStrategy = AfterMatchSkipStrategy.skipPastLastEvent();
	Pattern<Event, ?> pattern = Pattern.<Event>begin("start", matchSkipStrategy)
		.where(new SimpleCondition<Event>() {
			private static final long serialVersionUID = 5726188262756267490L;

			@Override
			public boolean filter(Event value) throws Exception {
				return true;
			}
		}).times(2);

	SharedBuffer<Event> sharedBuffer = TestSharedBuffer.createTestBuffer(Event.createTypeSerializer());
	NFATestHarness nfaTestHarness = NFATestHarness.forPattern(pattern).withSharedBuffer(sharedBuffer).build();

	nfaTestHarness.feedRecords(inputEvents);

	assertThat(sharedBuffer.isEmpty(), Matchers.is(true));
}
 

@Test
public void testPatternWithSubtyping() {
	Pattern<Event, ?> pattern = Pattern.<Event>begin("start").next("subevent").subtype(SubEvent.class).followedBy("end");

	Pattern<Event, ?> previous;
	Pattern<Event, ?> previous2;

	assertNotNull(previous = pattern.getPrevious());
	assertNotNull(previous2 = previous.getPrevious());
	assertNull(previous2.getPrevious());

	assertNotNull(previous.getCondition());
	assertTrue(previous.getCondition() instanceof SubtypeCondition);

	assertEquals(pattern.getName(), "end");
	assertEquals(previous.getName(), "subevent");
	assertEquals(previous2.getName(), "start");
}
 

@Test
public void testTimeoutWindowPruning() throws Exception {
	List<StreamRecord<Event>> streamEvents = new ArrayList<>();

	streamEvents.add(new StreamRecord<>(new Event(1, "start", 1.0), 1L));
	streamEvents.add(new StreamRecord<>(new Event(2, "bar", 2.0), 2L));
	streamEvents.add(new StreamRecord<>(new Event(3, "start", 3.0), 3L));
	streamEvents.add(new StreamRecord<>(new Event(4, "end", 4.0), 4L));

	List<Map<String, List<Event>>> expectedPatterns = new ArrayList<>();

	Map<String, List<Event>> secondPattern = new HashMap<>();
	secondPattern.put("start", Collections.singletonList(new Event(3, "start", 3.0)));
	secondPattern.put("end", Collections.singletonList(new Event(4, "end", 4.0)));

	expectedPatterns.add(secondPattern);

	NFA<Event> nfa = createStartEndNFA();
	NFATestHarness nfaTestHarness = NFATestHarness.forNFA(nfa).build();

	Collection<Map<String, List<Event>>> actualPatterns = nfaTestHarness.consumeRecords(streamEvents);

	assertEquals(expectedPatterns, actualPatterns);
}
 

private KeyedOneInputStreamOperatorTestHarness<Integer, Event, Map<String, List<Event>>> getTestHarness(
		int maxParallelism,
		int taskParallelism,
		int subtaskIdx) throws Exception {

	KeySelector<Event, Integer> keySelector = new TestKeySelector();
	KeyedOneInputStreamOperatorTestHarness<Integer, Event, Map<String, List<Event>>> harness =
			new KeyedOneInputStreamOperatorTestHarness<>(
					getKeyedCepOpearator(
							false,
							new NFAFactory()),
					keySelector,
					BasicTypeInfo.INT_TYPE_INFO,
					maxParallelism,
					taskParallelism,
					subtaskIdx);
	harness.setStateBackend(new RocksDBStateBackend(new MemoryStateBackend()));
	return harness;
}
 

@Test
public void testEventTimestamp() throws Exception {
	final Event event = event().withId(1).build();
	final long timestamp = 3;

	final Pattern<Event, ?> pattern = Pattern.<Event>begin("start").where(new IterativeCondition<Event>() {
		@Override
		public boolean filter(Event value, Context<Event> ctx) throws Exception {
			return ctx.timestamp() == timestamp;
		}
	});

	final NFATestHarness testHarness = forPattern(pattern).build();

	final List<List<Event>> resultingPattern = testHarness.feedRecord(new StreamRecord<>(event, timestamp));

	compareMaps(resultingPattern, Collections.singletonList(
		Collections.singletonList(event)
	));
}
 

@Test
public void testCurrentProcessingTimeForTimedOutInEventTime() throws Exception {

	OutputTag<String> sideOutputTag = new OutputTag<String>("timedOut") {};

	try (
		OneInputStreamOperatorTestHarness<Event, String> harness = getCepTestHarness(
			createCepOperator(
				extractCurrentProcessingTimeAndNames(2, sideOutputTag),
				new NFATimingOutFactory(),
				EVENT_TIME))) {
		harness.open();

		// events out of order to test if internal sorting does not mess up the timestamps
		harness.processElement(event().withName("A").withTimestamp(5).asStreamRecord());
		harness.processElement(event().withName("B").withTimestamp(20).asStreamRecord());
		harness.processElement(event().withName("C").withTimestamp(3).asStreamRecord());

		harness.setProcessingTime(100);
		harness.processWatermark(22);

		assertOutput(harness.getOutput())
			.nextElementEquals("100:C:A")
			.watermarkEquals(22)
			.hasNoMoreElements();

		assertOutput(harness.getSideOutput(sideOutputTag))
			.nextElementEquals("100:A")
			.hasNoMoreElements();
	}
}
 

@Test
public void testSkipToLastNonExistentPositionWithoutException() throws Exception {
	List<List<Event>> resultingPatterns = MissedSkipTo.compute(AfterMatchSkipStrategy.skipToFirst("b"));

	comparePatterns(resultingPatterns, Collections.singletonList(
		Lists.newArrayList(MissedSkipTo.a, MissedSkipTo.c)
	));
}
 

String extractResult(Map<String, List<Event>> match, Context ctx) {
	StringBuilder stringBuilder = new StringBuilder(contextAccessor.apply(ctx));
	for (int i = 1; i <= stateCount; i++) {
		List<Event> events = match.get("" + i);
		if (events != null) {
			stringBuilder.append(":").append(events.get(0).getName());
		}
	}
	return stringBuilder.toString();
}
 

@Test
public void testNotNextAfterOneOrMoreSkipTillAny() throws Exception {
	final List<List<Event>> matches = testNotNextAfterOneOrMore(true);
	comparePatterns(matches, Lists.<List<Event>>newArrayList(
		Lists.newArrayList(NotFollowByData.A_1, NotFollowByData.B_2, NotFollowByData.D_1)
	));
}
 

private CepOperator<Event, Integer, Map<String, List<Event>>> getKeyedCepOperatorWithComparator(
	boolean isProcessingTime) {

	return CepOperatorTestUtilities.getKeyedCepOpearator(isProcessingTime, new NFAFactory(), new org.apache.flink.cep.EventComparator<Event>() {
		@Override
		public int compare(Event o1, Event o2) {
			return Double.compare(o1.getPrice(), o2.getPrice());
		}
	});
}
 

public Collection<Map<String, List<Event>>> consumeRecord(StreamRecord<Event> inputEvent) throws Exception {
	try (SharedBufferAccessor<Event> sharedBufferAccessor = sharedBuffer.getAccessor()) {
		nfa.advanceTime(sharedBufferAccessor, nfaState, inputEvent.getTimestamp());
		return nfa.process(
			sharedBufferAccessor,
			nfaState,
			inputEvent.getValue(),
			inputEvent.getTimestamp(),
			afterMatchSkipStrategy,
			timerService);
	}
}
 

@Override
public NFATestHarness build() {
	final NFA<Event> nfa = NFAUtils.compile(pattern, timeoutHandling);
	return new NFATestHarness(
		sharedBuffer,
		nfa,
		nfa.createInitialNFAState(),
		afterMatchSkipStrategy,
		timerService);
}
 

@Test
public void testNotFollowedByBeforeZeroOrMoreEagerSkipTillNext() throws Exception {
	final List<List<Event>> matches = testNotFollowedByBeforeZeroOrMore(true, false);
	comparePatterns(matches, Lists.<List<Event>>newArrayList(
		Lists.newArrayList(NotFollowByData.A_1, NotFollowByData.B_1, NotFollowByData.B_4, NotFollowByData.B_5, NotFollowByData.B_6, NotFollowByData.D_1),
		Lists.newArrayList(NotFollowByData.A_1, NotFollowByData.B_1, NotFollowByData.B_4, NotFollowByData.B_5, NotFollowByData.D_1),
		Lists.newArrayList(NotFollowByData.A_1, NotFollowByData.B_1, NotFollowByData.B_4, NotFollowByData.D_1),
		Lists.newArrayList(NotFollowByData.A_1, NotFollowByData.B_1, NotFollowByData.D_1)
	));
}
 

@Test
public void testNotFollowedByBeforeZeroOrMoreCombinationsSkipTillAny() throws Exception {
	final List<List<Event>> matches = testNotFollowedByBeforeZeroOrMore(false, true);
	compareMaps(matches, Lists.<List<Event>>newArrayList(
		Lists.newArrayList(NotFollowByData.A_1, NotFollowByData.B_1, NotFollowByData.B_4, NotFollowByData.B_5, NotFollowByData.B_6, NotFollowByData.D_1),
		Lists.newArrayList(NotFollowByData.A_1, NotFollowByData.B_1, NotFollowByData.B_4, NotFollowByData.B_5, NotFollowByData.D_1),
		Lists.newArrayList(NotFollowByData.A_1, NotFollowByData.B_1, NotFollowByData.B_4, NotFollowByData.B_6, NotFollowByData.D_1),
		Lists.newArrayList(NotFollowByData.A_1, NotFollowByData.B_1, NotFollowByData.B_4, NotFollowByData.D_1),
		Lists.newArrayList(NotFollowByData.A_1, NotFollowByData.B_1, NotFollowByData.D_1),
		Lists.newArrayList(NotFollowByData.A_1, NotFollowByData.B_1, NotFollowByData.B_5, NotFollowByData.B_6, NotFollowByData.D_1),
		Lists.newArrayList(NotFollowByData.A_1, NotFollowByData.B_1, NotFollowByData.B_5, NotFollowByData.D_1),
		Lists.newArrayList(NotFollowByData.A_1, NotFollowByData.B_1, NotFollowByData.B_6, NotFollowByData.D_1)
	));
}
 

@Test
public void testCurrentProcessingTimeForTimedOutInProcessingTime() throws Exception {

	OutputTag<String> sideOutputTag = new OutputTag<String>("timedOut") {};

	try (
		OneInputStreamOperatorTestHarness<Event, String> harness = getCepTestHarness(
			createCepOperator(
				extractCurrentProcessingTimeAndNames(2, sideOutputTag),
				new NFATimingOutFactory(),
				PROCESSING_TIME))) {
		harness.open();

		harness.setProcessingTime(3);
		harness.processElement(event().withName("A").asStreamRecord());
		harness.setProcessingTime(5);
		harness.processElement(event().withName("B").asStreamRecord());
		harness.setProcessingTime(20);
		harness.processElement(event().withName("C").asStreamRecord());

		assertOutput(harness.getOutput())
			.nextElementEquals("5:A:B")
			.hasNoMoreElements();

		// right now we time out only on next event in processing time, therefore the 20
		assertOutput(harness.getSideOutput(sideOutputTag))
			.nextElementEquals("20:B")
			.hasNoMoreElements();
	}
}
 

@Test
public void testClearingSharedBufferWithMultipleEdgesBetweenEntries() throws Exception {
	SharedBuffer<Event> sharedBuffer = TestSharedBuffer.createTestBuffer(Event.createTypeSerializer());
	int numberEvents = 8;
	Event[] events = new Event[numberEvents];
	EventId[] eventIds = new EventId[numberEvents];
	final long timestamp = 1L;

	for (int i = 0; i < numberEvents; i++) {
		events[i] = new Event(i + 1, "e" + (i + 1), i);
		eventIds[i] = sharedBuffer.registerEvent(events[i], timestamp);
	}

	try (SharedBufferAccessor<Event> sharedBufferAccessor = sharedBuffer.getAccessor()) {
		NodeId start = sharedBufferAccessor.put("start", eventIds[1], null, DeweyNumber.fromString("1"));
		NodeId b0 = sharedBufferAccessor.put("branching", eventIds[2], start, DeweyNumber.fromString("1.0"));
		NodeId b1 = sharedBufferAccessor.put("branching", eventIds[3], start, DeweyNumber.fromString("1.1"));
		NodeId b00 = sharedBufferAccessor.put("branching", eventIds[3], b0, DeweyNumber.fromString("1.0.0"));
		sharedBufferAccessor.put("branching", eventIds[4], b00, DeweyNumber.fromString("1.0.0.0"));
		NodeId b10 = sharedBufferAccessor.put("branching", eventIds[4], b1, DeweyNumber.fromString("1.1.0"));

		//simulate IGNORE (next event can point to events[2])
		sharedBufferAccessor.lockNode(b0);

		sharedBufferAccessor.releaseNode(b10);

		for (EventId eventId : eventIds) {
			sharedBufferAccessor.releaseEvent(eventId);
		}
	}

	//There should be still events[1] and events[2] in the buffer
	assertFalse(sharedBuffer.isEmpty());
}
 

@Test
public void testNoSkipWithFollowedByAny() throws Exception {
	List<List<Event>> resultingPatterns = TwoVariablesFollowedByAny.compute(AfterMatchSkipStrategy.noSkip());

	compareMaps(resultingPatterns, Lists.newArrayList(
		Lists.newArrayList(TwoVariablesFollowedByAny.a1, TwoVariablesFollowedByAny.b1),
		Lists.newArrayList(TwoVariablesFollowedByAny.a1, TwoVariablesFollowedByAny.b2),
		Lists.newArrayList(TwoVariablesFollowedByAny.a2, TwoVariablesFollowedByAny.b2)
	));
}
 

@Test
public void testMultipleWindowTimeWithZeroLength() {
	Pattern<Event, ?> pattern = Pattern.<Event>begin("start").followedBy("middle").within(Time.seconds(10))
		.followedBy("then").within(Time.seconds(0)).followedBy("end");

	NFACompiler.NFAFactoryCompiler<Event> factory = new NFACompiler.NFAFactoryCompiler<>(pattern);
	factory.compileFactory();
	assertEquals(0, factory.getWindowTime());
}
 

/**
 * Manually run this to write binary snapshot data.
 */
@Ignore
@Test
public void writeSinglePatternAfterMigrationSnapshot() throws Exception {

	KeySelector<Event, Integer> keySelector = new KeySelector<Event, Integer>() {
		private static final long serialVersionUID = -4873366487571254798L;

		@Override
		public Integer getKey(Event value) throws Exception {
			return value.getId();
		}
	};

	final Event startEvent1 = new Event(42, "start", 1.0);

	OneInputStreamOperatorTestHarness<Event, Map<String, List<Event>>> harness =
			new KeyedOneInputStreamOperatorTestHarness<>(
					getKeyedCepOpearator(false, new SinglePatternNFAFactory()),
					keySelector,
					BasicTypeInfo.INT_TYPE_INFO);

	try {
		harness.setup();
		harness.open();
		harness.processWatermark(new Watermark(5));

		// do snapshot and save to file
		OperatorSubtaskState snapshot = harness.snapshot(0L, 0L);
		OperatorSnapshotUtil.writeStateHandle(snapshot,
			"src/test/resources/cep-migration-single-pattern-afterwards-flink" + flinkGenerateSavepointVersion + "-snapshot");
	} finally {
		harness.close();
	}
}