Java Code Examples for org.apache.flink.streaming.util.KeyedOneInputStreamOperatorTestHarness#setup()

The following examples show how to use org.apache.flink.streaming.util.KeyedOneInputStreamOperatorTestHarness#setup() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: CEPOperatorTest.java    From Flink-CEPplus with Apache License 2.0 4 votes vote down vote up 
/**
 * Tests that the internal time of a CEP operator advances only given watermarks. See FLINK-5033
 */
@Test
public void testKeyedAdvancingTimeWithoutElements() throws Exception {
	final Event startEvent = new Event(42, "start", 1.0);
	final long watermarkTimestamp1 = 5L;
	final long watermarkTimestamp2 = 13L;

	final Map<String, List<Event>> expectedSequence = new HashMap<>(2);
	expectedSequence.put("start", Collections.<Event>singletonList(startEvent));

	final OutputTag<Tuple2<Map<String, List<Event>>, Long>> timedOut =
		new OutputTag<Tuple2<Map<String, List<Event>>, Long>>("timedOut") {};
	final KeyedOneInputStreamOperatorTestHarness<Integer, Event, Map<String, List<Event>>> harness =
		new KeyedOneInputStreamOperatorTestHarness<>(
			new CepOperator<>(
				Event.createTypeSerializer(),
				false,
				new NFAFactory(true),
				null,
				null,
				new TimedOutProcessFunction(timedOut),
				null), new KeySelector<Event, Integer>() {
			private static final long serialVersionUID = 7219185117566268366L;

			@Override
			public Integer getKey(Event value) throws Exception {
				return value.getId();
			}
		}, BasicTypeInfo.INT_TYPE_INFO);

	try {
		String rocksDbPath = tempFolder.newFolder().getAbsolutePath();
		RocksDBStateBackend rocksDBStateBackend = new RocksDBStateBackend(new MemoryStateBackend());
		rocksDBStateBackend.setDbStoragePath(rocksDbPath);

		harness.setStateBackend(rocksDBStateBackend);
		harness.setup(
			new KryoSerializer<>(
				(Class<Map<String, List<Event>>>) (Object) Map.class,
				new ExecutionConfig()));
		harness.open();

		harness.processElement(new StreamRecord<>(startEvent, 3L));
		harness.processWatermark(new Watermark(watermarkTimestamp1));
		harness.processWatermark(new Watermark(watermarkTimestamp2));

		Queue<Object> result = harness.getOutput();
		Queue<StreamRecord<Tuple2<Map<String, List<Event>>, Long>>> sideOutput = harness.getSideOutput(timedOut);

		assertEquals(2L, result.size());
		assertEquals(1L, sideOutput.size());

		Object watermark1 = result.poll();

		assertTrue(watermark1 instanceof Watermark);

		assertEquals(watermarkTimestamp1, ((Watermark) watermark1).getTimestamp());

		Tuple2<Map<String, List<Event>>, Long> leftResult = sideOutput.poll().getValue();

		assertEquals(watermarkTimestamp2, (long) leftResult.f1);
		assertEquals(expectedSequence, leftResult.f0);

		Object watermark2 = result.poll();

		assertTrue(watermark2 instanceof Watermark);

		assertEquals(watermarkTimestamp2, ((Watermark) watermark2).getTimestamp());
	} finally {
		harness.close();
	}
}
Example 2
Source File: AbstractStreamOperatorTest.java    From Flink-CEPplus with Apache License 2.0 4 votes vote down vote up 
/**
 * Verify that a low-level timer is set for processing-time timers in case of restore.
 */
@Test
public void testEnsureProcessingTimeTimerRegisteredOnRestore() throws Exception {
	TestOperator testOperator = new TestOperator();

	KeyedOneInputStreamOperatorTestHarness<Integer, Tuple2<Integer, String>, String> testHarness =
			new KeyedOneInputStreamOperatorTestHarness<>(testOperator, new TestKeySelector(), BasicTypeInfo.INT_TYPE_INFO);

	testHarness.open();

	testHarness.setProcessingTime(0L);

	testHarness.processElement(new Tuple2<>(1, "SET_PROC_TIME_TIMER:20"), 0);

	testHarness.processElement(new Tuple2<>(0, "SET_STATE:HELLO"), 0);
	testHarness.processElement(new Tuple2<>(1, "SET_STATE:CIAO"), 0);

	testHarness.processElement(new Tuple2<>(0, "SET_PROC_TIME_TIMER:10"), 0);

	OperatorSubtaskState snapshot = testHarness.snapshot(0, 0);

	TestOperator testOperator1 = new TestOperator();

	KeyedOneInputStreamOperatorTestHarness<Integer, Tuple2<Integer, String>, String> testHarness1 =
			new KeyedOneInputStreamOperatorTestHarness<>(
					testOperator1,
					new TestKeySelector(),
					BasicTypeInfo.INT_TYPE_INFO);

	testHarness1.setProcessingTime(0L);

	testHarness1.setup();
	testHarness1.initializeState(snapshot);
	testHarness1.open();

	testHarness1.setProcessingTime(10L);

	assertThat(
			extractResult(testHarness1),
			contains("ON_PROC_TIME:HELLO"));

	testHarness1.setProcessingTime(20L);

	assertThat(
			extractResult(testHarness1),
			contains("ON_PROC_TIME:CIAO"));
}
Example 3
Source File: CEPOperatorTest.java    From flink with Apache License 2.0 4 votes vote down vote up 
/**
 * Tests that the internal time of a CEP operator advances only given watermarks. See FLINK-5033
 */
@Test
public void testKeyedAdvancingTimeWithoutElements() throws Exception {
	final Event startEvent = new Event(42, "start", 1.0);
	final long watermarkTimestamp1 = 5L;
	final long watermarkTimestamp2 = 13L;

	final Map<String, List<Event>> expectedSequence = new HashMap<>(2);
	expectedSequence.put("start", Collections.<Event>singletonList(startEvent));

	final OutputTag<Tuple2<Map<String, List<Event>>, Long>> timedOut =
		new OutputTag<Tuple2<Map<String, List<Event>>, Long>>("timedOut") {};
	final KeyedOneInputStreamOperatorTestHarness<Integer, Event, Map<String, List<Event>>> harness =
		new KeyedOneInputStreamOperatorTestHarness<>(
			new CepOperator<>(
				Event.createTypeSerializer(),
				false,
				new NFAFactory(true),
				null,
				null,
				new TimedOutProcessFunction(timedOut),
				null), new KeySelector<Event, Integer>() {
			private static final long serialVersionUID = 7219185117566268366L;

			@Override
			public Integer getKey(Event value) throws Exception {
				return value.getId();
			}
		}, BasicTypeInfo.INT_TYPE_INFO);

	try {
		String rocksDbPath = tempFolder.newFolder().getAbsolutePath();
		RocksDBStateBackend rocksDBStateBackend = new RocksDBStateBackend(new MemoryStateBackend());
		rocksDBStateBackend.setDbStoragePath(rocksDbPath);

		harness.setStateBackend(rocksDBStateBackend);
		harness.setup(
			new KryoSerializer<>(
				(Class<Map<String, List<Event>>>) (Object) Map.class,
				new ExecutionConfig()));
		harness.open();

		harness.processElement(new StreamRecord<>(startEvent, 3L));
		harness.processWatermark(new Watermark(watermarkTimestamp1));
		harness.processWatermark(new Watermark(watermarkTimestamp2));

		Queue<Object> result = harness.getOutput();
		Queue<StreamRecord<Tuple2<Map<String, List<Event>>, Long>>> sideOutput = harness.getSideOutput(timedOut);

		assertEquals(2L, result.size());
		assertEquals(1L, sideOutput.size());

		Object watermark1 = result.poll();

		assertTrue(watermark1 instanceof Watermark);

		assertEquals(watermarkTimestamp1, ((Watermark) watermark1).getTimestamp());

		Tuple2<Map<String, List<Event>>, Long> leftResult = sideOutput.poll().getValue();

		assertEquals(watermarkTimestamp2, (long) leftResult.f1);
		assertEquals(expectedSequence, leftResult.f0);

		Object watermark2 = result.poll();

		assertTrue(watermark2 instanceof Watermark);

		assertEquals(watermarkTimestamp2, ((Watermark) watermark2).getTimestamp());
	} finally {
		harness.close();
	}
}
Example 4
Source File: AbstractStreamOperatorTest.java    From flink with Apache License 2.0 4 votes vote down vote up 
/**
 * Verify that a low-level timer is set for processing-time timers in case of restore.
 */
@Test
public void testEnsureProcessingTimeTimerRegisteredOnRestore() throws Exception {
	TestOperator testOperator = new TestOperator();

	KeyedOneInputStreamOperatorTestHarness<Integer, Tuple2<Integer, String>, String> testHarness =
			new KeyedOneInputStreamOperatorTestHarness<>(testOperator, new TestKeySelector(), BasicTypeInfo.INT_TYPE_INFO);

	testHarness.open();

	testHarness.setProcessingTime(0L);

	testHarness.processElement(new Tuple2<>(1, "SET_PROC_TIME_TIMER:20"), 0);

	testHarness.processElement(new Tuple2<>(0, "SET_STATE:HELLO"), 0);
	testHarness.processElement(new Tuple2<>(1, "SET_STATE:CIAO"), 0);

	testHarness.processElement(new Tuple2<>(0, "SET_PROC_TIME_TIMER:10"), 0);

	OperatorSubtaskState snapshot = testHarness.snapshot(0, 0);

	TestOperator testOperator1 = new TestOperator();

	KeyedOneInputStreamOperatorTestHarness<Integer, Tuple2<Integer, String>, String> testHarness1 =
			new KeyedOneInputStreamOperatorTestHarness<>(
					testOperator1,
					new TestKeySelector(),
					BasicTypeInfo.INT_TYPE_INFO);

	testHarness1.setProcessingTime(0L);

	testHarness1.setup();
	testHarness1.initializeState(snapshot);
	testHarness1.open();

	testHarness1.setProcessingTime(10L);

	assertThat(
			extractResult(testHarness1),
			contains("ON_PROC_TIME:HELLO"));

	testHarness1.setProcessingTime(20L);

	assertThat(
			extractResult(testHarness1),
			contains("ON_PROC_TIME:CIAO"));
}
Example 5
Source File: DedupingOperatorTest.java    From beam with Apache License 2.0 4 votes vote down vote up 
@Test
public void testDeduping() throws Exception {

  KeyedOneInputStreamOperatorTestHarness<
          ByteBuffer, WindowedValue<ValueWithRecordId<String>>, WindowedValue<String>>
      harness = getDebupingHarness();

  harness.open();

  String key1 = "key1";
  String key2 = "key2";

  harness.processElement(
      new StreamRecord<>(
          WindowedValue.valueInGlobalWindow(
              new ValueWithRecordId<>(key1, key1.getBytes(StandardCharsets.UTF_8)))));

  harness.processElement(
      new StreamRecord<>(
          WindowedValue.valueInGlobalWindow(
              new ValueWithRecordId<>(key2, key2.getBytes(StandardCharsets.UTF_8)))));

  harness.processElement(
      new StreamRecord<>(
          WindowedValue.valueInGlobalWindow(
              new ValueWithRecordId<>(key1, key1.getBytes(StandardCharsets.UTF_8)))));

  assertThat(
      stripStreamRecordFromWindowedValue(harness.getOutput()),
      contains(WindowedValue.valueInGlobalWindow(key1), WindowedValue.valueInGlobalWindow(key2)));

  OperatorSubtaskState snapshot = harness.snapshot(0L, 0L);

  harness.close();

  harness = getDebupingHarness();
  harness.setup();
  harness.initializeState(snapshot);
  harness.open();

  String key3 = "key3";

  harness.processElement(
      new StreamRecord<>(
          WindowedValue.valueInGlobalWindow(
              new ValueWithRecordId<>(key2, key2.getBytes(StandardCharsets.UTF_8)))));

  harness.processElement(
      new StreamRecord<>(
          WindowedValue.valueInGlobalWindow(
              new ValueWithRecordId<>(key3, key3.getBytes(StandardCharsets.UTF_8)))));

  assertThat(
      stripStreamRecordFromWindowedValue(harness.getOutput()),
      contains(WindowedValue.valueInGlobalWindow(key3)));

  harness.close();
}
Example 6
Source File: CEPOperatorTest.java    From flink with Apache License 2.0 4 votes vote down vote up 
/**
 * Tests that the internal time of a CEP operator advances only given watermarks. See FLINK-5033
 */
@Test
public void testKeyedAdvancingTimeWithoutElements() throws Exception {
	final Event startEvent = new Event(42, "start", 1.0);
	final long watermarkTimestamp1 = 5L;
	final long watermarkTimestamp2 = 13L;

	final Map<String, List<Event>> expectedSequence = new HashMap<>(2);
	expectedSequence.put("start", Collections.<Event>singletonList(startEvent));

	final OutputTag<Tuple2<Map<String, List<Event>>, Long>> timedOut =
		new OutputTag<Tuple2<Map<String, List<Event>>, Long>>("timedOut") {};
	final KeyedOneInputStreamOperatorTestHarness<Integer, Event, Map<String, List<Event>>> harness =
		new KeyedOneInputStreamOperatorTestHarness<>(
			new CepOperator<>(
				Event.createTypeSerializer(),
				false,
				new NFAFactory(true),
				null,
				null,
				new TimedOutProcessFunction(timedOut),
				null), new KeySelector<Event, Integer>() {
			private static final long serialVersionUID = 7219185117566268366L;

			@Override
			public Integer getKey(Event value) throws Exception {
				return value.getId();
			}
		}, BasicTypeInfo.INT_TYPE_INFO);

	try {
		String rocksDbPath = tempFolder.newFolder().getAbsolutePath();
		RocksDBStateBackend rocksDBStateBackend = new RocksDBStateBackend(new MemoryStateBackend());
		rocksDBStateBackend.setDbStoragePath(rocksDbPath);

		harness.setStateBackend(rocksDBStateBackend);
		harness.setup(
			new KryoSerializer<>(
				(Class<Map<String, List<Event>>>) (Object) Map.class,
				new ExecutionConfig()));
		harness.open();

		harness.processElement(new StreamRecord<>(startEvent, 3L));
		harness.processWatermark(new Watermark(watermarkTimestamp1));
		harness.processWatermark(new Watermark(watermarkTimestamp2));

		Queue<Object> result = harness.getOutput();
		Queue<StreamRecord<Tuple2<Map<String, List<Event>>, Long>>> sideOutput = harness.getSideOutput(timedOut);

		assertEquals(2L, result.size());
		assertEquals(1L, sideOutput.size());

		Object watermark1 = result.poll();

		assertTrue(watermark1 instanceof Watermark);

		assertEquals(watermarkTimestamp1, ((Watermark) watermark1).getTimestamp());

		Tuple2<Map<String, List<Event>>, Long> leftResult = sideOutput.poll().getValue();

		assertEquals(watermarkTimestamp2, (long) leftResult.f1);
		assertEquals(expectedSequence, leftResult.f0);

		Object watermark2 = result.poll();

		assertTrue(watermark2 instanceof Watermark);

		assertEquals(watermarkTimestamp2, ((Watermark) watermark2).getTimestamp());
	} finally {
		harness.close();
	}
}