Java Code Examples for org.apache.flink.streaming.util.OneInputStreamOperatorTestHarness#setStateBackend()

The following examples show how to use org.apache.flink.streaming.util.OneInputStreamOperatorTestHarness#setStateBackend() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: PojoSerializerUpgradeTest.java    From Flink-CEPplus with Apache License 2.0 4 votes vote down vote up 
private OperatorSubtaskState runOperator(
		Configuration taskConfiguration,
		ExecutionConfig executionConfig,
		OneInputStreamOperator<Long, Long> operator,
		KeySelector<Long, Long> keySelector,
		boolean isKeyedState,
		StateBackend stateBackend,
		ClassLoader classLoader,
		OperatorSubtaskState operatorSubtaskState,
		Iterable<Long> input) throws Exception {

	try (final MockEnvironment environment =
			new MockEnvironmentBuilder()
				.setTaskName("test task")
				.setMemorySize(32 * 1024)
				.setInputSplitProvider(new MockInputSplitProvider())
				.setBufferSize(256)
				.setTaskConfiguration(taskConfiguration)
				.setExecutionConfig(executionConfig)
				.setMaxParallelism(16)
				.setUserCodeClassLoader(classLoader)
				.build()) {

		OneInputStreamOperatorTestHarness<Long, Long> harness = null;
		try {
			if (isKeyedState) {
				harness = new KeyedOneInputStreamOperatorTestHarness<>(
					operator,
					keySelector,
					BasicTypeInfo.LONG_TYPE_INFO,
					environment);
			} else {
				harness = new OneInputStreamOperatorTestHarness<>(operator, LongSerializer.INSTANCE, environment);
			}

			harness.setStateBackend(stateBackend);

			harness.setup();
			harness.initializeState(operatorSubtaskState);
			harness.open();

			long timestamp = 0L;

			for (Long value : input) {
				harness.processElement(value, timestamp++);
			}

			long checkpointId = 1L;
			long checkpointTimestamp = timestamp + 1L;

			return harness.snapshot(checkpointId, checkpointTimestamp);
		} finally {
			IOUtils.closeQuietly(harness);
		}
	}
}
Example 2
Source File: CEPOperatorTest.java    From Flink-CEPplus with Apache License 2.0 4 votes vote down vote up 
@Test
public void testKeyedCEPOperatorNFAUpdateTimesWithRocksDB() throws Exception {

	String rocksDbPath = tempFolder.newFolder().getAbsolutePath();
	RocksDBStateBackend rocksDBStateBackend = new RocksDBStateBackend(new MemoryStateBackend());
	rocksDBStateBackend.setDbStoragePath(rocksDbPath);

	CepOperator<Event, Integer, Map<String, List<Event>>> operator = CepOperatorTestUtilities.getKeyedCepOpearator(
		true,
		new SimpleNFAFactory());
	OneInputStreamOperatorTestHarness<Event, Map<String, List<Event>>> harness = CepOperatorTestUtilities.getCepTestHarness(
		operator);

	try {
		harness.setStateBackend(rocksDBStateBackend);

		harness.open();

		final ValueState nfaOperatorState = (ValueState) Whitebox.<ValueState>getInternalState(operator, "computationStates");
		final ValueState nfaOperatorStateSpy = Mockito.spy(nfaOperatorState);
		Whitebox.setInternalState(operator, "computationStates", nfaOperatorStateSpy);

		Event startEvent = new Event(42, "c", 1.0);
		SubEvent middleEvent = new SubEvent(42, "a", 1.0, 10.0);
		Event endEvent = new Event(42, "b", 1.0);

		harness.processElement(new StreamRecord<>(startEvent, 1L));
		harness.processElement(new StreamRecord<>(new Event(42, "d", 1.0), 4L));
		harness.processElement(new StreamRecord<Event>(middleEvent, 4L));
		harness.processElement(new StreamRecord<>(endEvent, 4L));

		// verify the number of invocations NFA is updated
		Mockito.verify(nfaOperatorStateSpy, Mockito.times(3)).update(Mockito.any());

		// get and verify the output
		Queue<Object> result = harness.getOutput();

		assertEquals(1, result.size());

		verifyPattern(result.poll(), startEvent, middleEvent, endEvent);
	} finally {
		harness.close();
	}
}
Example 3
Source File: PojoSerializerUpgradeTest.java    From flink with Apache License 2.0 4 votes vote down vote up 
private OperatorSubtaskState runOperator(
		Configuration taskConfiguration,
		ExecutionConfig executionConfig,
		OneInputStreamOperator<Long, Long> operator,
		KeySelector<Long, Long> keySelector,
		boolean isKeyedState,
		StateBackend stateBackend,
		ClassLoader classLoader,
		OperatorSubtaskState operatorSubtaskState,
		Iterable<Long> input) throws Exception {

	try (final MockEnvironment environment =
			new MockEnvironmentBuilder()
				.setTaskName("test task")
				.setMemorySize(32 * 1024)
				.setInputSplitProvider(new MockInputSplitProvider())
				.setBufferSize(256)
				.setTaskConfiguration(taskConfiguration)
				.setExecutionConfig(executionConfig)
				.setMaxParallelism(16)
				.setUserCodeClassLoader(classLoader)
				.build()) {

		OneInputStreamOperatorTestHarness<Long, Long> harness = null;
		try {
			if (isKeyedState) {
				harness = new KeyedOneInputStreamOperatorTestHarness<>(
					operator,
					keySelector,
					BasicTypeInfo.LONG_TYPE_INFO,
					environment);
			} else {
				harness = new OneInputStreamOperatorTestHarness<>(operator, LongSerializer.INSTANCE, environment);
			}

			harness.setStateBackend(stateBackend);

			harness.setup();
			harness.initializeState(operatorSubtaskState);
			harness.open();

			long timestamp = 0L;

			for (Long value : input) {
				harness.processElement(value, timestamp++);
			}

			long checkpointId = 1L;
			long checkpointTimestamp = timestamp + 1L;

			return harness.snapshot(checkpointId, checkpointTimestamp);
		} finally {
			IOUtils.closeQuietly(harness);
		}
	}
}
Example 4
Source File: CEPOperatorTest.java    From flink with Apache License 2.0 4 votes vote down vote up 
@Test
public void testKeyedCEPOperatorNFAUpdateTimesWithRocksDB() throws Exception {

	String rocksDbPath = tempFolder.newFolder().getAbsolutePath();
	RocksDBStateBackend rocksDBStateBackend = new RocksDBStateBackend(new MemoryStateBackend());
	rocksDBStateBackend.setDbStoragePath(rocksDbPath);

	CepOperator<Event, Integer, Map<String, List<Event>>> operator = CepOperatorTestUtilities.getKeyedCepOpearator(
		true,
		new SimpleNFAFactory());
	OneInputStreamOperatorTestHarness<Event, Map<String, List<Event>>> harness = CepOperatorTestUtilities.getCepTestHarness(
		operator);

	try {
		harness.setStateBackend(rocksDBStateBackend);

		harness.open();

		final ValueState nfaOperatorState = (ValueState) Whitebox.<ValueState>getInternalState(operator, "computationStates");
		final ValueState nfaOperatorStateSpy = Mockito.spy(nfaOperatorState);
		Whitebox.setInternalState(operator, "computationStates", nfaOperatorStateSpy);

		Event startEvent = new Event(42, "c", 1.0);
		SubEvent middleEvent = new SubEvent(42, "a", 1.0, 10.0);
		Event endEvent = new Event(42, "b", 1.0);

		harness.processElement(new StreamRecord<>(startEvent, 1L));
		harness.processElement(new StreamRecord<>(new Event(42, "d", 1.0), 4L));
		harness.processElement(new StreamRecord<Event>(middleEvent, 4L));
		harness.processElement(new StreamRecord<>(endEvent, 4L));

		// verify the number of invocations NFA is updated
		Mockito.verify(nfaOperatorStateSpy, Mockito.times(3)).update(Mockito.any());

		// get and verify the output
		Queue<Object> result = harness.getOutput();

		assertEquals(1, result.size());

		verifyPattern(result.poll(), startEvent, middleEvent, endEvent);
	} finally {
		harness.close();
	}
}
Example 5
Source File: PojoSerializerUpgradeTest.java    From flink with Apache License 2.0 4 votes vote down vote up 
private OperatorSubtaskState runOperator(
		Configuration taskConfiguration,
		ExecutionConfig executionConfig,
		OneInputStreamOperator<Long, Long> operator,
		KeySelector<Long, Long> keySelector,
		boolean isKeyedState,
		StateBackend stateBackend,
		ClassLoader classLoader,
		OperatorSubtaskState operatorSubtaskState,
		Iterable<Long> input) throws Exception {

	try (final MockEnvironment environment =
			new MockEnvironmentBuilder()
				.setTaskName("test task")
				.setManagedMemorySize(32 * 1024)
				.setInputSplitProvider(new MockInputSplitProvider())
				.setBufferSize(256)
				.setTaskConfiguration(taskConfiguration)
				.setExecutionConfig(executionConfig)
				.setMaxParallelism(16)
				.setUserCodeClassLoader(classLoader)
				.build()) {

		OneInputStreamOperatorTestHarness<Long, Long> harness = null;
		try {
			if (isKeyedState) {
				harness = new KeyedOneInputStreamOperatorTestHarness<>(
					operator,
					keySelector,
					BasicTypeInfo.LONG_TYPE_INFO,
					environment);
			} else {
				harness = new OneInputStreamOperatorTestHarness<>(operator, LongSerializer.INSTANCE, environment);
			}

			harness.setStateBackend(stateBackend);

			harness.setup();
			harness.initializeState(operatorSubtaskState);
			harness.open();

			long timestamp = 0L;

			for (Long value : input) {
				harness.processElement(value, timestamp++);
			}

			long checkpointId = 1L;
			long checkpointTimestamp = timestamp + 1L;

			return harness.snapshot(checkpointId, checkpointTimestamp);
		} finally {
			IOUtils.closeQuietly(harness);
		}
	}
}
Example 6
Source File: CEPOperatorTest.java    From flink with Apache License 2.0 4 votes vote down vote up 
@Test
public void testKeyedCEPOperatorNFAUpdateTimesWithRocksDB() throws Exception {

	String rocksDbPath = tempFolder.newFolder().getAbsolutePath();
	RocksDBStateBackend rocksDBStateBackend = new RocksDBStateBackend(new MemoryStateBackend());
	rocksDBStateBackend.setDbStoragePath(rocksDbPath);

	CepOperator<Event, Integer, Map<String, List<Event>>> operator = CepOperatorTestUtilities.getKeyedCepOpearator(
		true,
		new SimpleNFAFactory());
	OneInputStreamOperatorTestHarness<Event, Map<String, List<Event>>> harness = CepOperatorTestUtilities.getCepTestHarness(
		operator);

	try {
		harness.setStateBackend(rocksDBStateBackend);

		harness.open();

		final ValueState nfaOperatorState = (ValueState) Whitebox.<ValueState>getInternalState(operator, "computationStates");
		final ValueState nfaOperatorStateSpy = Mockito.spy(nfaOperatorState);
		Whitebox.setInternalState(operator, "computationStates", nfaOperatorStateSpy);

		Event startEvent = new Event(42, "c", 1.0);
		SubEvent middleEvent = new SubEvent(42, "a", 1.0, 10.0);
		Event endEvent = new Event(42, "b", 1.0);

		harness.processElement(new StreamRecord<>(startEvent, 1L));
		harness.processElement(new StreamRecord<>(new Event(42, "d", 1.0), 4L));
		harness.processElement(new StreamRecord<Event>(middleEvent, 4L));
		harness.processElement(new StreamRecord<>(endEvent, 4L));

		// verify the number of invocations NFA is updated
		Mockito.verify(nfaOperatorStateSpy, Mockito.times(3)).update(Mockito.any());

		// get and verify the output
		Queue<Object> result = harness.getOutput();

		assertEquals(1, result.size());

		verifyPattern(result.poll(), startEvent, middleEvent, endEvent);
	} finally {
		harness.close();
	}
}