org.apache.flink.runtime.state.heap.HeapPriorityQueueSetFactory Java Examples

The following examples show how to use org.apache.flink.runtime.state.heap.HeapPriorityQueueSetFactory. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: KvStateRequestSerializerTest.java    From flink with Apache License 2.0 6 votes vote down vote up 
private HeapKeyedStateBackend<Long> getLongHeapKeyedStateBackend(final long key) throws BackendBuildingException {
	final KeyGroupRange keyGroupRange = new KeyGroupRange(0, 0);
	ExecutionConfig executionConfig = new ExecutionConfig();
	// objects for heap state list serialisation
	final HeapKeyedStateBackend<Long> longHeapKeyedStateBackend =
		new HeapKeyedStateBackendBuilder<>(
			mock(TaskKvStateRegistry.class),
			LongSerializer.INSTANCE,
			ClassLoader.getSystemClassLoader(),
			keyGroupRange.getNumberOfKeyGroups(),
			keyGroupRange,
			executionConfig,
			TtlTimeProvider.DEFAULT,
			Collections.emptyList(),
			AbstractStateBackend.getCompressionDecorator(executionConfig),
			TestLocalRecoveryConfig.disabled(),
			new HeapPriorityQueueSetFactory(keyGroupRange, keyGroupRange.getNumberOfKeyGroups(), 128),
			async,
			new CloseableRegistry()).build();
	longHeapKeyedStateBackend.setCurrentKey(key);
	return longHeapKeyedStateBackend;
}
Example #2
Source File: StateSnapshotCompressionTest.java    From flink with Apache License 2.0 6 votes vote down vote up 
private HeapKeyedStateBackend<String> getStringHeapKeyedStateBackend(
	ExecutionConfig executionConfig,
	Collection<KeyedStateHandle> stateHandles)
	throws BackendBuildingException {
	return new HeapKeyedStateBackendBuilder<>(
		mock(TaskKvStateRegistry.class),
		StringSerializer.INSTANCE,
		StateSnapshotCompressionTest.class.getClassLoader(),
		16,
		new KeyGroupRange(0, 15),
		executionConfig,
		TtlTimeProvider.DEFAULT,
		stateHandles,
		AbstractStateBackend.getCompressionDecorator(executionConfig),
		TestLocalRecoveryConfig.disabled(),
		mock(HeapPriorityQueueSetFactory.class),
		true,
		new CloseableRegistry()).build();
}
Example #3
Source File: StateSnapshotCompressionTest.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up 
private HeapKeyedStateBackend<String> getStringHeapKeyedStateBackend(
	ExecutionConfig executionConfig,
	Collection<KeyedStateHandle> stateHandles)
	throws BackendBuildingException {
	return new HeapKeyedStateBackendBuilder<>(
		mock(TaskKvStateRegistry.class),
		StringSerializer.INSTANCE,
		StateSnapshotCompressionTest.class.getClassLoader(),
		16,
		new KeyGroupRange(0, 15),
		executionConfig,
		TtlTimeProvider.DEFAULT,
		stateHandles,
		AbstractStateBackend.getCompressionDecorator(executionConfig),
		TestLocalRecoveryConfig.disabled(),
		mock(HeapPriorityQueueSetFactory.class),
		true,
		new CloseableRegistry()).build();
}
Example #4
Source File: KvStateRequestSerializerTest.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up 
private HeapKeyedStateBackend<Long> getLongHeapKeyedStateBackend(final long key) throws BackendBuildingException {
	final KeyGroupRange keyGroupRange = new KeyGroupRange(0, 0);
	ExecutionConfig executionConfig = new ExecutionConfig();
	// objects for heap state list serialisation
	final HeapKeyedStateBackend<Long> longHeapKeyedStateBackend =
		new HeapKeyedStateBackendBuilder<>(
			mock(TaskKvStateRegistry.class),
			LongSerializer.INSTANCE,
			ClassLoader.getSystemClassLoader(),
			keyGroupRange.getNumberOfKeyGroups(),
			keyGroupRange,
			executionConfig,
			TtlTimeProvider.DEFAULT,
			Collections.emptyList(),
			AbstractStateBackend.getCompressionDecorator(executionConfig),
			TestLocalRecoveryConfig.disabled(),
			new HeapPriorityQueueSetFactory(keyGroupRange, keyGroupRange.getNumberOfKeyGroups(), 128),
			async,
			new CloseableRegistry()).build();
	longHeapKeyedStateBackend.setCurrentKey(key);
	return longHeapKeyedStateBackend;
}
Example #5
Source File: StateBackendBenchmarkUtils.java    From flink with Apache License 2.0 6 votes vote down vote up 
private static HeapKeyedStateBackend<Long> createHeapKeyedStateBackend(File rootDir) throws IOException {
	File recoveryBaseDir = prepareDirectory(recoveryDirName, rootDir);
	KeyGroupRange keyGroupRange = new KeyGroupRange(0, 1);
	int numberOfKeyGroups = keyGroupRange.getNumberOfKeyGroups();
	ExecutionConfig executionConfig = new ExecutionConfig();
	HeapPriorityQueueSetFactory priorityQueueSetFactory =
		new HeapPriorityQueueSetFactory(keyGroupRange, numberOfKeyGroups, 128);
	HeapKeyedStateBackendBuilder<Long> backendBuilder = new HeapKeyedStateBackendBuilder<>(
		null,
		new LongSerializer(),
		Thread.currentThread().getContextClassLoader(),
		numberOfKeyGroups,
		keyGroupRange,
		executionConfig,
		TtlTimeProvider.DEFAULT,
		Collections.emptyList(),
		AbstractStateBackend.getCompressionDecorator(executionConfig),
		new LocalRecoveryConfig(false, new LocalRecoveryDirectoryProviderImpl(recoveryBaseDir, new JobID(), new JobVertexID(), 0)),
		priorityQueueSetFactory,
		false,
		new CloseableRegistry()
	);
	return backendBuilder.build();
}
Example #6
Source File: KvStateRequestSerializerTest.java    From flink with Apache License 2.0 6 votes vote down vote up 
private HeapKeyedStateBackend<Long> getLongHeapKeyedStateBackend(final long key) throws BackendBuildingException {
	final KeyGroupRange keyGroupRange = new KeyGroupRange(0, 0);
	ExecutionConfig executionConfig = new ExecutionConfig();
	// objects for heap state list serialisation
	final HeapKeyedStateBackend<Long> longHeapKeyedStateBackend =
		new HeapKeyedStateBackendBuilder<>(
			mock(TaskKvStateRegistry.class),
			LongSerializer.INSTANCE,
			ClassLoader.getSystemClassLoader(),
			keyGroupRange.getNumberOfKeyGroups(),
			keyGroupRange,
			executionConfig,
			TtlTimeProvider.DEFAULT,
			Collections.emptyList(),
			AbstractStateBackend.getCompressionDecorator(executionConfig),
			TestLocalRecoveryConfig.disabled(),
			new HeapPriorityQueueSetFactory(keyGroupRange, keyGroupRange.getNumberOfKeyGroups(), 128),
			async,
			new CloseableRegistry()).build();
	longHeapKeyedStateBackend.setCurrentKey(key);
	return longHeapKeyedStateBackend;
}
Example #7
Source File: StateSnapshotCompressionTest.java    From flink with Apache License 2.0 6 votes vote down vote up 
private HeapKeyedStateBackend<String> getStringHeapKeyedStateBackend(
	ExecutionConfig executionConfig,
	Collection<KeyedStateHandle> stateHandles)
	throws BackendBuildingException {
	return new HeapKeyedStateBackendBuilder<>(
		mock(TaskKvStateRegistry.class),
		StringSerializer.INSTANCE,
		StateSnapshotCompressionTest.class.getClassLoader(),
		16,
		new KeyGroupRange(0, 15),
		executionConfig,
		TtlTimeProvider.DEFAULT,
		stateHandles,
		AbstractStateBackend.getCompressionDecorator(executionConfig),
		TestLocalRecoveryConfig.disabled(),
		mock(HeapPriorityQueueSetFactory.class),
		true,
		new CloseableRegistry()).build();
}
Example #8
Source File: RocksDBStateBackendConfigTest.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up 
@Test
public void testConfigureTimerService() throws Exception {

	final Environment env = getMockEnvironment(tempFolder.newFolder());

	// Fix the option key string
	Assert.assertEquals("state.backend.rocksdb.timer-service.factory", RocksDBOptions.TIMER_SERVICE_FACTORY.key());

	// Fix the option value string and ensure all are covered
	Assert.assertEquals(2, RocksDBStateBackend.PriorityQueueStateType.values().length);
	Assert.assertEquals("ROCKSDB", RocksDBStateBackend.PriorityQueueStateType.ROCKSDB.toString());
	Assert.assertEquals("HEAP", RocksDBStateBackend.PriorityQueueStateType.HEAP.toString());

	// Fix the default
	Assert.assertEquals(
		RocksDBStateBackend.PriorityQueueStateType.HEAP.toString(),
		RocksDBOptions.TIMER_SERVICE_FACTORY.defaultValue());

	RocksDBStateBackend rocksDbBackend = new RocksDBStateBackend(tempFolder.newFolder().toURI().toString());

	RocksDBKeyedStateBackend<Integer> keyedBackend = createKeyedStateBackend(rocksDbBackend, env);
	Assert.assertEquals(HeapPriorityQueueSetFactory.class, keyedBackend.getPriorityQueueFactory().getClass());
	keyedBackend.dispose();

	Configuration conf = new Configuration();
	conf.setString(
		RocksDBOptions.TIMER_SERVICE_FACTORY,
		RocksDBStateBackend.PriorityQueueStateType.ROCKSDB.toString());

	rocksDbBackend = rocksDbBackend.configure(conf, Thread.currentThread().getContextClassLoader());
	keyedBackend = createKeyedStateBackend(rocksDbBackend, env);
	Assert.assertEquals(
		RocksDBPriorityQueueSetFactory.class,
		keyedBackend.getPriorityQueueFactory().getClass());
	keyedBackend.dispose();
}
Example #9
Source File: MemoryStateBackend.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Override
public <K> AbstractKeyedStateBackend<K> createKeyedStateBackend(
	Environment env,
	JobID jobID,
	String operatorIdentifier,
	TypeSerializer<K> keySerializer,
	int numberOfKeyGroups,
	KeyGroupRange keyGroupRange,
	TaskKvStateRegistry kvStateRegistry,
	TtlTimeProvider ttlTimeProvider,
	MetricGroup metricGroup,
	@Nonnull Collection<KeyedStateHandle> stateHandles,
	CloseableRegistry cancelStreamRegistry) throws BackendBuildingException {

	TaskStateManager taskStateManager = env.getTaskStateManager();
	HeapPriorityQueueSetFactory priorityQueueSetFactory =
		new HeapPriorityQueueSetFactory(keyGroupRange, numberOfKeyGroups, 128);
	return new HeapKeyedStateBackendBuilder<>(
		kvStateRegistry,
		keySerializer,
		env.getUserClassLoader(),
		numberOfKeyGroups,
		keyGroupRange,
		env.getExecutionConfig(),
		ttlTimeProvider,
		stateHandles,
		AbstractStateBackend.getCompressionDecorator(env.getExecutionConfig()),
		taskStateManager.createLocalRecoveryConfig(),
		priorityQueueSetFactory,
		isUsingAsynchronousSnapshots(),
		cancelStreamRegistry).build();
}
Example #10
Source File: FsStateBackend.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Override
public <K> AbstractKeyedStateBackend<K> createKeyedStateBackend(
	Environment env,
	JobID jobID,
	String operatorIdentifier,
	TypeSerializer<K> keySerializer,
	int numberOfKeyGroups,
	KeyGroupRange keyGroupRange,
	TaskKvStateRegistry kvStateRegistry,
	TtlTimeProvider ttlTimeProvider,
	MetricGroup metricGroup,
	@Nonnull Collection<KeyedStateHandle> stateHandles,
	CloseableRegistry cancelStreamRegistry) throws BackendBuildingException {

	TaskStateManager taskStateManager = env.getTaskStateManager();
	LocalRecoveryConfig localRecoveryConfig = taskStateManager.createLocalRecoveryConfig();
	HeapPriorityQueueSetFactory priorityQueueSetFactory =
		new HeapPriorityQueueSetFactory(keyGroupRange, numberOfKeyGroups, 128);

	return new HeapKeyedStateBackendBuilder<>(
		kvStateRegistry,
		keySerializer,
		env.getUserClassLoader(),
		numberOfKeyGroups,
		keyGroupRange,
		env.getExecutionConfig(),
		ttlTimeProvider,
		stateHandles,
		AbstractStateBackend.getCompressionDecorator(env.getExecutionConfig()),
		localRecoveryConfig,
		priorityQueueSetFactory,
		isUsingAsynchronousSnapshots(),
		cancelStreamRegistry).build();
}
Example #11
Source File: RocksDBStateBackendConfigTest.java    From flink with Apache License 2.0 5 votes vote down vote up 
/**
 * Validates that user custom configuration from code should override the flink-conf.yaml.
 */
@Test
public void testConfigureTimerServiceLoadingFromApplication() throws Exception {
	final MockEnvironment env = new MockEnvironmentBuilder().build();

	// priorityQueueStateType of the job backend
	final RocksDBStateBackend backend = new RocksDBStateBackend(tempFolder.newFolder().toURI().toString());
	backend.setPriorityQueueStateType(RocksDBStateBackend.PriorityQueueStateType.HEAP);

	// priorityQueueStateType in the cluster config
	final Configuration configFromConfFile = new Configuration();
	configFromConfFile.setString(
		RocksDBOptions.TIMER_SERVICE_FACTORY.key(),
		RocksDBStateBackend.PriorityQueueStateType.ROCKSDB.toString());

	// configure final backend from job and cluster config
	final RocksDBStateBackend configuredRocksDBStateBackend = backend.configure(
		configFromConfFile,
		Thread.currentThread().getContextClassLoader());
	final RocksDBKeyedStateBackend<Integer> keyedBackend = createKeyedStateBackend(configuredRocksDBStateBackend, env, IntSerializer.INSTANCE);

	// priorityQueueStateType of the job backend should be preserved
	assertThat(keyedBackend.getPriorityQueueFactory(), instanceOf(HeapPriorityQueueSetFactory.class));

	keyedBackend.close();
	keyedBackend.dispose();
	env.close();
}
Example #12
Source File: RocksDBStateBackendConfigTest.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Test
public void testConfigureTimerService() throws Exception {

	final MockEnvironment env = getMockEnvironment(tempFolder.newFolder());

	// Fix the option key string
	Assert.assertEquals("state.backend.rocksdb.timer-service.factory", RocksDBOptions.TIMER_SERVICE_FACTORY.key());

	// Fix the option value string and ensure all are covered
	Assert.assertEquals(2, RocksDBStateBackend.PriorityQueueStateType.values().length);
	Assert.assertEquals("ROCKSDB", RocksDBStateBackend.PriorityQueueStateType.ROCKSDB.toString());
	Assert.assertEquals("HEAP", RocksDBStateBackend.PriorityQueueStateType.HEAP.toString());

	// Fix the default
	Assert.assertEquals(
		RocksDBStateBackend.PriorityQueueStateType.ROCKSDB,
		RocksDBOptions.TIMER_SERVICE_FACTORY.defaultValue());

	RocksDBStateBackend rocksDbBackend = new RocksDBStateBackend(tempFolder.newFolder().toURI().toString());

	RocksDBKeyedStateBackend<Integer> keyedBackend = createKeyedStateBackend(rocksDbBackend, env, IntSerializer.INSTANCE);
	Assert.assertEquals(RocksDBPriorityQueueSetFactory.class, keyedBackend.getPriorityQueueFactory().getClass());
	keyedBackend.dispose();

	Configuration conf = new Configuration();
	conf.set(RocksDBOptions.TIMER_SERVICE_FACTORY, RocksDBStateBackend.PriorityQueueStateType.HEAP);

	rocksDbBackend = rocksDbBackend.configure(conf, Thread.currentThread().getContextClassLoader());
	keyedBackend = createKeyedStateBackend(rocksDbBackend, env, IntSerializer.INSTANCE);
	Assert.assertEquals(
		HeapPriorityQueueSetFactory.class,
		keyedBackend.getPriorityQueueFactory().getClass());
	keyedBackend.dispose();
	env.close();
}
Example #13
Source File: MemoryStateBackend.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Override
public <K> AbstractKeyedStateBackend<K> createKeyedStateBackend(
	Environment env,
	JobID jobID,
	String operatorIdentifier,
	TypeSerializer<K> keySerializer,
	int numberOfKeyGroups,
	KeyGroupRange keyGroupRange,
	TaskKvStateRegistry kvStateRegistry,
	TtlTimeProvider ttlTimeProvider,
	MetricGroup metricGroup,
	@Nonnull Collection<KeyedStateHandle> stateHandles,
	CloseableRegistry cancelStreamRegistry) throws BackendBuildingException {

	TaskStateManager taskStateManager = env.getTaskStateManager();
	HeapPriorityQueueSetFactory priorityQueueSetFactory =
		new HeapPriorityQueueSetFactory(keyGroupRange, numberOfKeyGroups, 128);
	return new HeapKeyedStateBackendBuilder<>(
		kvStateRegistry,
		keySerializer,
		env.getUserClassLoader(),
		numberOfKeyGroups,
		keyGroupRange,
		env.getExecutionConfig(),
		ttlTimeProvider,
		stateHandles,
		AbstractStateBackend.getCompressionDecorator(env.getExecutionConfig()),
		taskStateManager.createLocalRecoveryConfig(),
		priorityQueueSetFactory,
		isUsingAsynchronousSnapshots(),
		cancelStreamRegistry).build();
}
Example #14
Source File: FsStateBackend.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Override
public <K> AbstractKeyedStateBackend<K> createKeyedStateBackend(
	Environment env,
	JobID jobID,
	String operatorIdentifier,
	TypeSerializer<K> keySerializer,
	int numberOfKeyGroups,
	KeyGroupRange keyGroupRange,
	TaskKvStateRegistry kvStateRegistry,
	TtlTimeProvider ttlTimeProvider,
	MetricGroup metricGroup,
	@Nonnull Collection<KeyedStateHandle> stateHandles,
	CloseableRegistry cancelStreamRegistry) throws BackendBuildingException {

	TaskStateManager taskStateManager = env.getTaskStateManager();
	LocalRecoveryConfig localRecoveryConfig = taskStateManager.createLocalRecoveryConfig();
	HeapPriorityQueueSetFactory priorityQueueSetFactory =
		new HeapPriorityQueueSetFactory(keyGroupRange, numberOfKeyGroups, 128);

	return new HeapKeyedStateBackendBuilder<>(
		kvStateRegistry,
		keySerializer,
		env.getUserClassLoader(),
		numberOfKeyGroups,
		keyGroupRange,
		env.getExecutionConfig(),
		ttlTimeProvider,
		stateHandles,
		AbstractStateBackend.getCompressionDecorator(env.getExecutionConfig()),
		localRecoveryConfig,
		priorityQueueSetFactory,
		isUsingAsynchronousSnapshots(),
		cancelStreamRegistry).build();
}
Example #15
Source File: RocksDBStateBackendConfigTest.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Test
public void testConfigureTimerService() throws Exception {

	final Environment env = getMockEnvironment(tempFolder.newFolder());

	// Fix the option key string
	Assert.assertEquals("state.backend.rocksdb.timer-service.factory", RocksDBOptions.TIMER_SERVICE_FACTORY.key());

	// Fix the option value string and ensure all are covered
	Assert.assertEquals(2, RocksDBStateBackend.PriorityQueueStateType.values().length);
	Assert.assertEquals("ROCKSDB", RocksDBStateBackend.PriorityQueueStateType.ROCKSDB.toString());
	Assert.assertEquals("HEAP", RocksDBStateBackend.PriorityQueueStateType.HEAP.toString());

	// Fix the default
	Assert.assertEquals(
		RocksDBStateBackend.PriorityQueueStateType.HEAP.toString(),
		RocksDBOptions.TIMER_SERVICE_FACTORY.defaultValue());

	RocksDBStateBackend rocksDbBackend = new RocksDBStateBackend(tempFolder.newFolder().toURI().toString());

	RocksDBKeyedStateBackend<Integer> keyedBackend = createKeyedStateBackend(rocksDbBackend, env);
	Assert.assertEquals(HeapPriorityQueueSetFactory.class, keyedBackend.getPriorityQueueFactory().getClass());
	keyedBackend.dispose();

	Configuration conf = new Configuration();
	conf.setString(
		RocksDBOptions.TIMER_SERVICE_FACTORY,
		RocksDBStateBackend.PriorityQueueStateType.ROCKSDB.toString());

	rocksDbBackend = rocksDbBackend.configure(conf, Thread.currentThread().getContextClassLoader());
	keyedBackend = createKeyedStateBackend(rocksDbBackend, env);
	Assert.assertEquals(
		RocksDBPriorityQueueSetFactory.class,
		keyedBackend.getPriorityQueueFactory().getClass());
	keyedBackend.dispose();
}
Example #16
Source File: MemoryStateBackend.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up 
@Override
public <K> AbstractKeyedStateBackend<K> createKeyedStateBackend(
	Environment env,
	JobID jobID,
	String operatorIdentifier,
	TypeSerializer<K> keySerializer,
	int numberOfKeyGroups,
	KeyGroupRange keyGroupRange,
	TaskKvStateRegistry kvStateRegistry,
	TtlTimeProvider ttlTimeProvider,
	MetricGroup metricGroup,
	@Nonnull Collection<KeyedStateHandle> stateHandles,
	CloseableRegistry cancelStreamRegistry) throws BackendBuildingException {

	TaskStateManager taskStateManager = env.getTaskStateManager();
	HeapPriorityQueueSetFactory priorityQueueSetFactory =
		new HeapPriorityQueueSetFactory(keyGroupRange, numberOfKeyGroups, 128);
	return new HeapKeyedStateBackendBuilder<>(
		kvStateRegistry,
		keySerializer,
		env.getUserClassLoader(),
		numberOfKeyGroups,
		keyGroupRange,
		env.getExecutionConfig(),
		ttlTimeProvider,
		stateHandles,
		AbstractStateBackend.getCompressionDecorator(env.getExecutionConfig()),
		taskStateManager.createLocalRecoveryConfig(),
		priorityQueueSetFactory,
		isUsingAsynchronousSnapshots(),
		cancelStreamRegistry).build();
}
Example #17
Source File: FsStateBackend.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up 
@Override
public <K> AbstractKeyedStateBackend<K> createKeyedStateBackend(
	Environment env,
	JobID jobID,
	String operatorIdentifier,
	TypeSerializer<K> keySerializer,
	int numberOfKeyGroups,
	KeyGroupRange keyGroupRange,
	TaskKvStateRegistry kvStateRegistry,
	TtlTimeProvider ttlTimeProvider,
	MetricGroup metricGroup,
	@Nonnull Collection<KeyedStateHandle> stateHandles,
	CloseableRegistry cancelStreamRegistry) throws BackendBuildingException {

	TaskStateManager taskStateManager = env.getTaskStateManager();
	LocalRecoveryConfig localRecoveryConfig = taskStateManager.createLocalRecoveryConfig();
	HeapPriorityQueueSetFactory priorityQueueSetFactory =
		new HeapPriorityQueueSetFactory(keyGroupRange, numberOfKeyGroups, 128);

	return new HeapKeyedStateBackendBuilder<>(
		kvStateRegistry,
		keySerializer,
		env.getUserClassLoader(),
		numberOfKeyGroups,
		keyGroupRange,
		env.getExecutionConfig(),
		ttlTimeProvider,
		stateHandles,
		AbstractStateBackend.getCompressionDecorator(env.getExecutionConfig()),
		localRecoveryConfig,
		priorityQueueSetFactory,
		isUsingAsynchronousSnapshots(),
		cancelStreamRegistry).build();
}
Example #18
Source File: InternalTimerServiceImplTest.java    From flink with Apache License 2.0 4 votes vote down vote up 
/**
 * Verify that we only ever have one processing-time task registered at the
 * {@link ProcessingTimeService}.
 */
@Test
public void testOnlySetsOnePhysicalProcessingTimeTimer() throws Exception {
	@SuppressWarnings("unchecked")
	Triggerable<Integer, String> mockTriggerable = mock(Triggerable.class);

	TestKeyContext keyContext = new TestKeyContext();

	TestProcessingTimeService processingTimeService = new TestProcessingTimeService();
	PriorityQueueSetFactory priorityQueueSetFactory =
		new HeapPriorityQueueSetFactory(testKeyGroupRange, maxParallelism, 128);
	InternalTimerServiceImpl<Integer, String> timerService =
			createAndStartInternalTimerService(mockTriggerable, keyContext, processingTimeService, testKeyGroupRange, priorityQueueSetFactory);

	int key = getKeyInKeyGroupRange(testKeyGroupRange, maxParallelism);
	keyContext.setCurrentKey(key);

	timerService.registerProcessingTimeTimer("ciao", 10);
	timerService.registerProcessingTimeTimer("ciao", 20);
	timerService.registerProcessingTimeTimer("ciao", 30);
	timerService.registerProcessingTimeTimer("hello", 10);
	timerService.registerProcessingTimeTimer("hello", 20);

	assertEquals(5, timerService.numProcessingTimeTimers());
	assertEquals(2, timerService.numProcessingTimeTimers("hello"));
	assertEquals(3, timerService.numProcessingTimeTimers("ciao"));

	assertEquals(1, processingTimeService.getNumActiveTimers());
	assertThat(processingTimeService.getActiveTimerTimestamps(), containsInAnyOrder(10L));

	processingTimeService.setCurrentTime(10);

	assertEquals(3, timerService.numProcessingTimeTimers());
	assertEquals(1, timerService.numProcessingTimeTimers("hello"));
	assertEquals(2, timerService.numProcessingTimeTimers("ciao"));

	assertEquals(1, processingTimeService.getNumActiveTimers());
	assertThat(processingTimeService.getActiveTimerTimestamps(), containsInAnyOrder(20L));

	processingTimeService.setCurrentTime(20);

	assertEquals(1, timerService.numProcessingTimeTimers());
	assertEquals(0, timerService.numProcessingTimeTimers("hello"));
	assertEquals(1, timerService.numProcessingTimeTimers("ciao"));

	assertEquals(1, processingTimeService.getNumActiveTimers());
	assertThat(processingTimeService.getActiveTimerTimestamps(), containsInAnyOrder(30L));

	processingTimeService.setCurrentTime(30);

	assertEquals(0, timerService.numProcessingTimeTimers());

	assertEquals(0, processingTimeService.getNumActiveTimers());

	timerService.registerProcessingTimeTimer("ciao", 40);

	assertEquals(1, processingTimeService.getNumActiveTimers());
}
Example #19
Source File: InternalTimerServiceImplTest.java    From flink with Apache License 2.0 4 votes vote down vote up 
protected PriorityQueueSetFactory createQueueFactory(KeyGroupRange keyGroupRange, int numKeyGroups) {
	return new HeapPriorityQueueSetFactory(keyGroupRange, numKeyGroups, 128);
}
Example #20
Source File: RocksDBKeyedStateBackend.java    From flink with Apache License 2.0 4 votes vote down vote up 
@Override
public boolean requiresLegacySynchronousTimerSnapshots() {
	return priorityQueueFactory instanceof HeapPriorityQueueSetFactory;
}
Example #21
Source File: RocksDBKeyedStateBackend.java    From flink with Apache License 2.0 4 votes vote down vote up 
@Override
public boolean requiresLegacySynchronousTimerSnapshots() {
	return priorityQueueFactory instanceof HeapPriorityQueueSetFactory;
}
Example #22
Source File: InternalTimerServiceImplTest.java    From Flink-CEPplus with Apache License 2.0 4 votes vote down vote up 
protected PriorityQueueSetFactory createQueueFactory(KeyGroupRange keyGroupRange, int numKeyGroups) {
	return new HeapPriorityQueueSetFactory(keyGroupRange, numKeyGroups, 128);
}
Example #23
Source File: InternalTimerServiceImplTest.java    From Flink-CEPplus with Apache License 2.0 4 votes vote down vote up 
/**
 * Verify that we only ever have one processing-time task registered at the
 * {@link ProcessingTimeService}.
 */
@Test
public void testOnlySetsOnePhysicalProcessingTimeTimer() throws Exception {
	@SuppressWarnings("unchecked")
	Triggerable<Integer, String> mockTriggerable = mock(Triggerable.class);

	TestKeyContext keyContext = new TestKeyContext();

	TestProcessingTimeService processingTimeService = new TestProcessingTimeService();
	PriorityQueueSetFactory priorityQueueSetFactory =
		new HeapPriorityQueueSetFactory(testKeyGroupRange, maxParallelism, 128);
	InternalTimerServiceImpl<Integer, String> timerService =
			createAndStartInternalTimerService(mockTriggerable, keyContext, processingTimeService, testKeyGroupRange, priorityQueueSetFactory);

	int key = getKeyInKeyGroupRange(testKeyGroupRange, maxParallelism);
	keyContext.setCurrentKey(key);

	timerService.registerProcessingTimeTimer("ciao", 10);
	timerService.registerProcessingTimeTimer("ciao", 20);
	timerService.registerProcessingTimeTimer("ciao", 30);
	timerService.registerProcessingTimeTimer("hello", 10);
	timerService.registerProcessingTimeTimer("hello", 20);

	assertEquals(5, timerService.numProcessingTimeTimers());
	assertEquals(2, timerService.numProcessingTimeTimers("hello"));
	assertEquals(3, timerService.numProcessingTimeTimers("ciao"));

	assertEquals(1, processingTimeService.getNumActiveTimers());
	assertThat(processingTimeService.getActiveTimerTimestamps(), containsInAnyOrder(10L));

	processingTimeService.setCurrentTime(10);

	assertEquals(3, timerService.numProcessingTimeTimers());
	assertEquals(1, timerService.numProcessingTimeTimers("hello"));
	assertEquals(2, timerService.numProcessingTimeTimers("ciao"));

	assertEquals(1, processingTimeService.getNumActiveTimers());
	assertThat(processingTimeService.getActiveTimerTimestamps(), containsInAnyOrder(20L));

	processingTimeService.setCurrentTime(20);

	assertEquals(1, timerService.numProcessingTimeTimers());
	assertEquals(0, timerService.numProcessingTimeTimers("hello"));
	assertEquals(1, timerService.numProcessingTimeTimers("ciao"));

	assertEquals(1, processingTimeService.getNumActiveTimers());
	assertThat(processingTimeService.getActiveTimerTimestamps(), containsInAnyOrder(30L));

	processingTimeService.setCurrentTime(30);

	assertEquals(0, timerService.numProcessingTimeTimers());

	assertEquals(0, processingTimeService.getNumActiveTimers());

	timerService.registerProcessingTimeTimer("ciao", 40);

	assertEquals(1, processingTimeService.getNumActiveTimers());
}
Example #24
Source File: InternalTimerServiceImplTest.java    From flink with Apache License 2.0 4 votes vote down vote up 
/**
 * Verify that we only ever have one processing-time task registered at the
 * {@link ProcessingTimeService}.
 */
@Test
public void testOnlySetsOnePhysicalProcessingTimeTimer() throws Exception {
	@SuppressWarnings("unchecked")
	Triggerable<Integer, String> mockTriggerable = mock(Triggerable.class);

	TestKeyContext keyContext = new TestKeyContext();

	TestProcessingTimeService processingTimeService = new TestProcessingTimeService();
	PriorityQueueSetFactory priorityQueueSetFactory =
		new HeapPriorityQueueSetFactory(testKeyGroupRange, maxParallelism, 128);
	InternalTimerServiceImpl<Integer, String> timerService =
			createAndStartInternalTimerService(mockTriggerable, keyContext, processingTimeService, testKeyGroupRange, priorityQueueSetFactory);

	int key = getKeyInKeyGroupRange(testKeyGroupRange, maxParallelism);
	keyContext.setCurrentKey(key);

	timerService.registerProcessingTimeTimer("ciao", 10);
	timerService.registerProcessingTimeTimer("ciao", 20);
	timerService.registerProcessingTimeTimer("ciao", 30);
	timerService.registerProcessingTimeTimer("hello", 10);
	timerService.registerProcessingTimeTimer("hello", 20);

	assertEquals(5, timerService.numProcessingTimeTimers());
	assertEquals(2, timerService.numProcessingTimeTimers("hello"));
	assertEquals(3, timerService.numProcessingTimeTimers("ciao"));

	assertEquals(1, processingTimeService.getNumActiveTimers());
	assertThat(processingTimeService.getActiveTimerTimestamps(), containsInAnyOrder(10L));

	processingTimeService.setCurrentTime(10);

	assertEquals(3, timerService.numProcessingTimeTimers());
	assertEquals(1, timerService.numProcessingTimeTimers("hello"));
	assertEquals(2, timerService.numProcessingTimeTimers("ciao"));

	assertEquals(1, processingTimeService.getNumActiveTimers());
	assertThat(processingTimeService.getActiveTimerTimestamps(), containsInAnyOrder(20L));

	processingTimeService.setCurrentTime(20);

	assertEquals(1, timerService.numProcessingTimeTimers());
	assertEquals(0, timerService.numProcessingTimeTimers("hello"));
	assertEquals(1, timerService.numProcessingTimeTimers("ciao"));

	assertEquals(1, processingTimeService.getNumActiveTimers());
	assertThat(processingTimeService.getActiveTimerTimestamps(), containsInAnyOrder(30L));

	processingTimeService.setCurrentTime(30);

	assertEquals(0, timerService.numProcessingTimeTimers());

	assertEquals(0, processingTimeService.getNumActiveTimers());

	timerService.registerProcessingTimeTimer("ciao", 40);

	assertEquals(1, processingTimeService.getNumActiveTimers());
}
Example #25
Source File: InternalTimerServiceImplTest.java    From flink with Apache License 2.0 4 votes vote down vote up 
protected PriorityQueueSetFactory createQueueFactory(KeyGroupRange keyGroupRange, int numKeyGroups) {
	return new HeapPriorityQueueSetFactory(keyGroupRange, numKeyGroups, 128);
}
Example #26
Source File: RocksDBKeyedStateBackend.java    From Flink-CEPplus with Apache License 2.0 4 votes vote down vote up 
@Override
public boolean requiresLegacySynchronousTimerSnapshots() {
	return priorityQueueFactory instanceof HeapPriorityQueueSetFactory;
}