org.apache.flink.runtime.state.heap.HeapKeyedStateBackend Java Examples

The following examples show how to use org.apache.flink.runtime.state.heap.HeapKeyedStateBackend. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: StateSnapshotCompressionTest.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up 
private HeapKeyedStateBackend<String> getStringHeapKeyedStateBackend(
	ExecutionConfig executionConfig,
	Collection<KeyedStateHandle> stateHandles)
	throws BackendBuildingException {
	return new HeapKeyedStateBackendBuilder<>(
		mock(TaskKvStateRegistry.class),
		StringSerializer.INSTANCE,
		StateSnapshotCompressionTest.class.getClassLoader(),
		16,
		new KeyGroupRange(0, 15),
		executionConfig,
		TtlTimeProvider.DEFAULT,
		stateHandles,
		AbstractStateBackend.getCompressionDecorator(executionConfig),
		TestLocalRecoveryConfig.disabled(),
		mock(HeapPriorityQueueSetFactory.class),
		true,
		new CloseableRegistry()).build();
}
Example #2
Source File: KvStateRequestSerializerTest.java    From flink with Apache License 2.0 6 votes vote down vote up 
private HeapKeyedStateBackend<Long> getLongHeapKeyedStateBackend(final long key) throws BackendBuildingException {
	final KeyGroupRange keyGroupRange = new KeyGroupRange(0, 0);
	ExecutionConfig executionConfig = new ExecutionConfig();
	// objects for heap state list serialisation
	final HeapKeyedStateBackend<Long> longHeapKeyedStateBackend =
		new HeapKeyedStateBackendBuilder<>(
			mock(TaskKvStateRegistry.class),
			LongSerializer.INSTANCE,
			ClassLoader.getSystemClassLoader(),
			keyGroupRange.getNumberOfKeyGroups(),
			keyGroupRange,
			executionConfig,
			TtlTimeProvider.DEFAULT,
			Collections.emptyList(),
			AbstractStateBackend.getCompressionDecorator(executionConfig),
			TestLocalRecoveryConfig.disabled(),
			new HeapPriorityQueueSetFactory(keyGroupRange, keyGroupRange.getNumberOfKeyGroups(), 128),
			async,
			new CloseableRegistry()).build();
	longHeapKeyedStateBackend.setCurrentKey(key);
	return longHeapKeyedStateBackend;
}
Example #3
Source File: StateSnapshotCompressionTest.java    From flink with Apache License 2.0 6 votes vote down vote up 
private HeapKeyedStateBackend<String> getStringHeapKeyedStateBackend(
	ExecutionConfig executionConfig,
	Collection<KeyedStateHandle> stateHandles)
	throws BackendBuildingException {
	return new HeapKeyedStateBackendBuilder<>(
		mock(TaskKvStateRegistry.class),
		StringSerializer.INSTANCE,
		StateSnapshotCompressionTest.class.getClassLoader(),
		16,
		new KeyGroupRange(0, 15),
		executionConfig,
		TtlTimeProvider.DEFAULT,
		stateHandles,
		AbstractStateBackend.getCompressionDecorator(executionConfig),
		TestLocalRecoveryConfig.disabled(),
		mock(HeapPriorityQueueSetFactory.class),
		true,
		new CloseableRegistry()).build();
}
Example #4
Source File: StateBackendBenchmarkUtils.java    From flink with Apache License 2.0 6 votes vote down vote up 
private static HeapKeyedStateBackend<Long> createHeapKeyedStateBackend(File rootDir) throws IOException {
	File recoveryBaseDir = prepareDirectory(recoveryDirName, rootDir);
	KeyGroupRange keyGroupRange = new KeyGroupRange(0, 1);
	int numberOfKeyGroups = keyGroupRange.getNumberOfKeyGroups();
	ExecutionConfig executionConfig = new ExecutionConfig();
	HeapPriorityQueueSetFactory priorityQueueSetFactory =
		new HeapPriorityQueueSetFactory(keyGroupRange, numberOfKeyGroups, 128);
	HeapKeyedStateBackendBuilder<Long> backendBuilder = new HeapKeyedStateBackendBuilder<>(
		null,
		new LongSerializer(),
		Thread.currentThread().getContextClassLoader(),
		numberOfKeyGroups,
		keyGroupRange,
		executionConfig,
		TtlTimeProvider.DEFAULT,
		Collections.emptyList(),
		AbstractStateBackend.getCompressionDecorator(executionConfig),
		new LocalRecoveryConfig(false, new LocalRecoveryDirectoryProviderImpl(recoveryBaseDir, new JobID(), new JobVertexID(), 0)),
		priorityQueueSetFactory,
		false,
		new CloseableRegistry()
	);
	return backendBuilder.build();
}
Example #5
Source File: KvStateRequestSerializerTest.java    From flink with Apache License 2.0 6 votes vote down vote up 
private HeapKeyedStateBackend<Long> getLongHeapKeyedStateBackend(final long key) throws BackendBuildingException {
	final KeyGroupRange keyGroupRange = new KeyGroupRange(0, 0);
	ExecutionConfig executionConfig = new ExecutionConfig();
	// objects for heap state list serialisation
	final HeapKeyedStateBackend<Long> longHeapKeyedStateBackend =
		new HeapKeyedStateBackendBuilder<>(
			mock(TaskKvStateRegistry.class),
			LongSerializer.INSTANCE,
			ClassLoader.getSystemClassLoader(),
			keyGroupRange.getNumberOfKeyGroups(),
			keyGroupRange,
			executionConfig,
			TtlTimeProvider.DEFAULT,
			Collections.emptyList(),
			AbstractStateBackend.getCompressionDecorator(executionConfig),
			TestLocalRecoveryConfig.disabled(),
			new HeapPriorityQueueSetFactory(keyGroupRange, keyGroupRange.getNumberOfKeyGroups(), 128),
			async,
			new CloseableRegistry()).build();
	longHeapKeyedStateBackend.setCurrentKey(key);
	return longHeapKeyedStateBackend;
}
Example #6
Source File: StateSnapshotCompressionTest.java    From flink with Apache License 2.0 6 votes vote down vote up 
private HeapKeyedStateBackend<String> getStringHeapKeyedStateBackend(
	ExecutionConfig executionConfig,
	Collection<KeyedStateHandle> stateHandles)
	throws BackendBuildingException {
	return new HeapKeyedStateBackendBuilder<>(
		mock(TaskKvStateRegistry.class),
		StringSerializer.INSTANCE,
		StateSnapshotCompressionTest.class.getClassLoader(),
		16,
		new KeyGroupRange(0, 15),
		executionConfig,
		TtlTimeProvider.DEFAULT,
		stateHandles,
		AbstractStateBackend.getCompressionDecorator(executionConfig),
		TestLocalRecoveryConfig.disabled(),
		mock(HeapPriorityQueueSetFactory.class),
		true,
		new CloseableRegistry()).build();
}
Example #7
Source File: KvStateRequestSerializerTest.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up 
private HeapKeyedStateBackend<Long> getLongHeapKeyedStateBackend(final long key) throws BackendBuildingException {
	final KeyGroupRange keyGroupRange = new KeyGroupRange(0, 0);
	ExecutionConfig executionConfig = new ExecutionConfig();
	// objects for heap state list serialisation
	final HeapKeyedStateBackend<Long> longHeapKeyedStateBackend =
		new HeapKeyedStateBackendBuilder<>(
			mock(TaskKvStateRegistry.class),
			LongSerializer.INSTANCE,
			ClassLoader.getSystemClassLoader(),
			keyGroupRange.getNumberOfKeyGroups(),
			keyGroupRange,
			executionConfig,
			TtlTimeProvider.DEFAULT,
			Collections.emptyList(),
			AbstractStateBackend.getCompressionDecorator(executionConfig),
			TestLocalRecoveryConfig.disabled(),
			new HeapPriorityQueueSetFactory(keyGroupRange, keyGroupRange.getNumberOfKeyGroups(), 128),
			async,
			new CloseableRegistry()).build();
	longHeapKeyedStateBackend.setCurrentKey(key);
	return longHeapKeyedStateBackend;
}
Example #8
Source File: KvStateRequestSerializerTest.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up 
/**
 * Tests list serialization utils.
 */
@Test
public void testListSerialization() throws Exception {
	final long key = 0L;
	final HeapKeyedStateBackend<Long> longHeapKeyedStateBackend = getLongHeapKeyedStateBackend(key);

	final InternalListState<Long, VoidNamespace, Long> listState = longHeapKeyedStateBackend.createInternalState(
		VoidNamespaceSerializer.INSTANCE,
		new ListStateDescriptor<>("test", LongSerializer.INSTANCE));

	testListSerialization(key, listState);
}
Example #9
Source File: KvStateRequestSerializerTest.java    From flink with Apache License 2.0 5 votes vote down vote up 
/**
 * Tests map serialization utils.
 */
@Test
public void testMapSerialization() throws Exception {
	final long key = 0L;
	final HeapKeyedStateBackend<Long> longHeapKeyedStateBackend = getLongHeapKeyedStateBackend(key);

	final InternalMapState<Long, VoidNamespace, Long, String> mapState =
			(InternalMapState<Long, VoidNamespace, Long, String>)
					longHeapKeyedStateBackend.getPartitionedState(
							VoidNamespace.INSTANCE,
							VoidNamespaceSerializer.INSTANCE,
							new MapStateDescriptor<>("test", LongSerializer.INSTANCE, StringSerializer.INSTANCE));

	testMapSerialization(key, mapState);
}
Example #10
Source File: KvStateRequestSerializerTest.java    From flink with Apache License 2.0 5 votes vote down vote up 
/**
 * Tests list serialization utils.
 */
@Test
public void testListSerialization() throws Exception {
	final long key = 0L;
	final HeapKeyedStateBackend<Long> longHeapKeyedStateBackend = getLongHeapKeyedStateBackend(key);

	final InternalListState<Long, VoidNamespace, Long> listState = longHeapKeyedStateBackend.createInternalState(
		VoidNamespaceSerializer.INSTANCE,
		new ListStateDescriptor<>("test", LongSerializer.INSTANCE));

	testListSerialization(key, listState);
}
Example #11
Source File: KeyedOneInputStreamOperatorTestHarness.java    From flink with Apache License 2.0 5 votes vote down vote up 
public <N> int numKeyedStateEntries(N namespace) {
	AbstractStreamOperator<?> abstractStreamOperator = (AbstractStreamOperator<?>) operator;
	KeyedStateBackend<Object> keyedStateBackend = abstractStreamOperator.getKeyedStateBackend();
	if (keyedStateBackend instanceof HeapKeyedStateBackend) {
		return ((HeapKeyedStateBackend) keyedStateBackend).numKeyValueStateEntries(namespace);
	} else {
		throw new UnsupportedOperationException();
	}
}
Example #12
Source File: KeyedOneInputStreamOperatorTestHarness.java    From flink with Apache License 2.0 5 votes vote down vote up 
public int numKeyedStateEntries() {
	AbstractStreamOperator<?> abstractStreamOperator = (AbstractStreamOperator<?>) operator;
	KeyedStateBackend<Object> keyedStateBackend = abstractStreamOperator.getKeyedStateBackend();
	if (keyedStateBackend instanceof HeapKeyedStateBackend) {
		return ((HeapKeyedStateBackend) keyedStateBackend).numKeyValueStateEntries();
	} else {
		throw new UnsupportedOperationException();
	}
}
Example #13
Source File: KeyedTwoInputStreamOperatorTestHarness.java    From flink with Apache License 2.0 5 votes vote down vote up 
public int numKeyedStateEntries() {
	AbstractStreamOperator<?> abstractStreamOperator = (AbstractStreamOperator<?>) operator;
	KeyedStateBackend<Object> keyedStateBackend = abstractStreamOperator.getKeyedStateBackend();
	if (keyedStateBackend instanceof HeapKeyedStateBackend) {
		return ((HeapKeyedStateBackend) keyedStateBackend).numKeyValueStateEntries();
	} else {
		throw new UnsupportedOperationException();
	}
}
Example #14
Source File: TriggerTestHarness.java    From flink with Apache License 2.0 5 votes vote down vote up 
public TriggerTestHarness(
		Trigger<T, W> trigger,
		TypeSerializer<W> windowSerializer) throws Exception {
	this.trigger = trigger;
	this.windowSerializer = windowSerializer;

	// we only ever use one key, other tests make sure that windows work across different
	// keys
	DummyEnvironment dummyEnv = new DummyEnvironment("test", 1, 0);
	MemoryStateBackend backend = new MemoryStateBackend();

	@SuppressWarnings("unchecked")
	HeapKeyedStateBackend<Integer> stateBackend = (HeapKeyedStateBackend<Integer>) backend.createKeyedStateBackend(
		dummyEnv,
		new JobID(),
		"test_op",
		IntSerializer.INSTANCE,
		1,
		new KeyGroupRange(0, 0),
		new KvStateRegistry().createTaskRegistry(new JobID(), new JobVertexID()),
		TtlTimeProvider.DEFAULT,
		new UnregisteredMetricsGroup(),
		Collections.emptyList(),
		new CloseableRegistry());
	this.stateBackend = stateBackend;

	this.stateBackend.setCurrentKey(KEY);

	this.internalTimerService = new TestInternalTimerService<>(new KeyContext() {
		@Override
		public void setCurrentKey(Object key) {
			// ignore
		}

		@Override
		public Object getCurrentKey() {
			return KEY;
		}
	});
}
Example #15
Source File: TriggerTestHarness.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up 
public TriggerTestHarness(
		Trigger<T, W> trigger,
		TypeSerializer<W> windowSerializer) throws Exception {
	this.trigger = trigger;
	this.windowSerializer = windowSerializer;

	// we only ever use one key, other tests make sure that windows work across different
	// keys
	DummyEnvironment dummyEnv = new DummyEnvironment("test", 1, 0);
	MemoryStateBackend backend = new MemoryStateBackend();

	@SuppressWarnings("unchecked")
	HeapKeyedStateBackend<Integer> stateBackend = (HeapKeyedStateBackend<Integer>) backend.createKeyedStateBackend(
		dummyEnv,
		new JobID(),
		"test_op",
		IntSerializer.INSTANCE,
		1,
		new KeyGroupRange(0, 0),
		new KvStateRegistry().createTaskRegistry(new JobID(), new JobVertexID()),
		TtlTimeProvider.DEFAULT,
		new UnregisteredMetricsGroup(),
		Collections.emptyList(),
		new CloseableRegistry());
	this.stateBackend = stateBackend;

	this.stateBackend.setCurrentKey(KEY);

	this.internalTimerService = new TestInternalTimerService<>(new KeyContext() {
		@Override
		public void setCurrentKey(Object key) {
			// ignore
		}

		@Override
		public Object getCurrentKey() {
			return KEY;
		}
	});
}
Example #16
Source File: KeyedTwoInputStreamOperatorTestHarness.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up 
public int numKeyedStateEntries() {
	AbstractStreamOperator<?> abstractStreamOperator = (AbstractStreamOperator<?>) operator;
	KeyedStateBackend<Object> keyedStateBackend = abstractStreamOperator.getKeyedStateBackend();
	if (keyedStateBackend instanceof HeapKeyedStateBackend) {
		return ((HeapKeyedStateBackend) keyedStateBackend).numKeyValueStateEntries();
	} else {
		throw new UnsupportedOperationException();
	}
}
Example #17
Source File: KeyedOneInputStreamOperatorTestHarness.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up 
public int numKeyedStateEntries() {
	AbstractStreamOperator<?> abstractStreamOperator = (AbstractStreamOperator<?>) operator;
	KeyedStateBackend<Object> keyedStateBackend = abstractStreamOperator.getKeyedStateBackend();
	if (keyedStateBackend instanceof HeapKeyedStateBackend) {
		return ((HeapKeyedStateBackend) keyedStateBackend).numKeyValueStateEntries();
	} else {
		throw new UnsupportedOperationException();
	}
}
Example #18
Source File: KvStateRequestSerializerTest.java    From flink with Apache License 2.0 5 votes vote down vote up 
/**
 * Tests map serialization utils.
 */
@Test
public void testMapSerialization() throws Exception {
	final long key = 0L;
	final HeapKeyedStateBackend<Long> longHeapKeyedStateBackend = getLongHeapKeyedStateBackend(key);

	final InternalMapState<Long, VoidNamespace, Long, String> mapState =
			(InternalMapState<Long, VoidNamespace, Long, String>)
					longHeapKeyedStateBackend.getPartitionedState(
							VoidNamespace.INSTANCE,
							VoidNamespaceSerializer.INSTANCE,
							new MapStateDescriptor<>("test", LongSerializer.INSTANCE, StringSerializer.INSTANCE));

	testMapSerialization(key, mapState);
}
Example #19
Source File: KvStateRequestSerializerTest.java    From flink with Apache License 2.0 5 votes vote down vote up 
/**
 * Tests list serialization utils.
 */
@Test
public void testListSerialization() throws Exception {
	final long key = 0L;
	final HeapKeyedStateBackend<Long> longHeapKeyedStateBackend = getLongHeapKeyedStateBackend(key);

	final InternalListState<Long, VoidNamespace, Long> listState = longHeapKeyedStateBackend.createInternalState(
		VoidNamespaceSerializer.INSTANCE,
		new ListStateDescriptor<>("test", LongSerializer.INSTANCE));

	testListSerialization(key, listState);
}
Example #20
Source File: KeyedOneInputStreamOperatorTestHarness.java    From flink with Apache License 2.0 5 votes vote down vote up 
public <N> int numKeyedStateEntries(N namespace) {
	AbstractStreamOperator<?> abstractStreamOperator = (AbstractStreamOperator<?>) operator;
	KeyedStateBackend<Object> keyedStateBackend = abstractStreamOperator.getKeyedStateBackend();
	if (keyedStateBackend instanceof HeapKeyedStateBackend) {
		return ((HeapKeyedStateBackend) keyedStateBackend).numKeyValueStateEntries(namespace);
	} else {
		throw new UnsupportedOperationException();
	}
}
Example #21
Source File: KeyedOneInputStreamOperatorTestHarness.java    From flink with Apache License 2.0 5 votes vote down vote up 
public int numKeyedStateEntries() {
	AbstractStreamOperator<?> abstractStreamOperator = (AbstractStreamOperator<?>) operator;
	KeyedStateBackend<Object> keyedStateBackend = abstractStreamOperator.getKeyedStateBackend();
	if (keyedStateBackend instanceof HeapKeyedStateBackend) {
		return ((HeapKeyedStateBackend) keyedStateBackend).numKeyValueStateEntries();
	} else {
		throw new UnsupportedOperationException();
	}
}
Example #22
Source File: KeyedTwoInputStreamOperatorTestHarness.java    From flink with Apache License 2.0 5 votes vote down vote up 
public int numKeyedStateEntries() {
	AbstractStreamOperator<?> abstractStreamOperator = (AbstractStreamOperator<?>) operator;
	KeyedStateBackend<Object> keyedStateBackend = abstractStreamOperator.getKeyedStateBackend();
	if (keyedStateBackend instanceof HeapKeyedStateBackend) {
		return ((HeapKeyedStateBackend) keyedStateBackend).numKeyValueStateEntries();
	} else {
		throw new UnsupportedOperationException();
	}
}
Example #23
Source File: TriggerTestHarness.java    From flink with Apache License 2.0 5 votes vote down vote up 
public TriggerTestHarness(
		Trigger<T, W> trigger,
		TypeSerializer<W> windowSerializer) throws Exception {
	this.trigger = trigger;
	this.windowSerializer = windowSerializer;

	// we only ever use one key, other tests make sure that windows work across different
	// keys
	DummyEnvironment dummyEnv = new DummyEnvironment("test", 1, 0);
	MemoryStateBackend backend = new MemoryStateBackend();

	@SuppressWarnings("unchecked")
	HeapKeyedStateBackend<Integer> stateBackend = (HeapKeyedStateBackend<Integer>) backend.createKeyedStateBackend(
		dummyEnv,
		new JobID(),
		"test_op",
		IntSerializer.INSTANCE,
		1,
		new KeyGroupRange(0, 0),
		new KvStateRegistry().createTaskRegistry(new JobID(), new JobVertexID()),
		TtlTimeProvider.DEFAULT,
		new UnregisteredMetricsGroup(),
		Collections.emptyList(),
		new CloseableRegistry());
	this.stateBackend = stateBackend;

	this.stateBackend.setCurrentKey(KEY);

	this.internalTimerService = new TestInternalTimerService<>(new KeyContext() {
		@Override
		public void setCurrentKey(Object key) {
			// ignore
		}

		@Override
		public Object getCurrentKey() {
			return KEY;
		}
	});
}
Example #24
Source File: KeyedOneInputStreamOperatorTestHarness.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up 
public <N> int numKeyedStateEntries(N namespace) {
	AbstractStreamOperator<?> abstractStreamOperator = (AbstractStreamOperator<?>) operator;
	KeyedStateBackend<Object> keyedStateBackend = abstractStreamOperator.getKeyedStateBackend();
	if (keyedStateBackend instanceof HeapKeyedStateBackend) {
		return ((HeapKeyedStateBackend) keyedStateBackend).numKeyValueStateEntries(namespace);
	} else {
		throw new UnsupportedOperationException();
	}
}
Example #25
Source File: KvStateRequestSerializerTest.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up 
/**
 * Tests map serialization utils.
 */
@Test
public void testMapSerialization() throws Exception {
	final long key = 0L;
	final HeapKeyedStateBackend<Long> longHeapKeyedStateBackend = getLongHeapKeyedStateBackend(key);

	final InternalMapState<Long, VoidNamespace, Long, String> mapState =
			(InternalMapState<Long, VoidNamespace, Long, String>)
					longHeapKeyedStateBackend.getPartitionedState(
							VoidNamespace.INSTANCE,
							VoidNamespaceSerializer.INSTANCE,
							new MapStateDescriptor<>("test", LongSerializer.INSTANCE, StringSerializer.INSTANCE));

	testMapSerialization(key, mapState);
}
Example #26
Source File: StateSnapshotCompressionTest.java    From flink with Apache License 2.0 4 votes vote down vote up 
private HeapKeyedStateBackend<String> getStringHeapKeyedStateBackend(ExecutionConfig executionConfig)
	throws BackendBuildingException {
	return getStringHeapKeyedStateBackend(executionConfig, Collections.emptyList());
}
Example #27
Source File: StateSnapshotCompressionTest.java    From flink with Apache License 2.0 4 votes vote down vote up 
private HeapKeyedStateBackend<String> getStringHeapKeyedStateBackend(ExecutionConfig executionConfig)
	throws BackendBuildingException {
	return getStringHeapKeyedStateBackend(executionConfig, Collections.emptyList());
}
Example #28
Source File: StateSnapshotCompressionTest.java    From Flink-CEPplus with Apache License 2.0 4 votes vote down vote up 
private HeapKeyedStateBackend<String> getStringHeapKeyedStateBackend(ExecutionConfig executionConfig)
	throws BackendBuildingException {
	return getStringHeapKeyedStateBackend(executionConfig, Collections.emptyList());
}