org.apache.flink.runtime.query.TaskKvStateRegistry Java Examples

The following examples show how to use org.apache.flink.runtime.query.TaskKvStateRegistry. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: AbstractKeyedStateBackendBuilder.java    From flink with Apache License 2.0 6 votes vote down vote up 
public AbstractKeyedStateBackendBuilder(
	TaskKvStateRegistry kvStateRegistry,
	TypeSerializer<K> keySerializer,
	ClassLoader userCodeClassLoader,
	int numberOfKeyGroups,
	KeyGroupRange keyGroupRange,
	ExecutionConfig executionConfig,
	TtlTimeProvider ttlTimeProvider,
	@Nonnull Collection<KeyedStateHandle> stateHandles,
	StreamCompressionDecorator keyGroupCompressionDecorator,
	CloseableRegistry cancelStreamRegistry) {
	this.kvStateRegistry = kvStateRegistry;
	this.keySerializerProvider = StateSerializerProvider.fromNewRegisteredSerializer(keySerializer);
	this.userCodeClassLoader = userCodeClassLoader;
	this.numberOfKeyGroups = numberOfKeyGroups;
	this.keyGroupRange = keyGroupRange;
	this.executionConfig = executionConfig;
	this.ttlTimeProvider = ttlTimeProvider;
	this.keyGroupCompressionDecorator = keyGroupCompressionDecorator;
	this.restoreStateHandles = stateHandles;
	this.cancelStreamRegistry = cancelStreamRegistry;
}
Example #2
Source File: MockStateBackend.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up 
@Override
public <K> AbstractKeyedStateBackend<K> createKeyedStateBackend(
	Environment env,
	JobID jobID,
	String operatorIdentifier,
	TypeSerializer<K> keySerializer,
	int numberOfKeyGroups,
	KeyGroupRange keyGroupRange,
	TaskKvStateRegistry kvStateRegistry,
	TtlTimeProvider ttlTimeProvider,
	MetricGroup metricGroup,
	@Nonnull Collection<KeyedStateHandle> stateHandles,
	CloseableRegistry cancelStreamRegistry) {
	return new MockKeyedStateBackendBuilder<>(
		new KvStateRegistry().createTaskRegistry(jobID, new JobVertexID()),
		keySerializer,
		env.getUserClassLoader(),
		numberOfKeyGroups,
		keyGroupRange,
		env.getExecutionConfig(),
		ttlTimeProvider,
		stateHandles,
		AbstractStateBackend.getCompressionDecorator(env.getExecutionConfig()),
		cancelStreamRegistry).build();
}
Example #3
Source File: MockKeyedStateBackend.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up 
MockKeyedStateBackend(
	TaskKvStateRegistry kvStateRegistry,
	TypeSerializer<K> keySerializer,
	ClassLoader userCodeClassLoader,
	int numberOfKeyGroups,
	KeyGroupRange keyGroupRange,
	ExecutionConfig executionConfig,
	TtlTimeProvider ttlTimeProvider,
	Map<String, Map<K, Map<Object, Object>>> stateValues,
	Map<String, StateSnapshotTransformer<Object>> stateSnapshotFilters,
	CloseableRegistry cancelStreamRegistry) {
	super(kvStateRegistry, keySerializer, userCodeClassLoader,
		numberOfKeyGroups, keyGroupRange, executionConfig, ttlTimeProvider, cancelStreamRegistry);
	this.stateValues = stateValues;
	this.stateSnapshotFilters = stateSnapshotFilters;
}
Example #4
Source File: StateSnapshotCompressionTest.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up 
private HeapKeyedStateBackend<String> getStringHeapKeyedStateBackend(
	ExecutionConfig executionConfig,
	Collection<KeyedStateHandle> stateHandles)
	throws BackendBuildingException {
	return new HeapKeyedStateBackendBuilder<>(
		mock(TaskKvStateRegistry.class),
		StringSerializer.INSTANCE,
		StateSnapshotCompressionTest.class.getClassLoader(),
		16,
		new KeyGroupRange(0, 15),
		executionConfig,
		TtlTimeProvider.DEFAULT,
		stateHandles,
		AbstractStateBackend.getCompressionDecorator(executionConfig),
		TestLocalRecoveryConfig.disabled(),
		mock(HeapPriorityQueueSetFactory.class),
		true,
		new CloseableRegistry()).build();
}
Example #5
Source File: HeapStateBackendTestBase.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up 
public <K> HeapKeyedStateBackend<K> createKeyedBackend(
	TypeSerializer<K> keySerializer,
	Collection<KeyedStateHandle> stateHandles) throws Exception {
	final KeyGroupRange keyGroupRange = new KeyGroupRange(0, 15);
	final int numKeyGroups = keyGroupRange.getNumberOfKeyGroups();
	ExecutionConfig executionConfig = new ExecutionConfig();

	return new HeapKeyedStateBackendBuilder<>(
		mock(TaskKvStateRegistry.class),
		keySerializer,
		HeapStateBackendTestBase.class.getClassLoader(),
		numKeyGroups,
		keyGroupRange,
		executionConfig,
		TtlTimeProvider.DEFAULT,
		stateHandles,
		AbstractStateBackend.getCompressionDecorator(executionConfig),
		TestLocalRecoveryConfig.disabled(),
		new HeapPriorityQueueSetFactory(keyGroupRange, numKeyGroups, 128),
		async,
		new CloseableRegistry()).build();
}
Example #6
Source File: AbstractKeyedStateBackend.java    From flink with Apache License 2.0 6 votes vote down vote up 
public AbstractKeyedStateBackend(
	TaskKvStateRegistry kvStateRegistry,
	TypeSerializer<K> keySerializer,
	ClassLoader userCodeClassLoader,
	ExecutionConfig executionConfig,
	TtlTimeProvider ttlTimeProvider,
	CloseableRegistry cancelStreamRegistry,
	InternalKeyContext<K> keyContext) {
	this(
		kvStateRegistry,
		keySerializer,
		userCodeClassLoader,
		executionConfig,
		ttlTimeProvider,
		cancelStreamRegistry,
		determineStreamCompression(executionConfig),
		keyContext
	);
}
Example #7
Source File: AbstractKeyedStateBackend.java    From flink with Apache License 2.0 6 votes vote down vote up 
public AbstractKeyedStateBackend(
	TaskKvStateRegistry kvStateRegistry,
	TypeSerializer<K> keySerializer,
	ClassLoader userCodeClassLoader,
	ExecutionConfig executionConfig,
	TtlTimeProvider ttlTimeProvider,
	CloseableRegistry cancelStreamRegistry,
	StreamCompressionDecorator keyGroupCompressionDecorator,
	InternalKeyContext<K> keyContext) {
	this.keyContext = Preconditions.checkNotNull(keyContext);
	this.numberOfKeyGroups = keyContext.getNumberOfKeyGroups();
	this.keyGroupRange = Preconditions.checkNotNull(keyContext.getKeyGroupRange());
	Preconditions.checkArgument(numberOfKeyGroups >= 1, "NumberOfKeyGroups must be a positive number");
	Preconditions.checkArgument(numberOfKeyGroups >= keyGroupRange.getNumberOfKeyGroups(), "The total number of key groups must be at least the number in the key group range assigned to this backend");

	this.kvStateRegistry = kvStateRegistry;
	this.keySerializer = keySerializer;
	this.userCodeClassLoader = Preconditions.checkNotNull(userCodeClassLoader);
	this.cancelStreamRegistry = cancelStreamRegistry;
	this.keyValueStatesByName = new HashMap<>();
	this.executionConfig = executionConfig;
	this.keyGroupCompressionDecorator = keyGroupCompressionDecorator;
	this.ttlTimeProvider = Preconditions.checkNotNull(ttlTimeProvider);
	this.keySelectionListeners = new ArrayList<>(1);
}
Example #8
Source File: MockKeyedStateBackendBuilder.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up 
public MockKeyedStateBackendBuilder(
	TaskKvStateRegistry kvStateRegistry,
	TypeSerializer<K> keySerializer,
	ClassLoader userCodeClassLoader,
	int numberOfKeyGroups,
	KeyGroupRange keyGroupRange,
	ExecutionConfig executionConfig,
	TtlTimeProvider ttlTimeProvider,
	@Nonnull Collection<KeyedStateHandle> stateHandles,
	StreamCompressionDecorator keyGroupCompressionDecorator,
	CloseableRegistry cancelStreamRegistry) {
	super(
		kvStateRegistry,
		keySerializer,
		userCodeClassLoader,
		numberOfKeyGroups,
		keyGroupRange,
		executionConfig,
		ttlTimeProvider,
		stateHandles,
		keyGroupCompressionDecorator,
		cancelStreamRegistry);
}
Example #9
Source File: AbstractKeyedStateBackendBuilder.java    From flink with Apache License 2.0 6 votes vote down vote up 
public AbstractKeyedStateBackendBuilder(
	TaskKvStateRegistry kvStateRegistry,
	TypeSerializer<K> keySerializer,
	ClassLoader userCodeClassLoader,
	int numberOfKeyGroups,
	KeyGroupRange keyGroupRange,
	ExecutionConfig executionConfig,
	TtlTimeProvider ttlTimeProvider,
	@Nonnull Collection<KeyedStateHandle> stateHandles,
	StreamCompressionDecorator keyGroupCompressionDecorator,
	CloseableRegistry cancelStreamRegistry) {
	this.kvStateRegistry = kvStateRegistry;
	this.keySerializerProvider = StateSerializerProvider.fromNewRegisteredSerializer(keySerializer);
	this.userCodeClassLoader = userCodeClassLoader;
	this.numberOfKeyGroups = numberOfKeyGroups;
	this.keyGroupRange = keyGroupRange;
	this.executionConfig = executionConfig;
	this.ttlTimeProvider = ttlTimeProvider;
	this.keyGroupCompressionDecorator = keyGroupCompressionDecorator;
	this.restoreStateHandles = stateHandles;
	this.cancelStreamRegistry = cancelStreamRegistry;
}
Example #10
Source File: HeapStateBackendTestBase.java    From flink with Apache License 2.0 6 votes vote down vote up 
public <K> HeapKeyedStateBackend<K> createKeyedBackend(
	TypeSerializer<K> keySerializer,
	Collection<KeyedStateHandle> stateHandles) throws Exception {
	final KeyGroupRange keyGroupRange = new KeyGroupRange(0, 15);
	final int numKeyGroups = keyGroupRange.getNumberOfKeyGroups();
	ExecutionConfig executionConfig = new ExecutionConfig();

	return new HeapKeyedStateBackendBuilder<>(
		mock(TaskKvStateRegistry.class),
		keySerializer,
		HeapStateBackendTestBase.class.getClassLoader(),
		numKeyGroups,
		keyGroupRange,
		executionConfig,
		TtlTimeProvider.DEFAULT,
		stateHandles,
		AbstractStateBackend.getCompressionDecorator(executionConfig),
		TestLocalRecoveryConfig.disabled(),
		new HeapPriorityQueueSetFactory(keyGroupRange, numKeyGroups, 128),
		async,
		new CloseableRegistry()).build();
}
Example #11
Source File: AbstractKeyedStateBackend.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up 
public AbstractKeyedStateBackend(
	TaskKvStateRegistry kvStateRegistry,
	StateSerializerProvider<K> keySerializerProvider,
	ClassLoader userCodeClassLoader,
	int numberOfKeyGroups,
	KeyGroupRange keyGroupRange,
	ExecutionConfig executionConfig,
	TtlTimeProvider ttlTimeProvider,
	CloseableRegistry cancelStreamRegistry,
	StreamCompressionDecorator keyGroupCompressionDecorator) {
	Preconditions.checkArgument(numberOfKeyGroups >= 1, "NumberOfKeyGroups must be a positive number");
	Preconditions.checkArgument(numberOfKeyGroups >= keyGroupRange.getNumberOfKeyGroups(), "The total number of key groups must be at least the number in the key group range assigned to this backend");

	this.kvStateRegistry = kvStateRegistry;
	this.keySerializerProvider = keySerializerProvider;
	this.numberOfKeyGroups = numberOfKeyGroups;
	this.userCodeClassLoader = Preconditions.checkNotNull(userCodeClassLoader);
	this.keyGroupRange = Preconditions.checkNotNull(keyGroupRange);
	this.cancelStreamRegistry = cancelStreamRegistry;
	this.keyValueStatesByName = new HashMap<>();
	this.executionConfig = executionConfig;
	this.keyGroupCompressionDecorator = keyGroupCompressionDecorator;
	this.ttlTimeProvider = Preconditions.checkNotNull(ttlTimeProvider);
	this.keySelectionListeners = new ArrayList<>(1);
}
Example #12
Source File: StateSnapshotCompressionTest.java    From flink with Apache License 2.0 6 votes vote down vote up 
private HeapKeyedStateBackend<String> getStringHeapKeyedStateBackend(
	ExecutionConfig executionConfig,
	Collection<KeyedStateHandle> stateHandles)
	throws BackendBuildingException {
	return new HeapKeyedStateBackendBuilder<>(
		mock(TaskKvStateRegistry.class),
		StringSerializer.INSTANCE,
		StateSnapshotCompressionTest.class.getClassLoader(),
		16,
		new KeyGroupRange(0, 15),
		executionConfig,
		TtlTimeProvider.DEFAULT,
		stateHandles,
		AbstractStateBackend.getCompressionDecorator(executionConfig),
		TestLocalRecoveryConfig.disabled(),
		mock(HeapPriorityQueueSetFactory.class),
		true,
		new CloseableRegistry()).build();
}
Example #13
Source File: AbstractKeyedStateBackendBuilder.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up 
public AbstractKeyedStateBackendBuilder(
	TaskKvStateRegistry kvStateRegistry,
	TypeSerializer<K> keySerializer,
	ClassLoader userCodeClassLoader,
	int numberOfKeyGroups,
	KeyGroupRange keyGroupRange,
	ExecutionConfig executionConfig,
	TtlTimeProvider ttlTimeProvider,
	@Nonnull Collection<KeyedStateHandle> stateHandles,
	StreamCompressionDecorator keyGroupCompressionDecorator,
	CloseableRegistry cancelStreamRegistry) {
	this.kvStateRegistry = kvStateRegistry;
	this.keySerializerProvider = StateSerializerProvider.fromNewRegisteredSerializer(keySerializer);
	this.userCodeClassLoader = userCodeClassLoader;
	this.numberOfKeyGroups = numberOfKeyGroups;
	this.keyGroupRange = keyGroupRange;
	this.executionConfig = executionConfig;
	this.ttlTimeProvider = ttlTimeProvider;
	this.keyGroupCompressionDecorator = keyGroupCompressionDecorator;
	this.restoreStateHandles = stateHandles;
	this.cancelStreamRegistry = cancelStreamRegistry;
}
Example #14
Source File: AbstractKeyedStateBackend.java    From flink with Apache License 2.0 6 votes vote down vote up 
public AbstractKeyedStateBackend(
	TaskKvStateRegistry kvStateRegistry,
	TypeSerializer<K> keySerializer,
	ClassLoader userCodeClassLoader,
	ExecutionConfig executionConfig,
	TtlTimeProvider ttlTimeProvider,
	CloseableRegistry cancelStreamRegistry,
	StreamCompressionDecorator keyGroupCompressionDecorator,
	InternalKeyContext<K> keyContext) {
	this.keyContext = Preconditions.checkNotNull(keyContext);
	this.numberOfKeyGroups = keyContext.getNumberOfKeyGroups();
	this.keyGroupRange = Preconditions.checkNotNull(keyContext.getKeyGroupRange());
	Preconditions.checkArgument(numberOfKeyGroups >= 1, "NumberOfKeyGroups must be a positive number");
	Preconditions.checkArgument(numberOfKeyGroups >= keyGroupRange.getNumberOfKeyGroups(), "The total number of key groups must be at least the number in the key group range assigned to this backend");

	this.kvStateRegistry = kvStateRegistry;
	this.keySerializer = keySerializer;
	this.userCodeClassLoader = Preconditions.checkNotNull(userCodeClassLoader);
	this.cancelStreamRegistry = cancelStreamRegistry;
	this.keyValueStatesByName = new HashMap<>();
	this.executionConfig = executionConfig;
	this.keyGroupCompressionDecorator = keyGroupCompressionDecorator;
	this.ttlTimeProvider = Preconditions.checkNotNull(ttlTimeProvider);
	this.keySelectionListeners = new ArrayList<>(1);
}
Example #15
Source File: HeapKeyedStateBackend.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up 
public HeapKeyedStateBackend(
	TaskKvStateRegistry kvStateRegistry,
	StateSerializerProvider<K> keySerializerProvider,
	ClassLoader userCodeClassLoader,
	int numberOfKeyGroups,
	KeyGroupRange keyGroupRange,
	ExecutionConfig executionConfig,
	TtlTimeProvider ttlTimeProvider,
	CloseableRegistry cancelStreamRegistry,
	StreamCompressionDecorator keyGroupCompressionDecorator,
	Map<String, StateTable<K, ?, ?>> registeredKVStates,
	Map<String, HeapPriorityQueueSnapshotRestoreWrapper> registeredPQStates,
	LocalRecoveryConfig localRecoveryConfig,
	HeapPriorityQueueSetFactory priorityQueueSetFactory,
	HeapSnapshotStrategy<K> snapshotStrategy
) {
	super(kvStateRegistry, keySerializerProvider, userCodeClassLoader, numberOfKeyGroups,
		keyGroupRange, executionConfig, ttlTimeProvider, cancelStreamRegistry, keyGroupCompressionDecorator);
	this.registeredKVStates = registeredKVStates;
	this.registeredPQStates = registeredPQStates;
	this.localRecoveryConfig = localRecoveryConfig;
	LOG.info("Initializing heap keyed state backend with stream factory.");
	this.priorityQueueSetFactory = priorityQueueSetFactory;
	this.snapshotStrategy = snapshotStrategy;
}
Example #16
Source File: KvStateRequestSerializerTest.java    From flink with Apache License 2.0 6 votes vote down vote up 
private HeapKeyedStateBackend<Long> getLongHeapKeyedStateBackend(final long key) throws BackendBuildingException {
	final KeyGroupRange keyGroupRange = new KeyGroupRange(0, 0);
	ExecutionConfig executionConfig = new ExecutionConfig();
	// objects for heap state list serialisation
	final HeapKeyedStateBackend<Long> longHeapKeyedStateBackend =
		new HeapKeyedStateBackendBuilder<>(
			mock(TaskKvStateRegistry.class),
			LongSerializer.INSTANCE,
			ClassLoader.getSystemClassLoader(),
			keyGroupRange.getNumberOfKeyGroups(),
			keyGroupRange,
			executionConfig,
			TtlTimeProvider.DEFAULT,
			Collections.emptyList(),
			AbstractStateBackend.getCompressionDecorator(executionConfig),
			TestLocalRecoveryConfig.disabled(),
			new HeapPriorityQueueSetFactory(keyGroupRange, keyGroupRange.getNumberOfKeyGroups(), 128),
			async,
			new CloseableRegistry()).build();
	longHeapKeyedStateBackend.setCurrentKey(key);
	return longHeapKeyedStateBackend;
}
Example #17
Source File: MockKeyedStateBackendBuilder.java    From flink with Apache License 2.0 6 votes vote down vote up 
public MockKeyedStateBackendBuilder(
	TaskKvStateRegistry kvStateRegistry,
	TypeSerializer<K> keySerializer,
	ClassLoader userCodeClassLoader,
	int numberOfKeyGroups,
	KeyGroupRange keyGroupRange,
	ExecutionConfig executionConfig,
	TtlTimeProvider ttlTimeProvider,
	@Nonnull Collection<KeyedStateHandle> stateHandles,
	StreamCompressionDecorator keyGroupCompressionDecorator,
	CloseableRegistry cancelStreamRegistry) {
	super(
		kvStateRegistry,
		keySerializer,
		userCodeClassLoader,
		numberOfKeyGroups,
		keyGroupRange,
		executionConfig,
		ttlTimeProvider,
		stateHandles,
		keyGroupCompressionDecorator,
		cancelStreamRegistry);
}
Example #18
Source File: MockKeyedStateBackendBuilder.java    From flink with Apache License 2.0 6 votes vote down vote up 
public MockKeyedStateBackendBuilder(
	TaskKvStateRegistry kvStateRegistry,
	TypeSerializer<K> keySerializer,
	ClassLoader userCodeClassLoader,
	int numberOfKeyGroups,
	KeyGroupRange keyGroupRange,
	ExecutionConfig executionConfig,
	TtlTimeProvider ttlTimeProvider,
	@Nonnull Collection<KeyedStateHandle> stateHandles,
	StreamCompressionDecorator keyGroupCompressionDecorator,
	CloseableRegistry cancelStreamRegistry) {
	super(
		kvStateRegistry,
		keySerializer,
		userCodeClassLoader,
		numberOfKeyGroups,
		keyGroupRange,
		executionConfig,
		ttlTimeProvider,
		stateHandles,
		keyGroupCompressionDecorator,
		cancelStreamRegistry);
}
Example #19
Source File: HeapStateBackendTestBase.java    From flink with Apache License 2.0 6 votes vote down vote up 
public <K> HeapKeyedStateBackend<K> createKeyedBackend(
	TypeSerializer<K> keySerializer,
	Collection<KeyedStateHandle> stateHandles) throws Exception {
	final KeyGroupRange keyGroupRange = new KeyGroupRange(0, 15);
	final int numKeyGroups = keyGroupRange.getNumberOfKeyGroups();
	ExecutionConfig executionConfig = new ExecutionConfig();

	return new HeapKeyedStateBackendBuilder<>(
		mock(TaskKvStateRegistry.class),
		keySerializer,
		HeapStateBackendTestBase.class.getClassLoader(),
		numKeyGroups,
		keyGroupRange,
		executionConfig,
		TtlTimeProvider.DEFAULT,
		stateHandles,
		AbstractStateBackend.getCompressionDecorator(executionConfig),
		TestLocalRecoveryConfig.disabled(),
		new HeapPriorityQueueSetFactory(keyGroupRange, numKeyGroups, 128),
		async,
		new CloseableRegistry()).build();
}
Example #20
Source File: TestSpyWrapperStateBackend.java    From flink with Apache License 2.0 6 votes vote down vote up 
@Override
public <K> AbstractKeyedStateBackend<K> createKeyedStateBackend(
	Environment env,
	JobID jobID,
	String operatorIdentifier,
	TypeSerializer<K> keySerializer,
	int numberOfKeyGroups,
	KeyGroupRange keyGroupRange,
	TaskKvStateRegistry kvStateRegistry,
	TtlTimeProvider ttlTimeProvider,
	MetricGroup metricGroup,
	@Nonnull Collection<KeyedStateHandle> stateHandles,
	CloseableRegistry cancelStreamRegistry) throws IOException {
	return spy(delegate.createKeyedStateBackend(
		env,
		jobID,
		operatorIdentifier,
		keySerializer,
		numberOfKeyGroups,
		keyGroupRange,
		kvStateRegistry,
		ttlTimeProvider,
		metricGroup,
		stateHandles,
		cancelStreamRegistry));
}
Example #21
Source File: StateSnapshotCompressionTest.java    From flink with Apache License 2.0 6 votes vote down vote up 
private HeapKeyedStateBackend<String> getStringHeapKeyedStateBackend(
	ExecutionConfig executionConfig,
	Collection<KeyedStateHandle> stateHandles)
	throws BackendBuildingException {
	return new HeapKeyedStateBackendBuilder<>(
		mock(TaskKvStateRegistry.class),
		StringSerializer.INSTANCE,
		StateSnapshotCompressionTest.class.getClassLoader(),
		16,
		new KeyGroupRange(0, 15),
		executionConfig,
		TtlTimeProvider.DEFAULT,
		stateHandles,
		AbstractStateBackend.getCompressionDecorator(executionConfig),
		TestLocalRecoveryConfig.disabled(),
		mock(HeapPriorityQueueSetFactory.class),
		true,
		new CloseableRegistry()).build();
}
Example #22
Source File: KvStateRequestSerializerTest.java    From flink with Apache License 2.0 6 votes vote down vote up 
private HeapKeyedStateBackend<Long> getLongHeapKeyedStateBackend(final long key) throws BackendBuildingException {
	final KeyGroupRange keyGroupRange = new KeyGroupRange(0, 0);
	ExecutionConfig executionConfig = new ExecutionConfig();
	// objects for heap state list serialisation
	final HeapKeyedStateBackend<Long> longHeapKeyedStateBackend =
		new HeapKeyedStateBackendBuilder<>(
			mock(TaskKvStateRegistry.class),
			LongSerializer.INSTANCE,
			ClassLoader.getSystemClassLoader(),
			keyGroupRange.getNumberOfKeyGroups(),
			keyGroupRange,
			executionConfig,
			TtlTimeProvider.DEFAULT,
			Collections.emptyList(),
			AbstractStateBackend.getCompressionDecorator(executionConfig),
			TestLocalRecoveryConfig.disabled(),
			new HeapPriorityQueueSetFactory(keyGroupRange, keyGroupRange.getNumberOfKeyGroups(), 128),
			async,
			new CloseableRegistry()).build();
	longHeapKeyedStateBackend.setCurrentKey(key);
	return longHeapKeyedStateBackend;
}
Example #23
Source File: AbstractKeyedStateBackend.java    From flink with Apache License 2.0 6 votes vote down vote up 
public AbstractKeyedStateBackend(
	TaskKvStateRegistry kvStateRegistry,
	TypeSerializer<K> keySerializer,
	ClassLoader userCodeClassLoader,
	ExecutionConfig executionConfig,
	TtlTimeProvider ttlTimeProvider,
	CloseableRegistry cancelStreamRegistry,
	InternalKeyContext<K> keyContext) {
	this(
		kvStateRegistry,
		keySerializer,
		userCodeClassLoader,
		executionConfig,
		ttlTimeProvider,
		cancelStreamRegistry,
		determineStreamCompression(executionConfig),
		keyContext
	);
}
Example #24
Source File: StateBackendITCase.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Override
public <K> AbstractKeyedStateBackend<K> createKeyedStateBackend(
	Environment env,
	JobID jobID,
	String operatorIdentifier,
	TypeSerializer<K> keySerializer,
	int numberOfKeyGroups,
	KeyGroupRange keyGroupRange,
	TaskKvStateRegistry kvStateRegistry,
	TtlTimeProvider ttlTimeProvider,
	MetricGroup metricGroup,
	@Nonnull Collection<KeyedStateHandle> stateHandles,
	CloseableRegistry cancelStreamRegistry) throws IOException {
	throw new SuccessException();
}
Example #25
Source File: HeapKeyedStateBackend.java    From flink with Apache License 2.0 5 votes vote down vote up 
public HeapKeyedStateBackend(
	TaskKvStateRegistry kvStateRegistry,
	TypeSerializer<K> keySerializer,
	ClassLoader userCodeClassLoader,
	ExecutionConfig executionConfig,
	TtlTimeProvider ttlTimeProvider,
	CloseableRegistry cancelStreamRegistry,
	StreamCompressionDecorator keyGroupCompressionDecorator,
	Map<String, StateTable<K, ?, ?>> registeredKVStates,
	Map<String, HeapPriorityQueueSnapshotRestoreWrapper> registeredPQStates,
	LocalRecoveryConfig localRecoveryConfig,
	HeapPriorityQueueSetFactory priorityQueueSetFactory,
	HeapSnapshotStrategy<K> snapshotStrategy,
	InternalKeyContext<K> keyContext) {
	super(
		kvStateRegistry,
		keySerializer,
		userCodeClassLoader,
		executionConfig,
		ttlTimeProvider,
		cancelStreamRegistry,
		keyGroupCompressionDecorator,
		keyContext);
	this.registeredKVStates = registeredKVStates;
	this.registeredPQStates = registeredPQStates;
	this.localRecoveryConfig = localRecoveryConfig;
	LOG.info("Initializing heap keyed state backend with stream factory.");
	this.priorityQueueSetFactory = priorityQueueSetFactory;
	this.snapshotStrategy = snapshotStrategy;
}
Example #26
Source File: AbstractStateBackend.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Override
public abstract <K> AbstractKeyedStateBackend<K> createKeyedStateBackend(
	Environment env,
	JobID jobID,
	String operatorIdentifier,
	TypeSerializer<K> keySerializer,
	int numberOfKeyGroups,
	KeyGroupRange keyGroupRange,
	TaskKvStateRegistry kvStateRegistry,
	TtlTimeProvider ttlTimeProvider,
	MetricGroup metricGroup,
	@Nonnull Collection<KeyedStateHandle> stateHandles,
	CloseableRegistry cancelStreamRegistry) throws IOException;
Example #27
Source File: HeapKeyedStateBackendBuilder.java    From flink with Apache License 2.0 5 votes vote down vote up 
public HeapKeyedStateBackendBuilder(
	TaskKvStateRegistry kvStateRegistry,
	TypeSerializer<K> keySerializer,
	ClassLoader userCodeClassLoader,
	int numberOfKeyGroups,
	KeyGroupRange keyGroupRange,
	ExecutionConfig executionConfig,
	TtlTimeProvider ttlTimeProvider,
	@Nonnull Collection<KeyedStateHandle> stateHandles,
	StreamCompressionDecorator keyGroupCompressionDecorator,
	LocalRecoveryConfig localRecoveryConfig,
	HeapPriorityQueueSetFactory priorityQueueSetFactory,
	boolean asynchronousSnapshots,
	CloseableRegistry cancelStreamRegistry) {
	super(
		kvStateRegistry,
		keySerializer,
		userCodeClassLoader,
		numberOfKeyGroups,
		keyGroupRange,
		executionConfig,
		ttlTimeProvider,
		stateHandles,
		keyGroupCompressionDecorator,
		cancelStreamRegistry);
	this.localRecoveryConfig = localRecoveryConfig;
	this.priorityQueueSetFactory = priorityQueueSetFactory;
	this.asynchronousSnapshots = asynchronousSnapshots;
}
Example #28
Source File: FsStateBackend.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Override
public <K> AbstractKeyedStateBackend<K> createKeyedStateBackend(
	Environment env,
	JobID jobID,
	String operatorIdentifier,
	TypeSerializer<K> keySerializer,
	int numberOfKeyGroups,
	KeyGroupRange keyGroupRange,
	TaskKvStateRegistry kvStateRegistry,
	TtlTimeProvider ttlTimeProvider,
	MetricGroup metricGroup,
	@Nonnull Collection<KeyedStateHandle> stateHandles,
	CloseableRegistry cancelStreamRegistry) throws BackendBuildingException {

	TaskStateManager taskStateManager = env.getTaskStateManager();
	LocalRecoveryConfig localRecoveryConfig = taskStateManager.createLocalRecoveryConfig();
	HeapPriorityQueueSetFactory priorityQueueSetFactory =
		new HeapPriorityQueueSetFactory(keyGroupRange, numberOfKeyGroups, 128);

	return new HeapKeyedStateBackendBuilder<>(
		kvStateRegistry,
		keySerializer,
		env.getUserClassLoader(),
		numberOfKeyGroups,
		keyGroupRange,
		env.getExecutionConfig(),
		ttlTimeProvider,
		stateHandles,
		AbstractStateBackend.getCompressionDecorator(env.getExecutionConfig()),
		localRecoveryConfig,
		priorityQueueSetFactory,
		isUsingAsynchronousSnapshots(),
		cancelStreamRegistry).build();
}
Example #29
Source File: StateBackendITCase.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up 
@Override
public <K> AbstractKeyedStateBackend<K> createKeyedStateBackend(
	Environment env,
	JobID jobID,
	String operatorIdentifier,
	TypeSerializer<K> keySerializer,
	int numberOfKeyGroups,
	KeyGroupRange keyGroupRange,
	TaskKvStateRegistry kvStateRegistry,
	TtlTimeProvider ttlTimeProvider,
	MetricGroup metricGroup,
	@Nonnull Collection<KeyedStateHandle> stateHandles,
	CloseableRegistry cancelStreamRegistry) throws IOException {
	throw new SuccessException();
}
Example #30
Source File: MemoryStateBackend.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Override
public <K> AbstractKeyedStateBackend<K> createKeyedStateBackend(
	Environment env,
	JobID jobID,
	String operatorIdentifier,
	TypeSerializer<K> keySerializer,
	int numberOfKeyGroups,
	KeyGroupRange keyGroupRange,
	TaskKvStateRegistry kvStateRegistry,
	TtlTimeProvider ttlTimeProvider,
	MetricGroup metricGroup,
	@Nonnull Collection<KeyedStateHandle> stateHandles,
	CloseableRegistry cancelStreamRegistry) throws BackendBuildingException {

	TaskStateManager taskStateManager = env.getTaskStateManager();
	HeapPriorityQueueSetFactory priorityQueueSetFactory =
		new HeapPriorityQueueSetFactory(keyGroupRange, numberOfKeyGroups, 128);
	return new HeapKeyedStateBackendBuilder<>(
		kvStateRegistry,
		keySerializer,
		env.getUserClassLoader(),
		numberOfKeyGroups,
		keyGroupRange,
		env.getExecutionConfig(),
		ttlTimeProvider,
		stateHandles,
		AbstractStateBackend.getCompressionDecorator(env.getExecutionConfig()),
		taskStateManager.createLocalRecoveryConfig(),
		priorityQueueSetFactory,
		isUsingAsynchronousSnapshots(),
		cancelStreamRegistry).build();
}