org.apache.flink.runtime.state.LocalRecoveryConfig Java Examples

The following examples show how to use org.apache.flink.runtime.state.LocalRecoveryConfig. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: HeapKeyedStateBackend.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up 
public HeapKeyedStateBackend(
	TaskKvStateRegistry kvStateRegistry,
	StateSerializerProvider<K> keySerializerProvider,
	ClassLoader userCodeClassLoader,
	int numberOfKeyGroups,
	KeyGroupRange keyGroupRange,
	ExecutionConfig executionConfig,
	TtlTimeProvider ttlTimeProvider,
	CloseableRegistry cancelStreamRegistry,
	StreamCompressionDecorator keyGroupCompressionDecorator,
	Map<String, StateTable<K, ?, ?>> registeredKVStates,
	Map<String, HeapPriorityQueueSnapshotRestoreWrapper> registeredPQStates,
	LocalRecoveryConfig localRecoveryConfig,
	HeapPriorityQueueSetFactory priorityQueueSetFactory,
	HeapSnapshotStrategy<K> snapshotStrategy
) {
	super(kvStateRegistry, keySerializerProvider, userCodeClassLoader, numberOfKeyGroups,
		keyGroupRange, executionConfig, ttlTimeProvider, cancelStreamRegistry, keyGroupCompressionDecorator);
	this.registeredKVStates = registeredKVStates;
	this.registeredPQStates = registeredPQStates;
	this.localRecoveryConfig = localRecoveryConfig;
	LOG.info("Initializing heap keyed state backend with stream factory.");
	this.priorityQueueSetFactory = priorityQueueSetFactory;
	this.snapshotStrategy = snapshotStrategy;
}
Example #2
Source File: OneInputStreamTaskTestHarness.java    From flink with Apache License 2.0 6 votes vote down vote up 
/**
 * Creates a test harness with the specified number of input gates and specified number
 * of channels per input gate and specified localRecoveryConfig.
 */
public OneInputStreamTaskTestHarness(
		Function<Environment, ? extends StreamTask<OUT, ?>> taskFactory,
		int numInputGates,
		int numInputChannelsPerGate,
		TypeInformation<IN> inputType,
		TypeInformation<OUT> outputType,
		LocalRecoveryConfig localRecoveryConfig) {

	super(taskFactory, outputType, localRecoveryConfig);

	this.inputType = inputType;
	inputSerializer = inputType.createSerializer(executionConfig);

	this.numInputGates = numInputGates;
	this.numInputChannelsPerGate = numInputChannelsPerGate;
}
Example #3
Source File: StreamTaskTestHarness.java    From flink with Apache License 2.0 6 votes vote down vote up 
public StreamTaskTestHarness(
	Function<Environment, ? extends StreamTask<OUT, ?>> taskFactory,
	TypeInformation<OUT> outputType,
	LocalRecoveryConfig localRecoveryConfig) {
	this.taskFactory = checkNotNull(taskFactory);
	this.memorySize = DEFAULT_MEMORY_MANAGER_SIZE;
	this.bufferSize = DEFAULT_NETWORK_BUFFER_SIZE;

	this.jobConfig = new Configuration();
	this.taskConfig = new Configuration();
	this.executionConfig = new ExecutionConfig();

	streamConfig = new StreamConfig(taskConfig);

	outputSerializer = outputType.createSerializer(executionConfig);
	outputStreamRecordSerializer = new StreamElementSerializer<OUT>(outputSerializer);

	this.taskStateManager = new TestTaskStateManager(localRecoveryConfig);
}
Example #4
Source File: RocksDBSnapshotStrategyBase.java    From flink with Apache License 2.0 6 votes vote down vote up 
public RocksDBSnapshotStrategyBase(
	@Nonnull String description,
	@Nonnull RocksDB db,
	@Nonnull ResourceGuard rocksDBResourceGuard,
	@Nonnull TypeSerializer<K> keySerializer,
	@Nonnull LinkedHashMap<String, RocksDbKvStateInfo> kvStateInformation,
	@Nonnull KeyGroupRange keyGroupRange,
	@Nonnegative int keyGroupPrefixBytes,
	@Nonnull LocalRecoveryConfig localRecoveryConfig,
	@Nonnull CloseableRegistry cancelStreamRegistry) {

	super(description);
	this.db = db;
	this.rocksDBResourceGuard = rocksDBResourceGuard;
	this.keySerializer = keySerializer;
	this.kvStateInformation = kvStateInformation;
	this.keyGroupRange = keyGroupRange;
	this.keyGroupPrefixBytes = keyGroupPrefixBytes;
	this.localRecoveryConfig = localRecoveryConfig;
	this.cancelStreamRegistry = cancelStreamRegistry;
}
Example #5
Source File: OneInputStreamTaskTestHarness.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up 
/**
 * Creates a test harness with the specified number of input gates and specified number
 * of channels per input gate and specified localRecoveryConfig.
 */
public OneInputStreamTaskTestHarness(
		Function<Environment, ? extends StreamTask<OUT, ?>> taskFactory,
		int numInputGates,
		int numInputChannelsPerGate,
		TypeInformation<IN> inputType,
		TypeInformation<OUT> outputType,
		LocalRecoveryConfig localRecoveryConfig) {

	super(taskFactory, outputType, localRecoveryConfig);

	this.inputType = inputType;
	inputSerializer = inputType.createSerializer(executionConfig);

	this.numInputGates = numInputGates;
	this.numInputChannelsPerGate = numInputChannelsPerGate;
}
Example #6
Source File: StreamTaskTestHarness.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up 
public StreamTaskTestHarness(
	Function<Environment, ? extends StreamTask<OUT, ?>> taskFactory,
	TypeInformation<OUT> outputType,
	LocalRecoveryConfig localRecoveryConfig) {
	this.taskFactory = checkNotNull(taskFactory);
	this.memorySize = DEFAULT_MEMORY_MANAGER_SIZE;
	this.bufferSize = DEFAULT_NETWORK_BUFFER_SIZE;

	this.jobConfig = new Configuration();
	this.taskConfig = new Configuration();
	this.executionConfig = new ExecutionConfig();

	streamConfig = new StreamConfig(taskConfig);

	outputSerializer = outputType.createSerializer(executionConfig);
	outputStreamRecordSerializer = new StreamElementSerializer<OUT>(outputSerializer);

	this.taskStateManager = new TestTaskStateManager(localRecoveryConfig);
}
Example #7
Source File: RocksFullSnapshotStrategy.java    From flink with Apache License 2.0 6 votes vote down vote up 
public RocksFullSnapshotStrategy(
	@Nonnull RocksDB db,
	@Nonnull ResourceGuard rocksDBResourceGuard,
	@Nonnull TypeSerializer<K> keySerializer,
	@Nonnull LinkedHashMap<String, RocksDbKvStateInfo> kvStateInformation,
	@Nonnull KeyGroupRange keyGroupRange,
	@Nonnegative int keyGroupPrefixBytes,
	@Nonnull LocalRecoveryConfig localRecoveryConfig,
	@Nonnull CloseableRegistry cancelStreamRegistry,
	@Nonnull StreamCompressionDecorator keyGroupCompressionDecorator) {
	super(
		DESCRIPTION,
		db,
		rocksDBResourceGuard,
		keySerializer,
		kvStateInformation,
		keyGroupRange,
		keyGroupPrefixBytes,
		localRecoveryConfig,
		cancelStreamRegistry);

	this.keyGroupCompressionDecorator = keyGroupCompressionDecorator;
}
Example #8
Source File: HeapSnapshotStrategy.java    From flink with Apache License 2.0 6 votes vote down vote up 
HeapSnapshotStrategy(
	SnapshotStrategySynchronicityBehavior<K> snapshotStrategySynchronicityTrait,
	Map<String, StateTable<K, ?, ?>> registeredKVStates,
	Map<String, HeapPriorityQueueSnapshotRestoreWrapper> registeredPQStates,
	StreamCompressionDecorator keyGroupCompressionDecorator,
	LocalRecoveryConfig localRecoveryConfig,
	KeyGroupRange keyGroupRange,
	CloseableRegistry cancelStreamRegistry,
	StateSerializerProvider<K> keySerializerProvider) {
	super("Heap backend snapshot");
	this.snapshotStrategySynchronicityTrait = snapshotStrategySynchronicityTrait;
	this.registeredKVStates = registeredKVStates;
	this.registeredPQStates = registeredPQStates;
	this.keyGroupCompressionDecorator = keyGroupCompressionDecorator;
	this.localRecoveryConfig = localRecoveryConfig;
	this.keyGroupRange = keyGroupRange;
	this.cancelStreamRegistry = cancelStreamRegistry;
	this.keySerializerProvider = keySerializerProvider;
}
Example #9
Source File: HeapSnapshotStrategy.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up 
HeapSnapshotStrategy(
	SnapshotStrategySynchronicityBehavior<K> snapshotStrategySynchronicityTrait,
	Map<String, StateTable<K, ?, ?>> registeredKVStates,
	Map<String, HeapPriorityQueueSnapshotRestoreWrapper> registeredPQStates,
	StreamCompressionDecorator keyGroupCompressionDecorator,
	LocalRecoveryConfig localRecoveryConfig,
	KeyGroupRange keyGroupRange,
	CloseableRegistry cancelStreamRegistry,
	StateSerializerProvider<K> keySerializerProvider) {
	super("Heap backend snapshot");
	this.snapshotStrategySynchronicityTrait = snapshotStrategySynchronicityTrait;
	this.registeredKVStates = registeredKVStates;
	this.registeredPQStates = registeredPQStates;
	this.keyGroupCompressionDecorator = keyGroupCompressionDecorator;
	this.localRecoveryConfig = localRecoveryConfig;
	this.keyGroupRange = keyGroupRange;
	this.cancelStreamRegistry = cancelStreamRegistry;
	this.keySerializerProvider = keySerializerProvider;
}
Example #10
Source File: StateBackendBenchmarkUtils.java    From flink with Apache License 2.0 6 votes vote down vote up 
private static HeapKeyedStateBackend<Long> createHeapKeyedStateBackend(File rootDir) throws IOException {
	File recoveryBaseDir = prepareDirectory(recoveryDirName, rootDir);
	KeyGroupRange keyGroupRange = new KeyGroupRange(0, 1);
	int numberOfKeyGroups = keyGroupRange.getNumberOfKeyGroups();
	ExecutionConfig executionConfig = new ExecutionConfig();
	HeapPriorityQueueSetFactory priorityQueueSetFactory =
		new HeapPriorityQueueSetFactory(keyGroupRange, numberOfKeyGroups, 128);
	HeapKeyedStateBackendBuilder<Long> backendBuilder = new HeapKeyedStateBackendBuilder<>(
		null,
		new LongSerializer(),
		Thread.currentThread().getContextClassLoader(),
		numberOfKeyGroups,
		keyGroupRange,
		executionConfig,
		TtlTimeProvider.DEFAULT,
		Collections.emptyList(),
		AbstractStateBackend.getCompressionDecorator(executionConfig),
		new LocalRecoveryConfig(false, new LocalRecoveryDirectoryProviderImpl(recoveryBaseDir, new JobID(), new JobVertexID(), 0)),
		priorityQueueSetFactory,
		false,
		new CloseableRegistry()
	);
	return backendBuilder.build();
}
Example #11
Source File: RocksDBSnapshotStrategyBase.java    From flink with Apache License 2.0 6 votes vote down vote up 
public RocksDBSnapshotStrategyBase(
	@Nonnull String description,
	@Nonnull RocksDB db,
	@Nonnull ResourceGuard rocksDBResourceGuard,
	@Nonnull TypeSerializer<K> keySerializer,
	@Nonnull LinkedHashMap<String, RocksDbKvStateInfo> kvStateInformation,
	@Nonnull KeyGroupRange keyGroupRange,
	@Nonnegative int keyGroupPrefixBytes,
	@Nonnull LocalRecoveryConfig localRecoveryConfig,
	@Nonnull CloseableRegistry cancelStreamRegistry) {

	super(description);
	this.db = db;
	this.rocksDBResourceGuard = rocksDBResourceGuard;
	this.keySerializer = keySerializer;
	this.kvStateInformation = kvStateInformation;
	this.keyGroupRange = keyGroupRange;
	this.keyGroupPrefixBytes = keyGroupPrefixBytes;
	this.localRecoveryConfig = localRecoveryConfig;
	this.cancelStreamRegistry = cancelStreamRegistry;
}
Example #12
Source File: RocksFullSnapshotStrategy.java    From flink with Apache License 2.0 6 votes vote down vote up 
public RocksFullSnapshotStrategy(
	@Nonnull RocksDB db,
	@Nonnull ResourceGuard rocksDBResourceGuard,
	@Nonnull TypeSerializer<K> keySerializer,
	@Nonnull LinkedHashMap<String, RocksDbKvStateInfo> kvStateInformation,
	@Nonnull KeyGroupRange keyGroupRange,
	@Nonnegative int keyGroupPrefixBytes,
	@Nonnull LocalRecoveryConfig localRecoveryConfig,
	@Nonnull CloseableRegistry cancelStreamRegistry,
	@Nonnull StreamCompressionDecorator keyGroupCompressionDecorator) {
	super(
		DESCRIPTION,
		db,
		rocksDBResourceGuard,
		keySerializer,
		kvStateInformation,
		keyGroupRange,
		keyGroupPrefixBytes,
		localRecoveryConfig,
		cancelStreamRegistry);

	this.keyGroupCompressionDecorator = keyGroupCompressionDecorator;
}
Example #13
Source File: RocksFullSnapshotStrategy.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up 
public RocksFullSnapshotStrategy(
	@Nonnull RocksDB db,
	@Nonnull ResourceGuard rocksDBResourceGuard,
	@Nonnull TypeSerializer<K> keySerializer,
	@Nonnull LinkedHashMap<String, RocksDbKvStateInfo> kvStateInformation,
	@Nonnull KeyGroupRange keyGroupRange,
	@Nonnegative int keyGroupPrefixBytes,
	@Nonnull LocalRecoveryConfig localRecoveryConfig,
	@Nonnull CloseableRegistry cancelStreamRegistry,
	@Nonnull StreamCompressionDecorator keyGroupCompressionDecorator) {
	super(
		DESCRIPTION,
		db,
		rocksDBResourceGuard,
		keySerializer,
		kvStateInformation,
		keyGroupRange,
		keyGroupPrefixBytes,
		localRecoveryConfig,
		cancelStreamRegistry);

	this.keyGroupCompressionDecorator = keyGroupCompressionDecorator;
}
Example #14
Source File: RocksDBSnapshotStrategyBase.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up 
public RocksDBSnapshotStrategyBase(
	@Nonnull String description,
	@Nonnull RocksDB db,
	@Nonnull ResourceGuard rocksDBResourceGuard,
	@Nonnull TypeSerializer<K> keySerializer,
	@Nonnull LinkedHashMap<String, RocksDbKvStateInfo> kvStateInformation,
	@Nonnull KeyGroupRange keyGroupRange,
	@Nonnegative int keyGroupPrefixBytes,
	@Nonnull LocalRecoveryConfig localRecoveryConfig,
	@Nonnull CloseableRegistry cancelStreamRegistry) {

	super(description);
	this.db = db;
	this.rocksDBResourceGuard = rocksDBResourceGuard;
	this.keySerializer = keySerializer;
	this.kvStateInformation = kvStateInformation;
	this.keyGroupRange = keyGroupRange;
	this.keyGroupPrefixBytes = keyGroupPrefixBytes;
	this.localRecoveryConfig = localRecoveryConfig;
	this.cancelStreamRegistry = cancelStreamRegistry;
}
Example #15
Source File: HeapSnapshotStrategy.java    From flink with Apache License 2.0 6 votes vote down vote up 
HeapSnapshotStrategy(
	SnapshotStrategySynchronicityBehavior<K> snapshotStrategySynchronicityTrait,
	Map<String, StateTable<K, ?, ?>> registeredKVStates,
	Map<String, HeapPriorityQueueSnapshotRestoreWrapper> registeredPQStates,
	StreamCompressionDecorator keyGroupCompressionDecorator,
	LocalRecoveryConfig localRecoveryConfig,
	KeyGroupRange keyGroupRange,
	CloseableRegistry cancelStreamRegistry,
	StateSerializerProvider<K> keySerializerProvider) {
	super("Heap backend snapshot");
	this.snapshotStrategySynchronicityTrait = snapshotStrategySynchronicityTrait;
	this.registeredKVStates = registeredKVStates;
	this.registeredPQStates = registeredPQStates;
	this.keyGroupCompressionDecorator = keyGroupCompressionDecorator;
	this.localRecoveryConfig = localRecoveryConfig;
	this.keyGroupRange = keyGroupRange;
	this.cancelStreamRegistry = cancelStreamRegistry;
	this.keySerializerProvider = keySerializerProvider;
}
Example #16
Source File: StreamTaskTestHarness.java    From flink with Apache License 2.0 6 votes vote down vote up 
public StreamTaskTestHarness(
	FunctionWithException<Environment, ? extends StreamTask<OUT, ?>, Exception> taskFactory,
	TypeInformation<OUT> outputType,
	LocalRecoveryConfig localRecoveryConfig) {
	this.taskFactory = checkNotNull(taskFactory);
	this.memorySize = DEFAULT_MEMORY_MANAGER_SIZE;
	this.bufferSize = DEFAULT_NETWORK_BUFFER_SIZE;

	this.jobConfig = new Configuration();
	this.taskConfig = new Configuration();
	this.executionConfig = new ExecutionConfig();

	streamConfig = new StreamConfig(taskConfig);
	streamConfig.setBufferTimeout(0);

	outputSerializer = outputType.createSerializer(executionConfig);
	outputStreamRecordSerializer = new StreamElementSerializer<>(outputSerializer);

	this.taskStateManager = new TestTaskStateManager(localRecoveryConfig);
}
Example #17
Source File: OneInputStreamTaskTestHarness.java    From flink with Apache License 2.0 6 votes vote down vote up 
/**
 * Creates a test harness with the specified number of input gates and specified number
 * of channels per input gate and specified localRecoveryConfig.
 */
public OneInputStreamTaskTestHarness(
		FunctionWithException<Environment, ? extends StreamTask<OUT, ?>, Exception> taskFactory,
		int numInputGates,
		int numInputChannelsPerGate,
		TypeInformation<IN> inputType,
		TypeInformation<OUT> outputType,
		LocalRecoveryConfig localRecoveryConfig) {

	super(taskFactory, outputType, localRecoveryConfig);

	this.inputType = inputType;
	inputSerializer = inputType.createSerializer(executionConfig);

	this.numInputGates = numInputGates;
	this.numInputChannelsPerGate = numInputChannelsPerGate;
}
Example #18
Source File: HeapKeyedStateBackend.java    From flink with Apache License 2.0 5 votes vote down vote up 
public HeapKeyedStateBackend(
	TaskKvStateRegistry kvStateRegistry,
	TypeSerializer<K> keySerializer,
	ClassLoader userCodeClassLoader,
	ExecutionConfig executionConfig,
	TtlTimeProvider ttlTimeProvider,
	CloseableRegistry cancelStreamRegistry,
	StreamCompressionDecorator keyGroupCompressionDecorator,
	Map<String, StateTable<K, ?, ?>> registeredKVStates,
	Map<String, HeapPriorityQueueSnapshotRestoreWrapper> registeredPQStates,
	LocalRecoveryConfig localRecoveryConfig,
	HeapPriorityQueueSetFactory priorityQueueSetFactory,
	HeapSnapshotStrategy<K> snapshotStrategy,
	InternalKeyContext<K> keyContext) {
	super(
		kvStateRegistry,
		keySerializer,
		userCodeClassLoader,
		executionConfig,
		ttlTimeProvider,
		cancelStreamRegistry,
		keyGroupCompressionDecorator,
		keyContext);
	this.registeredKVStates = registeredKVStates;
	this.registeredPQStates = registeredPQStates;
	this.localRecoveryConfig = localRecoveryConfig;
	LOG.info("Initializing heap keyed state backend with stream factory.");
	this.priorityQueueSetFactory = priorityQueueSetFactory;
	this.snapshotStrategy = snapshotStrategy;
}
Example #19
Source File: HeapKeyedStateBackendBuilder.java    From flink with Apache License 2.0 5 votes vote down vote up 
public HeapKeyedStateBackendBuilder(
	TaskKvStateRegistry kvStateRegistry,
	TypeSerializer<K> keySerializer,
	ClassLoader userCodeClassLoader,
	int numberOfKeyGroups,
	KeyGroupRange keyGroupRange,
	ExecutionConfig executionConfig,
	TtlTimeProvider ttlTimeProvider,
	@Nonnull Collection<KeyedStateHandle> stateHandles,
	StreamCompressionDecorator keyGroupCompressionDecorator,
	LocalRecoveryConfig localRecoveryConfig,
	HeapPriorityQueueSetFactory priorityQueueSetFactory,
	boolean asynchronousSnapshots,
	CloseableRegistry cancelStreamRegistry) {
	super(
		kvStateRegistry,
		keySerializer,
		userCodeClassLoader,
		numberOfKeyGroups,
		keyGroupRange,
		executionConfig,
		ttlTimeProvider,
		stateHandles,
		keyGroupCompressionDecorator,
		cancelStreamRegistry);
	this.localRecoveryConfig = localRecoveryConfig;
	this.priorityQueueSetFactory = priorityQueueSetFactory;
	this.asynchronousSnapshots = asynchronousSnapshots;
}
Example #20
Source File: RocksIncrementalSnapshotStrategy.java    From flink with Apache License 2.0 5 votes vote down vote up 
public RocksIncrementalSnapshotStrategy(
	@Nonnull RocksDB db,
	@Nonnull ResourceGuard rocksDBResourceGuard,
	@Nonnull TypeSerializer<K> keySerializer,
	@Nonnull LinkedHashMap<String, RocksDbKvStateInfo> kvStateInformation,
	@Nonnull KeyGroupRange keyGroupRange,
	@Nonnegative int keyGroupPrefixBytes,
	@Nonnull LocalRecoveryConfig localRecoveryConfig,
	@Nonnull CloseableRegistry cancelStreamRegistry,
	@Nonnull File instanceBasePath,
	@Nonnull UUID backendUID,
	@Nonnull SortedMap<Long, Set<StateHandleID>> materializedSstFiles,
	long lastCompletedCheckpointId,
	int numberOfTransferingThreads) {

	super(
		DESCRIPTION,
		db,
		rocksDBResourceGuard,
		keySerializer,
		kvStateInformation,
		keyGroupRange,
		keyGroupPrefixBytes,
		localRecoveryConfig,
		cancelStreamRegistry);

	this.instanceBasePath = instanceBasePath;
	this.backendUID = backendUID;
	this.materializedSstFiles = materializedSstFiles;
	this.lastCompletedCheckpointId = lastCompletedCheckpointId;
	this.stateUploader = new RocksDBStateUploader(numberOfTransferingThreads);
	this.localDirectoryName = backendUID.toString().replaceAll("[\\-]", "");
}
Example #21
Source File: StateBackendBenchmarkUtils.java    From flink with Apache License 2.0 5 votes vote down vote up 
private static RocksDBKeyedStateBackend<Long> createRocksDBKeyedStateBackend(File rootDir) throws IOException {
	File recoveryBaseDir = prepareDirectory(recoveryDirName, rootDir);
	File dbPathFile = prepareDirectory(dbDirName, rootDir);
	ExecutionConfig executionConfig = new ExecutionConfig();
	RocksDBResourceContainer resourceContainer = new RocksDBResourceContainer();
	RocksDBKeyedStateBackendBuilder<Long> builder = new RocksDBKeyedStateBackendBuilder<>(
		"Test",
		Thread.currentThread().getContextClassLoader(),
		dbPathFile,
		resourceContainer,
		stateName -> resourceContainer.getColumnOptions(),
		null,
		LongSerializer.INSTANCE,
		2,
		new KeyGroupRange(0, 1),
		executionConfig,
		new LocalRecoveryConfig(false, new LocalRecoveryDirectoryProviderImpl(recoveryBaseDir, new JobID(), new JobVertexID(), 0)),
		RocksDBStateBackend.PriorityQueueStateType.ROCKSDB,
		TtlTimeProvider.DEFAULT,
		new UnregisteredMetricsGroup(),
		Collections.emptyList(),
		AbstractStateBackend.getCompressionDecorator(executionConfig),
		new CloseableRegistry());
	try {
		return builder.build();
	} catch (Exception e) {
		IOUtils.closeQuietly(resourceContainer);
		throw e;
	}
}
Example #22
Source File: FsStateBackend.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Override
public <K> AbstractKeyedStateBackend<K> createKeyedStateBackend(
	Environment env,
	JobID jobID,
	String operatorIdentifier,
	TypeSerializer<K> keySerializer,
	int numberOfKeyGroups,
	KeyGroupRange keyGroupRange,
	TaskKvStateRegistry kvStateRegistry,
	TtlTimeProvider ttlTimeProvider,
	MetricGroup metricGroup,
	@Nonnull Collection<KeyedStateHandle> stateHandles,
	CloseableRegistry cancelStreamRegistry) throws BackendBuildingException {

	TaskStateManager taskStateManager = env.getTaskStateManager();
	LocalRecoveryConfig localRecoveryConfig = taskStateManager.createLocalRecoveryConfig();
	HeapPriorityQueueSetFactory priorityQueueSetFactory =
		new HeapPriorityQueueSetFactory(keyGroupRange, numberOfKeyGroups, 128);

	return new HeapKeyedStateBackendBuilder<>(
		kvStateRegistry,
		keySerializer,
		env.getUserClassLoader(),
		numberOfKeyGroups,
		keyGroupRange,
		env.getExecutionConfig(),
		ttlTimeProvider,
		stateHandles,
		AbstractStateBackend.getCompressionDecorator(env.getExecutionConfig()),
		localRecoveryConfig,
		priorityQueueSetFactory,
		isUsingAsynchronousSnapshots(),
		cancelStreamRegistry).build();
}
Example #23
Source File: HeapKeyedStateBackendBuilder.java    From flink with Apache License 2.0 5 votes vote down vote up 
public HeapKeyedStateBackendBuilder(
	TaskKvStateRegistry kvStateRegistry,
	TypeSerializer<K> keySerializer,
	ClassLoader userCodeClassLoader,
	int numberOfKeyGroups,
	KeyGroupRange keyGroupRange,
	ExecutionConfig executionConfig,
	TtlTimeProvider ttlTimeProvider,
	@Nonnull Collection<KeyedStateHandle> stateHandles,
	StreamCompressionDecorator keyGroupCompressionDecorator,
	LocalRecoveryConfig localRecoveryConfig,
	HeapPriorityQueueSetFactory priorityQueueSetFactory,
	boolean asynchronousSnapshots,
	CloseableRegistry cancelStreamRegistry) {
	super(
		kvStateRegistry,
		keySerializer,
		userCodeClassLoader,
		numberOfKeyGroups,
		keyGroupRange,
		executionConfig,
		ttlTimeProvider,
		stateHandles,
		keyGroupCompressionDecorator,
		cancelStreamRegistry);
	this.localRecoveryConfig = localRecoveryConfig;
	this.priorityQueueSetFactory = priorityQueueSetFactory;
	this.asynchronousSnapshots = asynchronousSnapshots;
}
Example #24
Source File: HeapKeyedStateBackend.java    From flink with Apache License 2.0 5 votes vote down vote up 
public HeapKeyedStateBackend(
	TaskKvStateRegistry kvStateRegistry,
	TypeSerializer<K> keySerializer,
	ClassLoader userCodeClassLoader,
	ExecutionConfig executionConfig,
	TtlTimeProvider ttlTimeProvider,
	CloseableRegistry cancelStreamRegistry,
	StreamCompressionDecorator keyGroupCompressionDecorator,
	Map<String, StateTable<K, ?, ?>> registeredKVStates,
	Map<String, HeapPriorityQueueSnapshotRestoreWrapper> registeredPQStates,
	LocalRecoveryConfig localRecoveryConfig,
	HeapPriorityQueueSetFactory priorityQueueSetFactory,
	HeapSnapshotStrategy<K> snapshotStrategy,
	InternalKeyContext<K> keyContext) {
	super(
		kvStateRegistry,
		keySerializer,
		userCodeClassLoader,
		executionConfig,
		ttlTimeProvider,
		cancelStreamRegistry,
		keyGroupCompressionDecorator,
		keyContext);
	this.registeredKVStates = registeredKVStates;
	this.registeredPQStates = registeredPQStates;
	this.localRecoveryConfig = localRecoveryConfig;
	LOG.info("Initializing heap keyed state backend with stream factory.");
	this.priorityQueueSetFactory = priorityQueueSetFactory;
	this.snapshotStrategy = snapshotStrategy;
}
Example #25
Source File: RocksIncrementalSnapshotStrategy.java    From flink with Apache License 2.0 5 votes vote down vote up 
public RocksIncrementalSnapshotStrategy(
	@Nonnull RocksDB db,
	@Nonnull ResourceGuard rocksDBResourceGuard,
	@Nonnull TypeSerializer<K> keySerializer,
	@Nonnull LinkedHashMap<String, RocksDbKvStateInfo> kvStateInformation,
	@Nonnull KeyGroupRange keyGroupRange,
	@Nonnegative int keyGroupPrefixBytes,
	@Nonnull LocalRecoveryConfig localRecoveryConfig,
	@Nonnull CloseableRegistry cancelStreamRegistry,
	@Nonnull File instanceBasePath,
	@Nonnull UUID backendUID,
	@Nonnull SortedMap<Long, Set<StateHandleID>> materializedSstFiles,
	long lastCompletedCheckpointId,
	int numberOfTransferingThreads) {

	super(
		DESCRIPTION,
		db,
		rocksDBResourceGuard,
		keySerializer,
		kvStateInformation,
		keyGroupRange,
		keyGroupPrefixBytes,
		localRecoveryConfig,
		cancelStreamRegistry);

	this.instanceBasePath = instanceBasePath;
	this.backendUID = backendUID;
	this.materializedSstFiles = materializedSstFiles;
	this.lastCompletedCheckpointId = lastCompletedCheckpointId;
	this.stateUploader = new RocksDBStateUploader(numberOfTransferingThreads);
	this.localDirectoryName = backendUID.toString().replaceAll("[\\-]", "");
}
Example #26
Source File: RocksIncrementalSnapshotStrategy.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up 
public RocksIncrementalSnapshotStrategy(
	@Nonnull RocksDB db,
	@Nonnull ResourceGuard rocksDBResourceGuard,
	@Nonnull TypeSerializer<K> keySerializer,
	@Nonnull LinkedHashMap<String, RocksDbKvStateInfo> kvStateInformation,
	@Nonnull KeyGroupRange keyGroupRange,
	@Nonnegative int keyGroupPrefixBytes,
	@Nonnull LocalRecoveryConfig localRecoveryConfig,
	@Nonnull CloseableRegistry cancelStreamRegistry,
	@Nonnull File instanceBasePath,
	@Nonnull UUID backendUID,
	@Nonnull SortedMap<Long, Set<StateHandleID>> materializedSstFiles,
	long lastCompletedCheckpointId,
	int numberOfTransferingThreads) {

	super(
		DESCRIPTION,
		db,
		rocksDBResourceGuard,
		keySerializer,
		kvStateInformation,
		keyGroupRange,
		keyGroupPrefixBytes,
		localRecoveryConfig,
		cancelStreamRegistry);

	this.instanceBasePath = instanceBasePath;
	this.backendUID = backendUID;
	this.materializedSstFiles = materializedSstFiles;
	this.lastCompletedCheckpointId = lastCompletedCheckpointId;
	this.stateUploader = new RocksDBStateUploader(numberOfTransferingThreads);
	this.localDirectoryName = backendUID.toString().replaceAll("[\\-]", "");
}
Example #27
Source File: FsStateBackend.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up 
@Override
public <K> AbstractKeyedStateBackend<K> createKeyedStateBackend(
	Environment env,
	JobID jobID,
	String operatorIdentifier,
	TypeSerializer<K> keySerializer,
	int numberOfKeyGroups,
	KeyGroupRange keyGroupRange,
	TaskKvStateRegistry kvStateRegistry,
	TtlTimeProvider ttlTimeProvider,
	MetricGroup metricGroup,
	@Nonnull Collection<KeyedStateHandle> stateHandles,
	CloseableRegistry cancelStreamRegistry) throws BackendBuildingException {

	TaskStateManager taskStateManager = env.getTaskStateManager();
	LocalRecoveryConfig localRecoveryConfig = taskStateManager.createLocalRecoveryConfig();
	HeapPriorityQueueSetFactory priorityQueueSetFactory =
		new HeapPriorityQueueSetFactory(keyGroupRange, numberOfKeyGroups, 128);

	return new HeapKeyedStateBackendBuilder<>(
		kvStateRegistry,
		keySerializer,
		env.getUserClassLoader(),
		numberOfKeyGroups,
		keyGroupRange,
		env.getExecutionConfig(),
		ttlTimeProvider,
		stateHandles,
		AbstractStateBackend.getCompressionDecorator(env.getExecutionConfig()),
		localRecoveryConfig,
		priorityQueueSetFactory,
		isUsingAsynchronousSnapshots(),
		cancelStreamRegistry).build();
}
Example #28
Source File: HeapKeyedStateBackendBuilder.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up 
public HeapKeyedStateBackendBuilder(
	TaskKvStateRegistry kvStateRegistry,
	TypeSerializer<K> keySerializer,
	ClassLoader userCodeClassLoader,
	int numberOfKeyGroups,
	KeyGroupRange keyGroupRange,
	ExecutionConfig executionConfig,
	TtlTimeProvider ttlTimeProvider,
	@Nonnull Collection<KeyedStateHandle> stateHandles,
	StreamCompressionDecorator keyGroupCompressionDecorator,
	LocalRecoveryConfig localRecoveryConfig,
	HeapPriorityQueueSetFactory priorityQueueSetFactory,
	boolean asynchronousSnapshots,
	CloseableRegistry cancelStreamRegistry) {
	super(
		kvStateRegistry,
		keySerializer,
		userCodeClassLoader,
		numberOfKeyGroups,
		keyGroupRange,
		executionConfig,
		ttlTimeProvider,
		stateHandles,
		keyGroupCompressionDecorator,
		cancelStreamRegistry);
	this.localRecoveryConfig = localRecoveryConfig;
	this.priorityQueueSetFactory = priorityQueueSetFactory;
	this.asynchronousSnapshots = asynchronousSnapshots;
}
Example #29
Source File: FsStateBackend.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Override
public <K> AbstractKeyedStateBackend<K> createKeyedStateBackend(
	Environment env,
	JobID jobID,
	String operatorIdentifier,
	TypeSerializer<K> keySerializer,
	int numberOfKeyGroups,
	KeyGroupRange keyGroupRange,
	TaskKvStateRegistry kvStateRegistry,
	TtlTimeProvider ttlTimeProvider,
	MetricGroup metricGroup,
	@Nonnull Collection<KeyedStateHandle> stateHandles,
	CloseableRegistry cancelStreamRegistry) throws BackendBuildingException {

	TaskStateManager taskStateManager = env.getTaskStateManager();
	LocalRecoveryConfig localRecoveryConfig = taskStateManager.createLocalRecoveryConfig();
	HeapPriorityQueueSetFactory priorityQueueSetFactory =
		new HeapPriorityQueueSetFactory(keyGroupRange, numberOfKeyGroups, 128);

	return new HeapKeyedStateBackendBuilder<>(
		kvStateRegistry,
		keySerializer,
		env.getUserClassLoader(),
		numberOfKeyGroups,
		keyGroupRange,
		env.getExecutionConfig(),
		ttlTimeProvider,
		stateHandles,
		AbstractStateBackend.getCompressionDecorator(env.getExecutionConfig()),
		localRecoveryConfig,
		priorityQueueSetFactory,
		isUsingAsynchronousSnapshots(),
		cancelStreamRegistry).build();
}
Example #30
Source File: RocksDBStateBackend.java    From flink with Apache License 2.0 4 votes vote down vote up 
@Override
public <K> AbstractKeyedStateBackend<K> createKeyedStateBackend(
	Environment env,
	JobID jobID,
	String operatorIdentifier,
	TypeSerializer<K> keySerializer,
	int numberOfKeyGroups,
	KeyGroupRange keyGroupRange,
	TaskKvStateRegistry kvStateRegistry,
	TtlTimeProvider ttlTimeProvider,
	MetricGroup metricGroup,
	@Nonnull Collection<KeyedStateHandle> stateHandles,
	CloseableRegistry cancelStreamRegistry) throws IOException {

	// first, make sure that the RocksDB JNI library is loaded
	// we do this explicitly here to have better error handling
	String tempDir = env.getTaskManagerInfo().getTmpDirectories()[0];
	ensureRocksDBIsLoaded(tempDir);

	// replace all characters that are not legal for filenames with underscore
	String fileCompatibleIdentifier = operatorIdentifier.replaceAll("[^a-zA-Z0-9\\-]", "_");

	lazyInitializeForJob(env, fileCompatibleIdentifier);

	File instanceBasePath = new File(
		getNextStoragePath(),
		"job_" + jobId + "_op_" + fileCompatibleIdentifier + "_uuid_" + UUID.randomUUID());

	LocalRecoveryConfig localRecoveryConfig =
		env.getTaskStateManager().createLocalRecoveryConfig();

	ExecutionConfig executionConfig = env.getExecutionConfig();
	StreamCompressionDecorator keyGroupCompressionDecorator = getCompressionDecorator(executionConfig);
	RocksDBKeyedStateBackendBuilder<K> builder = new RocksDBKeyedStateBackendBuilder<>(
		operatorIdentifier,
		env.getUserClassLoader(),
		instanceBasePath,
		getDbOptions(),
		stateName -> getColumnOptions(),
		kvStateRegistry,
		keySerializer,
		numberOfKeyGroups,
		keyGroupRange,
		executionConfig,
		localRecoveryConfig,
		priorityQueueStateType,
		ttlTimeProvider,
		metricGroup,
		stateHandles,
		keyGroupCompressionDecorator,
		cancelStreamRegistry
	).setEnableIncrementalCheckpointing(isIncrementalCheckpointsEnabled())
		.setEnableTtlCompactionFilter(isTtlCompactionFilterEnabled())
		.setNumberOfTransferingThreads(getNumberOfTransferingThreads())
		.setNativeMetricOptions(getMemoryWatcherOptions());
	return builder.build();
}