org.apache.flink.contrib.streaming.state.ttl.RocksDbTtlCompactFiltersManager Java Examples

The following examples show how to use org.apache.flink.contrib.streaming.state.ttl.RocksDbTtlCompactFiltersManager. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: RocksDBOperationUtils.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up 
/**
 * Creates a column descriptor for sate column family.
 *
 * <p>Sets TTL compaction filter if {@code ttlCompactFiltersManager} is not {@code null}.
 */
public static ColumnFamilyDescriptor createColumnFamilyDescriptor(
	RegisteredStateMetaInfoBase metaInfoBase,
	Function<String, ColumnFamilyOptions> columnFamilyOptionsFactory,
	@Nullable RocksDbTtlCompactFiltersManager ttlCompactFiltersManager) {

	ColumnFamilyOptions options = createColumnFamilyOptions(columnFamilyOptionsFactory, metaInfoBase.getName());
	if (ttlCompactFiltersManager != null) {
		ttlCompactFiltersManager.setAndRegisterCompactFilterIfStateTtl(metaInfoBase, options);
	}
	byte[] nameBytes = metaInfoBase.getName().getBytes(ConfigConstants.DEFAULT_CHARSET);
	Preconditions.checkState(!Arrays.equals(RocksDB.DEFAULT_COLUMN_FAMILY, nameBytes),
		"The chosen state name 'default' collides with the name of the default column family!");

	return new ColumnFamilyDescriptor(nameBytes, options);
}
Example #2
Source File: RocksDBOperationUtils.java    From flink with Apache License 2.0 6 votes vote down vote up 
/**
 * Creates a column descriptor for sate column family.
 *
 * <p>Sets TTL compaction filter if {@code ttlCompactFiltersManager} is not {@code null}.
 */
public static ColumnFamilyDescriptor createColumnFamilyDescriptor(
	RegisteredStateMetaInfoBase metaInfoBase,
	Function<String, ColumnFamilyOptions> columnFamilyOptionsFactory,
	@Nullable RocksDbTtlCompactFiltersManager ttlCompactFiltersManager) {

	ColumnFamilyOptions options = createColumnFamilyOptions(columnFamilyOptionsFactory, metaInfoBase.getName());
	if (ttlCompactFiltersManager != null) {
		ttlCompactFiltersManager.setAndRegisterCompactFilterIfStateTtl(metaInfoBase, options);
	}
	byte[] nameBytes = metaInfoBase.getName().getBytes(ConfigConstants.DEFAULT_CHARSET);
	Preconditions.checkState(!Arrays.equals(RocksDB.DEFAULT_COLUMN_FAMILY, nameBytes),
		"The chosen state name 'default' collides with the name of the default column family!");

	return new ColumnFamilyDescriptor(nameBytes, options);
}
Example #3
Source File: RocksDBOperationUtils.java    From flink with Apache License 2.0 6 votes vote down vote up 
/**
 * Creates a column descriptor for sate column family.
 *
 * <p>Sets TTL compaction filter if {@code ttlCompactFiltersManager} is not {@code null}.
 */
public static ColumnFamilyDescriptor createColumnFamilyDescriptor(
	RegisteredStateMetaInfoBase metaInfoBase,
	Function<String, ColumnFamilyOptions> columnFamilyOptionsFactory,
	@Nullable RocksDbTtlCompactFiltersManager ttlCompactFiltersManager) {

	ColumnFamilyOptions options = createColumnFamilyOptions(columnFamilyOptionsFactory, metaInfoBase.getName());
	if (ttlCompactFiltersManager != null) {
		ttlCompactFiltersManager.setAndRegisterCompactFilterIfStateTtl(metaInfoBase, options);
	}
	byte[] nameBytes = metaInfoBase.getName().getBytes(ConfigConstants.DEFAULT_CHARSET);
	Preconditions.checkState(!Arrays.equals(RocksDB.DEFAULT_COLUMN_FAMILY, nameBytes),
		"The chosen state name 'default' collides with the name of the default column family!");

	return new ColumnFamilyDescriptor(nameBytes, options);
}
Example #4
Source File: RocksDBIncrementalRestoreOperation.java    From flink with Apache License 2.0 5 votes vote down vote up 
public RocksDBIncrementalRestoreOperation(
	String operatorIdentifier,
	KeyGroupRange keyGroupRange,
	int keyGroupPrefixBytes,
	int numberOfTransferringThreads,
	CloseableRegistry cancelStreamRegistry,
	ClassLoader userCodeClassLoader,
	Map<String, RocksDbKvStateInfo> kvStateInformation,
	StateSerializerProvider<K> keySerializerProvider,
	File instanceBasePath,
	File instanceRocksDBPath,
	DBOptions dbOptions,
	Function<String, ColumnFamilyOptions> columnFamilyOptionsFactory,
	RocksDBNativeMetricOptions nativeMetricOptions,
	MetricGroup metricGroup,
	@Nonnull Collection<KeyedStateHandle> restoreStateHandles,
	@Nonnull RocksDbTtlCompactFiltersManager ttlCompactFiltersManager) {
	super(keyGroupRange,
		keyGroupPrefixBytes,
		numberOfTransferringThreads,
		cancelStreamRegistry,
		userCodeClassLoader,
		kvStateInformation,
		keySerializerProvider,
		instanceBasePath,
		instanceRocksDBPath,
		dbOptions,
		columnFamilyOptionsFactory,
		nativeMetricOptions,
		metricGroup,
		restoreStateHandles,
		ttlCompactFiltersManager);
	this.operatorIdentifier = operatorIdentifier;
	this.restoredSstFiles = new TreeMap<>();
	this.lastCompletedCheckpointId = -1L;
	this.backendUID = UUID.randomUUID();
}
Example #5
Source File: RocksDBNoneRestoreOperation.java    From flink with Apache License 2.0 5 votes vote down vote up 
public RocksDBNoneRestoreOperation(
	KeyGroupRange keyGroupRange,
	int keyGroupPrefixBytes,
	int numberOfTransferringThreads,
	CloseableRegistry cancelStreamRegistry,
	ClassLoader userCodeClassLoader,
	Map<String, RocksDbKvStateInfo> kvStateInformation,
	StateSerializerProvider<K> keySerializerProvider,
	File instanceBasePath,
	File instanceRocksDBPath,
	DBOptions dbOptions,
	Function<String, ColumnFamilyOptions> columnFamilyOptionsFactory,
	RocksDBNativeMetricOptions nativeMetricOptions,
	MetricGroup metricGroup,
	@Nonnull Collection<KeyedStateHandle> restoreStateHandles,
	@Nonnull RocksDbTtlCompactFiltersManager ttlCompactFiltersManager
) {
	super(keyGroupRange,
		keyGroupPrefixBytes,
		numberOfTransferringThreads,
		cancelStreamRegistry,
		userCodeClassLoader,
		kvStateInformation,
		keySerializerProvider,
		instanceBasePath,
		instanceRocksDBPath,
		dbOptions,
		columnFamilyOptionsFactory,
		nativeMetricOptions,
		metricGroup,
		restoreStateHandles,
		ttlCompactFiltersManager);
}
Example #6
Source File: RocksDBIncrementalRestoreOperation.java    From flink with Apache License 2.0 5 votes vote down vote up 
public RocksDBIncrementalRestoreOperation(
	String operatorIdentifier,
	KeyGroupRange keyGroupRange,
	int keyGroupPrefixBytes,
	int numberOfTransferringThreads,
	CloseableRegistry cancelStreamRegistry,
	ClassLoader userCodeClassLoader,
	Map<String, RocksDbKvStateInfo> kvStateInformation,
	StateSerializerProvider<K> keySerializerProvider,
	File instanceBasePath,
	File instanceRocksDBPath,
	DBOptions dbOptions,
	Function<String, ColumnFamilyOptions> columnFamilyOptionsFactory,
	RocksDBNativeMetricOptions nativeMetricOptions,
	MetricGroup metricGroup,
	@Nonnull Collection<KeyedStateHandle> restoreStateHandles,
	@Nonnull RocksDbTtlCompactFiltersManager ttlCompactFiltersManager,
	@Nonnegative long writeBatchSize) {
	super(keyGroupRange,
		keyGroupPrefixBytes,
		numberOfTransferringThreads,
		cancelStreamRegistry,
		userCodeClassLoader,
		kvStateInformation,
		keySerializerProvider,
		instanceBasePath,
		instanceRocksDBPath,
		dbOptions,
		columnFamilyOptionsFactory,
		nativeMetricOptions,
		metricGroup,
		restoreStateHandles,
		ttlCompactFiltersManager);
	this.operatorIdentifier = operatorIdentifier;
	this.restoredSstFiles = new TreeMap<>();
	this.lastCompletedCheckpointId = -1L;
	this.backendUID = UUID.randomUUID();
	checkArgument(writeBatchSize >= 0, "Write batch size have to be no negative.");
	this.writeBatchSize = writeBatchSize;
}
Example #7
Source File: RocksDBFullRestoreOperation.java    From flink with Apache License 2.0 5 votes vote down vote up 
public RocksDBFullRestoreOperation(
	KeyGroupRange keyGroupRange,
	int keyGroupPrefixBytes,
	int numberOfTransferringThreads,
	CloseableRegistry cancelStreamRegistry,
	ClassLoader userCodeClassLoader,
	Map<String, RocksDbKvStateInfo> kvStateInformation,
	StateSerializerProvider<K> keySerializerProvider,
	File instanceBasePath,
	File instanceRocksDBPath,
	DBOptions dbOptions,
	Function<String, ColumnFamilyOptions> columnFamilyOptionsFactory,
	RocksDBNativeMetricOptions nativeMetricOptions,
	MetricGroup metricGroup,
	@Nonnull Collection<KeyedStateHandle> restoreStateHandles,
	@Nonnull RocksDbTtlCompactFiltersManager ttlCompactFiltersManager,
	@Nonnegative long writeBatchSize) {
	super(
		keyGroupRange,
		keyGroupPrefixBytes,
		numberOfTransferringThreads,
		cancelStreamRegistry,
		userCodeClassLoader,
		kvStateInformation,
		keySerializerProvider,
		instanceBasePath,
		instanceRocksDBPath,
		dbOptions,
		columnFamilyOptionsFactory,
		nativeMetricOptions,
		metricGroup,
		restoreStateHandles,
		ttlCompactFiltersManager);
	checkArgument(writeBatchSize >= 0, "Write batch size have to be no negative.");
	this.writeBatchSize = writeBatchSize;
}
Example #8
Source File: AbstractRocksDBRestoreOperation.java    From flink with Apache License 2.0 5 votes vote down vote up 
protected AbstractRocksDBRestoreOperation(
	KeyGroupRange keyGroupRange,
	int keyGroupPrefixBytes,
	int numberOfTransferringThreads,
	CloseableRegistry cancelStreamRegistry,
	ClassLoader userCodeClassLoader,
	Map<String, RocksDbKvStateInfo> kvStateInformation,
	StateSerializerProvider<K> keySerializerProvider,
	File instanceBasePath,
	File instanceRocksDBPath,
	DBOptions dbOptions,
	Function<String, ColumnFamilyOptions> columnFamilyOptionsFactory,
	RocksDBNativeMetricOptions nativeMetricOptions,
	MetricGroup metricGroup,
	@Nonnull Collection<KeyedStateHandle> stateHandles,
	@Nonnull RocksDbTtlCompactFiltersManager ttlCompactFiltersManager) {
	this.keyGroupRange = keyGroupRange;
	this.keyGroupPrefixBytes = keyGroupPrefixBytes;
	this.numberOfTransferringThreads = numberOfTransferringThreads;
	this.cancelStreamRegistry = cancelStreamRegistry;
	this.userCodeClassLoader = userCodeClassLoader;
	this.kvStateInformation = kvStateInformation;
	this.keySerializerProvider = keySerializerProvider;
	this.instanceBasePath = instanceBasePath;
	this.instanceRocksDBPath = instanceRocksDBPath;
	this.dbPath = instanceRocksDBPath.getAbsolutePath();
	this.dbOptions = dbOptions;
	this.columnFamilyOptionsFactory = columnFamilyOptionsFactory;
	this.nativeMetricOptions = nativeMetricOptions;
	this.metricGroup = metricGroup;
	this.restoreStateHandles = stateHandles;
	this.ttlCompactFiltersManager = ttlCompactFiltersManager;
	this.columnFamilyHandles = new ArrayList<>(1);
	this.columnFamilyDescriptors = Collections.emptyList();
}
Example #9
Source File: RocksDBOperationUtils.java    From flink with Apache License 2.0 5 votes vote down vote up 
/**
 * Creates a state info from a new meta info to use with a k/v state.
 *
 * <p>Creates the column family for the state.
 * Sets TTL compaction filter if {@code ttlCompactFiltersManager} is not {@code null}.
 */
public static RocksDBKeyedStateBackend.RocksDbKvStateInfo createStateInfo(
	RegisteredStateMetaInfoBase metaInfoBase,
	RocksDB db,
	Function<String, ColumnFamilyOptions> columnFamilyOptionsFactory,
	@Nullable RocksDbTtlCompactFiltersManager ttlCompactFiltersManager) {

	ColumnFamilyDescriptor columnFamilyDescriptor = createColumnFamilyDescriptor(
		metaInfoBase, columnFamilyOptionsFactory, ttlCompactFiltersManager);
	return new RocksDBKeyedStateBackend.RocksDbKvStateInfo(createColumnFamily(columnFamilyDescriptor, db), metaInfoBase);
}
Example #10
Source File: RocksDBNoneRestoreOperation.java    From flink with Apache License 2.0 5 votes vote down vote up 
public RocksDBNoneRestoreOperation(
	KeyGroupRange keyGroupRange,
	int keyGroupPrefixBytes,
	int numberOfTransferringThreads,
	CloseableRegistry cancelStreamRegistry,
	ClassLoader userCodeClassLoader,
	Map<String, RocksDbKvStateInfo> kvStateInformation,
	StateSerializerProvider<K> keySerializerProvider,
	File instanceBasePath,
	File instanceRocksDBPath,
	DBOptions dbOptions,
	Function<String, ColumnFamilyOptions> columnFamilyOptionsFactory,
	RocksDBNativeMetricOptions nativeMetricOptions,
	MetricGroup metricGroup,
	@Nonnull Collection<KeyedStateHandle> restoreStateHandles,
	@Nonnull RocksDbTtlCompactFiltersManager ttlCompactFiltersManager
) {
	super(keyGroupRange,
		keyGroupPrefixBytes,
		numberOfTransferringThreads,
		cancelStreamRegistry,
		userCodeClassLoader,
		kvStateInformation,
		keySerializerProvider,
		instanceBasePath,
		instanceRocksDBPath,
		dbOptions,
		columnFamilyOptionsFactory,
		nativeMetricOptions,
		metricGroup,
		restoreStateHandles,
		ttlCompactFiltersManager);
}
Example #11
Source File: RocksDBOperationUtils.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up 
/**
 * Creates a state info from a new meta info to use with a k/v state.
 *
 * <p>Creates the column family for the state.
 * Sets TTL compaction filter if {@code ttlCompactFiltersManager} is not {@code null}.
 */
public static RocksDBKeyedStateBackend.RocksDbKvStateInfo createStateInfo(
	RegisteredStateMetaInfoBase metaInfoBase,
	RocksDB db,
	Function<String, ColumnFamilyOptions> columnFamilyOptionsFactory,
	@Nullable RocksDbTtlCompactFiltersManager ttlCompactFiltersManager) {

	ColumnFamilyDescriptor columnFamilyDescriptor = createColumnFamilyDescriptor(
		metaInfoBase, columnFamilyOptionsFactory, ttlCompactFiltersManager);
	return new RocksDBKeyedStateBackend.RocksDbKvStateInfo(createColumnFamily(columnFamilyDescriptor, db), metaInfoBase);
}
Example #12
Source File: RocksDBFullRestoreOperation.java    From flink with Apache License 2.0 5 votes vote down vote up 
public RocksDBFullRestoreOperation(
	KeyGroupRange keyGroupRange,
	int keyGroupPrefixBytes,
	int numberOfTransferringThreads,
	CloseableRegistry cancelStreamRegistry,
	ClassLoader userCodeClassLoader,
	Map<String, RocksDbKvStateInfo> kvStateInformation,
	StateSerializerProvider<K> keySerializerProvider,
	File instanceBasePath,
	File instanceRocksDBPath,
	DBOptions dbOptions,
	Function<String, ColumnFamilyOptions> columnFamilyOptionsFactory,
	RocksDBNativeMetricOptions nativeMetricOptions,
	MetricGroup metricGroup,
	@Nonnull Collection<KeyedStateHandle> restoreStateHandles,
	@Nonnull RocksDbTtlCompactFiltersManager ttlCompactFiltersManager) {
	super(
		keyGroupRange,
		keyGroupPrefixBytes,
		numberOfTransferringThreads,
		cancelStreamRegistry,
		userCodeClassLoader,
		kvStateInformation,
		keySerializerProvider,
		instanceBasePath,
		instanceRocksDBPath,
		dbOptions,
		columnFamilyOptionsFactory,
		nativeMetricOptions,
		metricGroup,
		restoreStateHandles,
		ttlCompactFiltersManager);
}
Example #13
Source File: AbstractRocksDBRestoreOperation.java    From flink with Apache License 2.0 5 votes vote down vote up 
protected AbstractRocksDBRestoreOperation(
	KeyGroupRange keyGroupRange,
	int keyGroupPrefixBytes,
	int numberOfTransferringThreads,
	CloseableRegistry cancelStreamRegistry,
	ClassLoader userCodeClassLoader,
	Map<String, RocksDbKvStateInfo> kvStateInformation,
	StateSerializerProvider<K> keySerializerProvider,
	File instanceBasePath,
	File instanceRocksDBPath,
	DBOptions dbOptions,
	Function<String, ColumnFamilyOptions> columnFamilyOptionsFactory,
	RocksDBNativeMetricOptions nativeMetricOptions,
	MetricGroup metricGroup,
	@Nonnull Collection<KeyedStateHandle> stateHandles,
	@Nonnull RocksDbTtlCompactFiltersManager ttlCompactFiltersManager) {
	this.keyGroupRange = keyGroupRange;
	this.keyGroupPrefixBytes = keyGroupPrefixBytes;
	this.numberOfTransferringThreads = numberOfTransferringThreads;
	this.cancelStreamRegistry = cancelStreamRegistry;
	this.userCodeClassLoader = userCodeClassLoader;
	this.kvStateInformation = kvStateInformation;
	this.keySerializerProvider = keySerializerProvider;
	this.instanceBasePath = instanceBasePath;
	this.instanceRocksDBPath = instanceRocksDBPath;
	this.dbPath = instanceRocksDBPath.getAbsolutePath();
	this.dbOptions = dbOptions;
	this.columnFamilyOptionsFactory = columnFamilyOptionsFactory;
	this.nativeMetricOptions = nativeMetricOptions;
	this.metricGroup = metricGroup;
	this.restoreStateHandles = stateHandles;
	this.ttlCompactFiltersManager = ttlCompactFiltersManager;
	this.columnFamilyHandles = new ArrayList<>(1);
	this.columnFamilyDescriptors = Collections.emptyList();
}
Example #14
Source File: RocksDBOperationUtils.java    From flink with Apache License 2.0 5 votes vote down vote up 
/**
 * Creates a state info from a new meta info to use with a k/v state.
 *
 * <p>Creates the column family for the state.
 * Sets TTL compaction filter if {@code ttlCompactFiltersManager} is not {@code null}.
 */
public static RocksDBKeyedStateBackend.RocksDbKvStateInfo createStateInfo(
	RegisteredStateMetaInfoBase metaInfoBase,
	RocksDB db,
	Function<String, ColumnFamilyOptions> columnFamilyOptionsFactory,
	@Nullable RocksDbTtlCompactFiltersManager ttlCompactFiltersManager) {

	ColumnFamilyDescriptor columnFamilyDescriptor = createColumnFamilyDescriptor(
		metaInfoBase, columnFamilyOptionsFactory, ttlCompactFiltersManager);
	return new RocksDBKeyedStateBackend.RocksDbKvStateInfo(createColumnFamily(columnFamilyDescriptor, db), metaInfoBase);
}
Example #15
Source File: RocksDBNoneRestoreOperation.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up 
public RocksDBNoneRestoreOperation(
	KeyGroupRange keyGroupRange,
	int keyGroupPrefixBytes,
	int numberOfTransferringThreads,
	CloseableRegistry cancelStreamRegistry,
	ClassLoader userCodeClassLoader,
	Map<String, RocksDbKvStateInfo> kvStateInformation,
	StateSerializerProvider<K> keySerializerProvider,
	File instanceBasePath,
	File instanceRocksDBPath,
	DBOptions dbOptions,
	Function<String, ColumnFamilyOptions> columnFamilyOptionsFactory,
	RocksDBNativeMetricOptions nativeMetricOptions,
	MetricGroup metricGroup,
	@Nonnull Collection<KeyedStateHandle> restoreStateHandles,
	@Nonnull RocksDbTtlCompactFiltersManager ttlCompactFiltersManager
) {
	super(keyGroupRange,
		keyGroupPrefixBytes,
		numberOfTransferringThreads,
		cancelStreamRegistry,
		userCodeClassLoader,
		kvStateInformation,
		keySerializerProvider,
		instanceBasePath,
		instanceRocksDBPath,
		dbOptions,
		columnFamilyOptionsFactory,
		nativeMetricOptions,
		metricGroup,
		restoreStateHandles,
		ttlCompactFiltersManager);
}
Example #16
Source File: RocksDBIncrementalRestoreOperation.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up 
public RocksDBIncrementalRestoreOperation(
	String operatorIdentifier,
	KeyGroupRange keyGroupRange,
	int keyGroupPrefixBytes,
	int numberOfTransferringThreads,
	CloseableRegistry cancelStreamRegistry,
	ClassLoader userCodeClassLoader,
	Map<String, RocksDbKvStateInfo> kvStateInformation,
	StateSerializerProvider<K> keySerializerProvider,
	File instanceBasePath,
	File instanceRocksDBPath,
	DBOptions dbOptions,
	Function<String, ColumnFamilyOptions> columnFamilyOptionsFactory,
	RocksDBNativeMetricOptions nativeMetricOptions,
	MetricGroup metricGroup,
	@Nonnull Collection<KeyedStateHandle> restoreStateHandles,
	@Nonnull RocksDbTtlCompactFiltersManager ttlCompactFiltersManager) {
	super(keyGroupRange,
		keyGroupPrefixBytes,
		numberOfTransferringThreads,
		cancelStreamRegistry,
		userCodeClassLoader,
		kvStateInformation,
		keySerializerProvider,
		instanceBasePath,
		instanceRocksDBPath,
		dbOptions,
		columnFamilyOptionsFactory,
		nativeMetricOptions,
		metricGroup,
		restoreStateHandles,
		ttlCompactFiltersManager);
	this.operatorIdentifier = operatorIdentifier;
	this.restoredSstFiles = new TreeMap<>();
	this.lastCompletedCheckpointId = -1L;
	this.backendUID = UUID.randomUUID();
}
Example #17
Source File: RocksDBFullRestoreOperation.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up 
public RocksDBFullRestoreOperation(
	KeyGroupRange keyGroupRange,
	int keyGroupPrefixBytes,
	int numberOfTransferringThreads,
	CloseableRegistry cancelStreamRegistry,
	ClassLoader userCodeClassLoader,
	Map<String, RocksDbKvStateInfo> kvStateInformation,
	StateSerializerProvider<K> keySerializerProvider,
	File instanceBasePath,
	File instanceRocksDBPath,
	DBOptions dbOptions,
	Function<String, ColumnFamilyOptions> columnFamilyOptionsFactory,
	RocksDBNativeMetricOptions nativeMetricOptions,
	MetricGroup metricGroup,
	@Nonnull Collection<KeyedStateHandle> restoreStateHandles,
	@Nonnull RocksDbTtlCompactFiltersManager ttlCompactFiltersManager) {
	super(
		keyGroupRange,
		keyGroupPrefixBytes,
		numberOfTransferringThreads,
		cancelStreamRegistry,
		userCodeClassLoader,
		kvStateInformation,
		keySerializerProvider,
		instanceBasePath,
		instanceRocksDBPath,
		dbOptions,
		columnFamilyOptionsFactory,
		nativeMetricOptions,
		metricGroup,
		restoreStateHandles,
		ttlCompactFiltersManager);
}
Example #18
Source File: AbstractRocksDBRestoreOperation.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up 
protected AbstractRocksDBRestoreOperation(
	KeyGroupRange keyGroupRange,
	int keyGroupPrefixBytes,
	int numberOfTransferringThreads,
	CloseableRegistry cancelStreamRegistry,
	ClassLoader userCodeClassLoader,
	Map<String, RocksDbKvStateInfo> kvStateInformation,
	StateSerializerProvider<K> keySerializerProvider,
	File instanceBasePath,
	File instanceRocksDBPath,
	DBOptions dbOptions,
	Function<String, ColumnFamilyOptions> columnFamilyOptionsFactory,
	RocksDBNativeMetricOptions nativeMetricOptions,
	MetricGroup metricGroup,
	@Nonnull Collection<KeyedStateHandle> stateHandles,
	@Nonnull RocksDbTtlCompactFiltersManager ttlCompactFiltersManager) {
	this.keyGroupRange = keyGroupRange;
	this.keyGroupPrefixBytes = keyGroupPrefixBytes;
	this.numberOfTransferringThreads = numberOfTransferringThreads;
	this.cancelStreamRegistry = cancelStreamRegistry;
	this.userCodeClassLoader = userCodeClassLoader;
	this.kvStateInformation = kvStateInformation;
	this.keySerializerProvider = keySerializerProvider;
	this.instanceBasePath = instanceBasePath;
	this.instanceRocksDBPath = instanceRocksDBPath;
	this.dbPath = instanceRocksDBPath.getAbsolutePath();
	this.dbOptions = dbOptions;
	this.columnFamilyOptionsFactory = columnFamilyOptionsFactory;
	this.nativeMetricOptions = nativeMetricOptions;
	this.metricGroup = metricGroup;
	this.restoreStateHandles = stateHandles;
	this.ttlCompactFiltersManager = ttlCompactFiltersManager;
	this.columnFamilyHandles = new ArrayList<>(1);
	this.columnFamilyDescriptors = Collections.emptyList();
}
Example #19
Source File: RocksDBKeyedStateBackend.java    From flink with Apache License 2.0 4 votes vote down vote up 
public RocksDBKeyedStateBackend(
	ClassLoader userCodeClassLoader,
	File instanceBasePath,
	DBOptions dbOptions,
	Function<String, ColumnFamilyOptions> columnFamilyOptionsFactory,
	TaskKvStateRegistry kvStateRegistry,
	TypeSerializer<K> keySerializer,
	ExecutionConfig executionConfig,
	TtlTimeProvider ttlTimeProvider,
	RocksDB db,
	LinkedHashMap<String, RocksDbKvStateInfo> kvStateInformation,
	int keyGroupPrefixBytes,
	CloseableRegistry cancelStreamRegistry,
	StreamCompressionDecorator keyGroupCompressionDecorator,
	ResourceGuard rocksDBResourceGuard,
	RocksDBSnapshotStrategyBase<K> checkpointSnapshotStrategy,
	RocksDBSnapshotStrategyBase<K> savepointSnapshotStrategy,
	RocksDBWriteBatchWrapper writeBatchWrapper,
	ColumnFamilyHandle defaultColumnFamilyHandle,
	RocksDBNativeMetricMonitor nativeMetricMonitor,
	RocksDBSerializedCompositeKeyBuilder<K> sharedRocksKeyBuilder,
	PriorityQueueSetFactory priorityQueueFactory,
	RocksDbTtlCompactFiltersManager ttlCompactFiltersManager,
	InternalKeyContext<K> keyContext) {

	super(
		kvStateRegistry,
		keySerializer,
		userCodeClassLoader,
		executionConfig,
		ttlTimeProvider,
		cancelStreamRegistry,
		keyGroupCompressionDecorator,
		keyContext);

	this.ttlCompactFiltersManager = ttlCompactFiltersManager;

	// ensure that we use the right merge operator, because other code relies on this
	this.columnFamilyOptionsFactory = Preconditions.checkNotNull(columnFamilyOptionsFactory);

	this.dbOptions = Preconditions.checkNotNull(dbOptions);

	this.instanceBasePath = Preconditions.checkNotNull(instanceBasePath);

	this.keyGroupPrefixBytes = keyGroupPrefixBytes;
	this.kvStateInformation = kvStateInformation;

	this.writeOptions = new WriteOptions().setDisableWAL(true);
	this.db = db;
	this.rocksDBResourceGuard = rocksDBResourceGuard;
	this.checkpointSnapshotStrategy = checkpointSnapshotStrategy;
	this.savepointSnapshotStrategy = savepointSnapshotStrategy;
	this.writeBatchWrapper = writeBatchWrapper;
	this.defaultColumnFamily = defaultColumnFamilyHandle;
	this.nativeMetricMonitor = nativeMetricMonitor;
	this.sharedRocksKeyBuilder = sharedRocksKeyBuilder;
	this.priorityQueueFactory = priorityQueueFactory;
}
Example #20
Source File: RocksDBKeyedStateBackendBuilder.java    From flink with Apache License 2.0 4 votes vote down vote up 
private AbstractRocksDBRestoreOperation<K> getRocksDBRestoreOperation(
	int keyGroupPrefixBytes,
	CloseableRegistry cancelStreamRegistry,
	LinkedHashMap<String, RocksDBKeyedStateBackend.RocksDbKvStateInfo> kvStateInformation,
	RocksDbTtlCompactFiltersManager ttlCompactFiltersManager) {
	if (restoreStateHandles.isEmpty()) {
		return new RocksDBNoneRestoreOperation<>(
			keyGroupRange,
			keyGroupPrefixBytes,
			numberOfTransferingThreads,
			cancelStreamRegistry,
			userCodeClassLoader,
			kvStateInformation,
			keySerializerProvider,
			instanceBasePath,
			instanceRocksDBPath,
			dbOptions,
			columnFamilyOptionsFactory,
			nativeMetricOptions,
			metricGroup,
			restoreStateHandles,
			ttlCompactFiltersManager);
	}
	KeyedStateHandle firstStateHandle = restoreStateHandles.iterator().next();
	if (firstStateHandle instanceof IncrementalKeyedStateHandle) {
		return new RocksDBIncrementalRestoreOperation<>(
			operatorIdentifier,
			keyGroupRange,
			keyGroupPrefixBytes,
			numberOfTransferingThreads,
			cancelStreamRegistry,
			userCodeClassLoader,
			kvStateInformation,
			keySerializerProvider,
			instanceBasePath,
			instanceRocksDBPath,
			dbOptions,
			columnFamilyOptionsFactory,
			nativeMetricOptions,
			metricGroup,
			restoreStateHandles,
			ttlCompactFiltersManager);
	} else {
		return new RocksDBFullRestoreOperation<>(
			keyGroupRange,
			keyGroupPrefixBytes,
			numberOfTransferingThreads,
			cancelStreamRegistry,
			userCodeClassLoader,
			kvStateInformation,
			keySerializerProvider,
			instanceBasePath,
			instanceRocksDBPath,
			dbOptions,
			columnFamilyOptionsFactory,
			nativeMetricOptions,
			metricGroup,
			restoreStateHandles,
			ttlCompactFiltersManager);
	}
}
Example #21
Source File: RocksDBKeyedStateBackendBuilder.java    From Flink-CEPplus with Apache License 2.0 4 votes vote down vote up 
private AbstractRocksDBRestoreOperation<K> getRocksDBRestoreOperation(
	int keyGroupPrefixBytes,
	CloseableRegistry cancelStreamRegistry,
	LinkedHashMap<String, RocksDBKeyedStateBackend.RocksDbKvStateInfo> kvStateInformation,
	RocksDbTtlCompactFiltersManager ttlCompactFiltersManager) {
	if (restoreStateHandles.isEmpty()) {
		return new RocksDBNoneRestoreOperation<>(
			keyGroupRange,
			keyGroupPrefixBytes,
			numberOfTransferingThreads,
			cancelStreamRegistry,
			userCodeClassLoader,
			kvStateInformation,
			keySerializerProvider,
			instanceBasePath,
			instanceRocksDBPath,
			dbOptions,
			columnFamilyOptionsFactory,
			nativeMetricOptions,
			metricGroup,
			restoreStateHandles,
			ttlCompactFiltersManager);
	}
	KeyedStateHandle firstStateHandle = restoreStateHandles.iterator().next();
	if (firstStateHandle instanceof IncrementalKeyedStateHandle) {
		return new RocksDBIncrementalRestoreOperation<>(
			operatorIdentifier,
			keyGroupRange,
			keyGroupPrefixBytes,
			numberOfTransferingThreads,
			cancelStreamRegistry,
			userCodeClassLoader,
			kvStateInformation,
			keySerializerProvider,
			instanceBasePath,
			instanceRocksDBPath,
			dbOptions,
			columnFamilyOptionsFactory,
			nativeMetricOptions,
			metricGroup,
			restoreStateHandles,
			ttlCompactFiltersManager);
	} else {
		return new RocksDBFullRestoreOperation<>(
			keyGroupRange,
			keyGroupPrefixBytes,
			numberOfTransferingThreads,
			cancelStreamRegistry,
			userCodeClassLoader,
			kvStateInformation,
			keySerializerProvider,
			instanceBasePath,
			instanceRocksDBPath,
			dbOptions,
			columnFamilyOptionsFactory,
			nativeMetricOptions,
			metricGroup,
			restoreStateHandles,
			ttlCompactFiltersManager);
	}
}
Example #22
Source File: RocksDBKeyedStateBackend.java    From Flink-CEPplus with Apache License 2.0 4 votes vote down vote up 
public RocksDBKeyedStateBackend(
	ClassLoader userCodeClassLoader,
	File instanceBasePath,
	DBOptions dbOptions,
	Function<String, ColumnFamilyOptions> columnFamilyOptionsFactory,
	TaskKvStateRegistry kvStateRegistry,
	StateSerializerProvider<K> keySerializerProvider,
	int numberOfKeyGroups,
	KeyGroupRange keyGroupRange,
	ExecutionConfig executionConfig,
	TtlTimeProvider ttlTimeProvider,
	RocksDB db,
	LinkedHashMap<String, RocksDbKvStateInfo> kvStateInformation,
	int keyGroupPrefixBytes,
	CloseableRegistry cancelStreamRegistry,
	StreamCompressionDecorator keyGroupCompressionDecorator,
	ResourceGuard rocksDBResourceGuard,
	RocksDBSnapshotStrategyBase<K> checkpointSnapshotStrategy,
	RocksDBSnapshotStrategyBase<K> savepointSnapshotStrategy,
	RocksDBWriteBatchWrapper writeBatchWrapper,
	ColumnFamilyHandle defaultColumnFamilyHandle,
	RocksDBNativeMetricMonitor nativeMetricMonitor,
	RocksDBSerializedCompositeKeyBuilder<K> sharedRocksKeyBuilder,
	PriorityQueueSetFactory priorityQueueFactory,
	RocksDbTtlCompactFiltersManager ttlCompactFiltersManager) {

	super(kvStateRegistry, keySerializerProvider, userCodeClassLoader, numberOfKeyGroups,
		keyGroupRange, executionConfig, ttlTimeProvider, cancelStreamRegistry, keyGroupCompressionDecorator);

	this.ttlCompactFiltersManager = ttlCompactFiltersManager;

	// ensure that we use the right merge operator, because other code relies on this
	this.columnFamilyOptionsFactory = Preconditions.checkNotNull(columnFamilyOptionsFactory);

	this.dbOptions = Preconditions.checkNotNull(dbOptions);

	this.instanceBasePath = Preconditions.checkNotNull(instanceBasePath);

	this.keyGroupPrefixBytes = keyGroupPrefixBytes;
	this.kvStateInformation = kvStateInformation;

	this.writeOptions = new WriteOptions().setDisableWAL(true);
	this.db = db;
	this.rocksDBResourceGuard = rocksDBResourceGuard;
	this.checkpointSnapshotStrategy = checkpointSnapshotStrategy;
	this.savepointSnapshotStrategy = savepointSnapshotStrategy;
	this.writeBatchWrapper = writeBatchWrapper;
	this.defaultColumnFamily = defaultColumnFamilyHandle;
	this.nativeMetricMonitor = nativeMetricMonitor;
	this.sharedRocksKeyBuilder = sharedRocksKeyBuilder;
	this.priorityQueueFactory = priorityQueueFactory;
}
Example #23
Source File: RocksDBKeyedStateBackend.java    From flink with Apache License 2.0 4 votes vote down vote up 
public RocksDBKeyedStateBackend(
	ClassLoader userCodeClassLoader,
	File instanceBasePath,
	RocksDBResourceContainer optionsContainer,
	Function<String, ColumnFamilyOptions> columnFamilyOptionsFactory,
	TaskKvStateRegistry kvStateRegistry,
	TypeSerializer<K> keySerializer,
	ExecutionConfig executionConfig,
	TtlTimeProvider ttlTimeProvider,
	RocksDB db,
	LinkedHashMap<String, RocksDbKvStateInfo> kvStateInformation,
	int keyGroupPrefixBytes,
	CloseableRegistry cancelStreamRegistry,
	StreamCompressionDecorator keyGroupCompressionDecorator,
	ResourceGuard rocksDBResourceGuard,
	RocksDBSnapshotStrategyBase<K> checkpointSnapshotStrategy,
	RocksDBSnapshotStrategyBase<K> savepointSnapshotStrategy,
	RocksDBWriteBatchWrapper writeBatchWrapper,
	ColumnFamilyHandle defaultColumnFamilyHandle,
	RocksDBNativeMetricMonitor nativeMetricMonitor,
	RocksDBSerializedCompositeKeyBuilder<K> sharedRocksKeyBuilder,
	PriorityQueueSetFactory priorityQueueFactory,
	RocksDbTtlCompactFiltersManager ttlCompactFiltersManager,
	InternalKeyContext<K> keyContext,
	@Nonnegative long writeBatchSize) {

	super(
		kvStateRegistry,
		keySerializer,
		userCodeClassLoader,
		executionConfig,
		ttlTimeProvider,
		cancelStreamRegistry,
		keyGroupCompressionDecorator,
		keyContext);

	this.ttlCompactFiltersManager = ttlCompactFiltersManager;

	// ensure that we use the right merge operator, because other code relies on this
	this.columnFamilyOptionsFactory = Preconditions.checkNotNull(columnFamilyOptionsFactory);

	this.optionsContainer = Preconditions.checkNotNull(optionsContainer);

	this.instanceBasePath = Preconditions.checkNotNull(instanceBasePath);

	this.keyGroupPrefixBytes = keyGroupPrefixBytes;
	this.kvStateInformation = kvStateInformation;

	this.writeOptions = optionsContainer.getWriteOptions();
	this.readOptions = optionsContainer.getReadOptions();
	checkArgument(writeBatchSize >= 0, "Write batch size have to be no negative value.");
	this.writeBatchSize = writeBatchSize;
	this.db = db;
	this.rocksDBResourceGuard = rocksDBResourceGuard;
	this.checkpointSnapshotStrategy = checkpointSnapshotStrategy;
	this.savepointSnapshotStrategy = savepointSnapshotStrategy;
	this.writeBatchWrapper = writeBatchWrapper;
	this.defaultColumnFamily = defaultColumnFamilyHandle;
	this.nativeMetricMonitor = nativeMetricMonitor;
	this.sharedRocksKeyBuilder = sharedRocksKeyBuilder;
	this.priorityQueueFactory = priorityQueueFactory;
}
Example #24
Source File: RocksDBKeyedStateBackendBuilder.java    From flink with Apache License 2.0 4 votes vote down vote up 
private AbstractRocksDBRestoreOperation<K> getRocksDBRestoreOperation(
	int keyGroupPrefixBytes,
	CloseableRegistry cancelStreamRegistry,
	LinkedHashMap<String, RocksDBKeyedStateBackend.RocksDbKvStateInfo> kvStateInformation,
	RocksDbTtlCompactFiltersManager ttlCompactFiltersManager) {
	DBOptions dbOptions = optionsContainer.getDbOptions();
	if (restoreStateHandles.isEmpty()) {
		return new RocksDBNoneRestoreOperation<>(
			keyGroupRange,
			keyGroupPrefixBytes,
			numberOfTransferingThreads,
			cancelStreamRegistry,
			userCodeClassLoader,
			kvStateInformation,
			keySerializerProvider,
			instanceBasePath,
			instanceRocksDBPath,
			dbOptions,
			columnFamilyOptionsFactory,
			nativeMetricOptions,
			metricGroup,
			restoreStateHandles,
			ttlCompactFiltersManager);
	}
	KeyedStateHandle firstStateHandle = restoreStateHandles.iterator().next();
	if (firstStateHandle instanceof IncrementalKeyedStateHandle) {
		return new RocksDBIncrementalRestoreOperation<>(
			operatorIdentifier,
			keyGroupRange,
			keyGroupPrefixBytes,
			numberOfTransferingThreads,
			cancelStreamRegistry,
			userCodeClassLoader,
			kvStateInformation,
			keySerializerProvider,
			instanceBasePath,
			instanceRocksDBPath,
			dbOptions,
			columnFamilyOptionsFactory,
			nativeMetricOptions,
			metricGroup,
			restoreStateHandles,
			ttlCompactFiltersManager,
			writeBatchSize);
	} else {
		return new RocksDBFullRestoreOperation<>(
			keyGroupRange,
			keyGroupPrefixBytes,
			numberOfTransferingThreads,
			cancelStreamRegistry,
			userCodeClassLoader,
			kvStateInformation,
			keySerializerProvider,
			instanceBasePath,
			instanceRocksDBPath,
			dbOptions,
			columnFamilyOptionsFactory,
			nativeMetricOptions,
			metricGroup,
			restoreStateHandles,
			ttlCompactFiltersManager,
			writeBatchSize);
	}
}