org.apache.flink.contrib.streaming.state.RocksDBKeyedStateBackend.RocksDbKvStateInfo Java Examples

The following examples show how to use org.apache.flink.contrib.streaming.state.RocksDBKeyedStateBackend.RocksDbKvStateInfo. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: RocksDBFullRestoreOperation.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
/**
 * Restore the KV-state / ColumnFamily meta data for all key-groups referenced by the current state handle.
 */
private void restoreKVStateMetaData() throws IOException, StateMigrationException {
	KeyedBackendSerializationProxy<K> serializationProxy = readMetaData(currentStateHandleInView);

	this.keygroupStreamCompressionDecorator = serializationProxy.isUsingKeyGroupCompression() ?
		SnappyStreamCompressionDecorator.INSTANCE : UncompressedStreamCompressionDecorator.INSTANCE;

	List<StateMetaInfoSnapshot> restoredMetaInfos =
		serializationProxy.getStateMetaInfoSnapshots();
	currentStateHandleKVStateColumnFamilies = new ArrayList<>(restoredMetaInfos.size());

	for (StateMetaInfoSnapshot restoredMetaInfo : restoredMetaInfos) {
		RocksDbKvStateInfo registeredStateCFHandle =
			getOrRegisterStateColumnFamilyHandle(null, restoredMetaInfo);
		currentStateHandleKVStateColumnFamilies.add(registeredStateCFHandle.columnFamilyHandle);
	}
}
 
Example #2
Source File: RocksFullSnapshotStrategy.java    From flink with Apache License 2.0 6 votes vote down vote up
public RocksFullSnapshotStrategy(
	@Nonnull RocksDB db,
	@Nonnull ResourceGuard rocksDBResourceGuard,
	@Nonnull TypeSerializer<K> keySerializer,
	@Nonnull LinkedHashMap<String, RocksDbKvStateInfo> kvStateInformation,
	@Nonnull KeyGroupRange keyGroupRange,
	@Nonnegative int keyGroupPrefixBytes,
	@Nonnull LocalRecoveryConfig localRecoveryConfig,
	@Nonnull CloseableRegistry cancelStreamRegistry,
	@Nonnull StreamCompressionDecorator keyGroupCompressionDecorator) {
	super(
		DESCRIPTION,
		db,
		rocksDBResourceGuard,
		keySerializer,
		kvStateInformation,
		keyGroupRange,
		keyGroupPrefixBytes,
		localRecoveryConfig,
		cancelStreamRegistry);

	this.keyGroupCompressionDecorator = keyGroupCompressionDecorator;
}
 
Example #3
Source File: RocksDBSnapshotStrategyBase.java    From flink with Apache License 2.0 6 votes vote down vote up
public RocksDBSnapshotStrategyBase(
	@Nonnull String description,
	@Nonnull RocksDB db,
	@Nonnull ResourceGuard rocksDBResourceGuard,
	@Nonnull TypeSerializer<K> keySerializer,
	@Nonnull LinkedHashMap<String, RocksDbKvStateInfo> kvStateInformation,
	@Nonnull KeyGroupRange keyGroupRange,
	@Nonnegative int keyGroupPrefixBytes,
	@Nonnull LocalRecoveryConfig localRecoveryConfig,
	@Nonnull CloseableRegistry cancelStreamRegistry) {

	super(description);
	this.db = db;
	this.rocksDBResourceGuard = rocksDBResourceGuard;
	this.keySerializer = keySerializer;
	this.kvStateInformation = kvStateInformation;
	this.keyGroupRange = keyGroupRange;
	this.keyGroupPrefixBytes = keyGroupPrefixBytes;
	this.localRecoveryConfig = localRecoveryConfig;
	this.cancelStreamRegistry = cancelStreamRegistry;
}
 
Example #4
Source File: RocksFullSnapshotStrategy.java    From flink with Apache License 2.0 6 votes vote down vote up
public RocksFullSnapshotStrategy(
	@Nonnull RocksDB db,
	@Nonnull ResourceGuard rocksDBResourceGuard,
	@Nonnull TypeSerializer<K> keySerializer,
	@Nonnull LinkedHashMap<String, RocksDbKvStateInfo> kvStateInformation,
	@Nonnull KeyGroupRange keyGroupRange,
	@Nonnegative int keyGroupPrefixBytes,
	@Nonnull LocalRecoveryConfig localRecoveryConfig,
	@Nonnull CloseableRegistry cancelStreamRegistry,
	@Nonnull StreamCompressionDecorator keyGroupCompressionDecorator) {
	super(
		DESCRIPTION,
		db,
		rocksDBResourceGuard,
		keySerializer,
		kvStateInformation,
		keyGroupRange,
		keyGroupPrefixBytes,
		localRecoveryConfig,
		cancelStreamRegistry);

	this.keyGroupCompressionDecorator = keyGroupCompressionDecorator;
}
 
Example #5
Source File: RocksIncrementalSnapshotStrategy.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
private Set<StateHandleID> snapshotMetaData(
	long checkpointId,
	@Nonnull List<StateMetaInfoSnapshot> stateMetaInfoSnapshots) {

	final long lastCompletedCheckpoint;
	final Set<StateHandleID> baseSstFiles;

	// use the last completed checkpoint as the comparison base.
	synchronized (materializedSstFiles) {
		lastCompletedCheckpoint = lastCompletedCheckpointId;
		baseSstFiles = materializedSstFiles.get(lastCompletedCheckpoint);
	}
	LOG.trace("Taking incremental snapshot for checkpoint {}. Snapshot is based on last completed checkpoint {} " +
		"assuming the following (shared) files as base: {}.", checkpointId, lastCompletedCheckpoint, baseSstFiles);

	// snapshot meta data to save
	for (Map.Entry<String, RocksDbKvStateInfo> stateMetaInfoEntry : kvStateInformation.entrySet()) {
		stateMetaInfoSnapshots.add(stateMetaInfoEntry.getValue().metaInfo.snapshot());
	}
	return baseSstFiles;
}
 
Example #6
Source File: RocksDBSnapshotStrategyBase.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
public RocksDBSnapshotStrategyBase(
	@Nonnull String description,
	@Nonnull RocksDB db,
	@Nonnull ResourceGuard rocksDBResourceGuard,
	@Nonnull TypeSerializer<K> keySerializer,
	@Nonnull LinkedHashMap<String, RocksDbKvStateInfo> kvStateInformation,
	@Nonnull KeyGroupRange keyGroupRange,
	@Nonnegative int keyGroupPrefixBytes,
	@Nonnull LocalRecoveryConfig localRecoveryConfig,
	@Nonnull CloseableRegistry cancelStreamRegistry) {

	super(description);
	this.db = db;
	this.rocksDBResourceGuard = rocksDBResourceGuard;
	this.keySerializer = keySerializer;
	this.kvStateInformation = kvStateInformation;
	this.keyGroupRange = keyGroupRange;
	this.keyGroupPrefixBytes = keyGroupPrefixBytes;
	this.localRecoveryConfig = localRecoveryConfig;
	this.cancelStreamRegistry = cancelStreamRegistry;
}
 
Example #7
Source File: RocksFullSnapshotStrategy.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
public RocksFullSnapshotStrategy(
	@Nonnull RocksDB db,
	@Nonnull ResourceGuard rocksDBResourceGuard,
	@Nonnull TypeSerializer<K> keySerializer,
	@Nonnull LinkedHashMap<String, RocksDbKvStateInfo> kvStateInformation,
	@Nonnull KeyGroupRange keyGroupRange,
	@Nonnegative int keyGroupPrefixBytes,
	@Nonnull LocalRecoveryConfig localRecoveryConfig,
	@Nonnull CloseableRegistry cancelStreamRegistry,
	@Nonnull StreamCompressionDecorator keyGroupCompressionDecorator) {
	super(
		DESCRIPTION,
		db,
		rocksDBResourceGuard,
		keySerializer,
		kvStateInformation,
		keyGroupRange,
		keyGroupPrefixBytes,
		localRecoveryConfig,
		cancelStreamRegistry);

	this.keyGroupCompressionDecorator = keyGroupCompressionDecorator;
}
 
Example #8
Source File: RocksDBSnapshotStrategyBase.java    From flink with Apache License 2.0 6 votes vote down vote up
public RocksDBSnapshotStrategyBase(
	@Nonnull String description,
	@Nonnull RocksDB db,
	@Nonnull ResourceGuard rocksDBResourceGuard,
	@Nonnull TypeSerializer<K> keySerializer,
	@Nonnull LinkedHashMap<String, RocksDbKvStateInfo> kvStateInformation,
	@Nonnull KeyGroupRange keyGroupRange,
	@Nonnegative int keyGroupPrefixBytes,
	@Nonnull LocalRecoveryConfig localRecoveryConfig,
	@Nonnull CloseableRegistry cancelStreamRegistry) {

	super(description);
	this.db = db;
	this.rocksDBResourceGuard = rocksDBResourceGuard;
	this.keySerializer = keySerializer;
	this.kvStateInformation = kvStateInformation;
	this.keyGroupRange = keyGroupRange;
	this.keyGroupPrefixBytes = keyGroupPrefixBytes;
	this.localRecoveryConfig = localRecoveryConfig;
	this.cancelStreamRegistry = cancelStreamRegistry;
}
 
Example #9
Source File: RocksIncrementalSnapshotStrategy.java    From flink with Apache License 2.0 6 votes vote down vote up
private Set<StateHandleID> snapshotMetaData(
	long checkpointId,
	@Nonnull List<StateMetaInfoSnapshot> stateMetaInfoSnapshots) {

	final long lastCompletedCheckpoint;
	final Set<StateHandleID> baseSstFiles;

	// use the last completed checkpoint as the comparison base.
	synchronized (materializedSstFiles) {
		lastCompletedCheckpoint = lastCompletedCheckpointId;
		baseSstFiles = materializedSstFiles.get(lastCompletedCheckpoint);
	}
	LOG.trace("Taking incremental snapshot for checkpoint {}. Snapshot is based on last completed checkpoint {} " +
		"assuming the following (shared) files as base: {}.", checkpointId, lastCompletedCheckpoint, baseSstFiles);

	// snapshot meta data to save
	for (Map.Entry<String, RocksDbKvStateInfo> stateMetaInfoEntry : kvStateInformation.entrySet()) {
		stateMetaInfoSnapshots.add(stateMetaInfoEntry.getValue().metaInfo.snapshot());
	}
	return baseSstFiles;
}
 
Example #10
Source File: RocksIncrementalSnapshotStrategy.java    From flink with Apache License 2.0 6 votes vote down vote up
private Set<StateHandleID> snapshotMetaData(
	long checkpointId,
	@Nonnull List<StateMetaInfoSnapshot> stateMetaInfoSnapshots) {

	final long lastCompletedCheckpoint;
	final Set<StateHandleID> baseSstFiles;

	// use the last completed checkpoint as the comparison base.
	synchronized (materializedSstFiles) {
		lastCompletedCheckpoint = lastCompletedCheckpointId;
		baseSstFiles = materializedSstFiles.get(lastCompletedCheckpoint);
	}
	LOG.trace("Taking incremental snapshot for checkpoint {}. Snapshot is based on last completed checkpoint {} " +
		"assuming the following (shared) files as base: {}.", checkpointId, lastCompletedCheckpoint, baseSstFiles);

	// snapshot meta data to save
	for (Map.Entry<String, RocksDbKvStateInfo> stateMetaInfoEntry : kvStateInformation.entrySet()) {
		stateMetaInfoSnapshots.add(stateMetaInfoEntry.getValue().metaInfo.snapshot());
	}
	return baseSstFiles;
}
 
Example #11
Source File: RocksDBFullRestoreOperation.java    From flink with Apache License 2.0 6 votes vote down vote up
/**
 * Restore the KV-state / ColumnFamily meta data for all key-groups referenced by the current state handle.
 */
private void restoreKVStateMetaData() throws IOException, StateMigrationException {
	KeyedBackendSerializationProxy<K> serializationProxy = readMetaData(currentStateHandleInView);

	this.keygroupStreamCompressionDecorator = serializationProxy.isUsingKeyGroupCompression() ?
		SnappyStreamCompressionDecorator.INSTANCE : UncompressedStreamCompressionDecorator.INSTANCE;

	List<StateMetaInfoSnapshot> restoredMetaInfos =
		serializationProxy.getStateMetaInfoSnapshots();
	currentStateHandleKVStateColumnFamilies = new ArrayList<>(restoredMetaInfos.size());

	for (StateMetaInfoSnapshot restoredMetaInfo : restoredMetaInfos) {
		RocksDbKvStateInfo registeredStateCFHandle =
			getOrRegisterStateColumnFamilyHandle(null, restoredMetaInfo);
		currentStateHandleKVStateColumnFamilies.add(registeredStateCFHandle.columnFamilyHandle);
	}
}
 
Example #12
Source File: RocksDBFullRestoreOperation.java    From flink with Apache License 2.0 6 votes vote down vote up
/**
 * Restore the KV-state / ColumnFamily meta data for all key-groups referenced by the current state handle.
 */
private void restoreKVStateMetaData() throws IOException, StateMigrationException {
	KeyedBackendSerializationProxy<K> serializationProxy = readMetaData(currentStateHandleInView);

	this.keygroupStreamCompressionDecorator = serializationProxy.isUsingKeyGroupCompression() ?
		SnappyStreamCompressionDecorator.INSTANCE : UncompressedStreamCompressionDecorator.INSTANCE;

	List<StateMetaInfoSnapshot> restoredMetaInfos =
		serializationProxy.getStateMetaInfoSnapshots();
	currentStateHandleKVStateColumnFamilies = new ArrayList<>(restoredMetaInfos.size());

	for (StateMetaInfoSnapshot restoredMetaInfo : restoredMetaInfos) {
		RocksDbKvStateInfo registeredStateCFHandle =
			getOrRegisterStateColumnFamilyHandle(null, restoredMetaInfo);
		currentStateHandleKVStateColumnFamilies.add(registeredStateCFHandle.columnFamilyHandle);
	}
}
 
Example #13
Source File: RocksIncrementalSnapshotStrategy.java    From flink with Apache License 2.0 5 votes vote down vote up
public RocksIncrementalSnapshotStrategy(
	@Nonnull RocksDB db,
	@Nonnull ResourceGuard rocksDBResourceGuard,
	@Nonnull TypeSerializer<K> keySerializer,
	@Nonnull LinkedHashMap<String, RocksDbKvStateInfo> kvStateInformation,
	@Nonnull KeyGroupRange keyGroupRange,
	@Nonnegative int keyGroupPrefixBytes,
	@Nonnull LocalRecoveryConfig localRecoveryConfig,
	@Nonnull CloseableRegistry cancelStreamRegistry,
	@Nonnull File instanceBasePath,
	@Nonnull UUID backendUID,
	@Nonnull SortedMap<Long, Set<StateHandleID>> materializedSstFiles,
	long lastCompletedCheckpointId,
	int numberOfTransferingThreads) {

	super(
		DESCRIPTION,
		db,
		rocksDBResourceGuard,
		keySerializer,
		kvStateInformation,
		keyGroupRange,
		keyGroupPrefixBytes,
		localRecoveryConfig,
		cancelStreamRegistry);

	this.instanceBasePath = instanceBasePath;
	this.backendUID = backendUID;
	this.materializedSstFiles = materializedSstFiles;
	this.lastCompletedCheckpointId = lastCompletedCheckpointId;
	this.stateUploader = new RocksDBStateUploader(numberOfTransferingThreads);
	this.localDirectoryName = backendUID.toString().replaceAll("[\\-]", "");
}
 
Example #14
Source File: RocksFullSnapshotStrategy.java    From flink with Apache License 2.0 5 votes vote down vote up
private MetaData(
	RocksDbKvStateInfo rocksDbKvStateInfo,
	StateSnapshotTransformer<byte[]> stateSnapshotTransformer) {

	this.rocksDbKvStateInfo = rocksDbKvStateInfo;
	this.stateSnapshotTransformer = stateSnapshotTransformer;
}
 
Example #15
Source File: AbstractRocksDBRestoreOperation.java    From flink with Apache License 2.0 5 votes vote down vote up
protected AbstractRocksDBRestoreOperation(
	KeyGroupRange keyGroupRange,
	int keyGroupPrefixBytes,
	int numberOfTransferringThreads,
	CloseableRegistry cancelStreamRegistry,
	ClassLoader userCodeClassLoader,
	Map<String, RocksDbKvStateInfo> kvStateInformation,
	StateSerializerProvider<K> keySerializerProvider,
	File instanceBasePath,
	File instanceRocksDBPath,
	DBOptions dbOptions,
	Function<String, ColumnFamilyOptions> columnFamilyOptionsFactory,
	RocksDBNativeMetricOptions nativeMetricOptions,
	MetricGroup metricGroup,
	@Nonnull Collection<KeyedStateHandle> stateHandles,
	@Nonnull RocksDbTtlCompactFiltersManager ttlCompactFiltersManager) {
	this.keyGroupRange = keyGroupRange;
	this.keyGroupPrefixBytes = keyGroupPrefixBytes;
	this.numberOfTransferringThreads = numberOfTransferringThreads;
	this.cancelStreamRegistry = cancelStreamRegistry;
	this.userCodeClassLoader = userCodeClassLoader;
	this.kvStateInformation = kvStateInformation;
	this.keySerializerProvider = keySerializerProvider;
	this.instanceBasePath = instanceBasePath;
	this.instanceRocksDBPath = instanceRocksDBPath;
	this.dbPath = instanceRocksDBPath.getAbsolutePath();
	this.dbOptions = dbOptions;
	this.columnFamilyOptionsFactory = columnFamilyOptionsFactory;
	this.nativeMetricOptions = nativeMetricOptions;
	this.metricGroup = metricGroup;
	this.restoreStateHandles = stateHandles;
	this.ttlCompactFiltersManager = ttlCompactFiltersManager;
	this.columnFamilyHandles = new ArrayList<>(1);
	this.columnFamilyDescriptors = Collections.emptyList();
}
 
Example #16
Source File: RocksDBFullRestoreOperation.java    From flink with Apache License 2.0 5 votes vote down vote up
public RocksDBFullRestoreOperation(
	KeyGroupRange keyGroupRange,
	int keyGroupPrefixBytes,
	int numberOfTransferringThreads,
	CloseableRegistry cancelStreamRegistry,
	ClassLoader userCodeClassLoader,
	Map<String, RocksDbKvStateInfo> kvStateInformation,
	StateSerializerProvider<K> keySerializerProvider,
	File instanceBasePath,
	File instanceRocksDBPath,
	DBOptions dbOptions,
	Function<String, ColumnFamilyOptions> columnFamilyOptionsFactory,
	RocksDBNativeMetricOptions nativeMetricOptions,
	MetricGroup metricGroup,
	@Nonnull Collection<KeyedStateHandle> restoreStateHandles,
	@Nonnull RocksDbTtlCompactFiltersManager ttlCompactFiltersManager,
	@Nonnegative long writeBatchSize) {
	super(
		keyGroupRange,
		keyGroupPrefixBytes,
		numberOfTransferringThreads,
		cancelStreamRegistry,
		userCodeClassLoader,
		kvStateInformation,
		keySerializerProvider,
		instanceBasePath,
		instanceRocksDBPath,
		dbOptions,
		columnFamilyOptionsFactory,
		nativeMetricOptions,
		metricGroup,
		restoreStateHandles,
		ttlCompactFiltersManager);
	checkArgument(writeBatchSize >= 0, "Write batch size have to be no negative.");
	this.writeBatchSize = writeBatchSize;
}
 
Example #17
Source File: RocksDBIncrementalRestoreOperation.java    From flink with Apache License 2.0 5 votes vote down vote up
public RocksDBIncrementalRestoreOperation(
	String operatorIdentifier,
	KeyGroupRange keyGroupRange,
	int keyGroupPrefixBytes,
	int numberOfTransferringThreads,
	CloseableRegistry cancelStreamRegistry,
	ClassLoader userCodeClassLoader,
	Map<String, RocksDbKvStateInfo> kvStateInformation,
	StateSerializerProvider<K> keySerializerProvider,
	File instanceBasePath,
	File instanceRocksDBPath,
	DBOptions dbOptions,
	Function<String, ColumnFamilyOptions> columnFamilyOptionsFactory,
	RocksDBNativeMetricOptions nativeMetricOptions,
	MetricGroup metricGroup,
	@Nonnull Collection<KeyedStateHandle> restoreStateHandles,
	@Nonnull RocksDbTtlCompactFiltersManager ttlCompactFiltersManager,
	@Nonnegative long writeBatchSize) {
	super(keyGroupRange,
		keyGroupPrefixBytes,
		numberOfTransferringThreads,
		cancelStreamRegistry,
		userCodeClassLoader,
		kvStateInformation,
		keySerializerProvider,
		instanceBasePath,
		instanceRocksDBPath,
		dbOptions,
		columnFamilyOptionsFactory,
		nativeMetricOptions,
		metricGroup,
		restoreStateHandles,
		ttlCompactFiltersManager);
	this.operatorIdentifier = operatorIdentifier;
	this.restoredSstFiles = new TreeMap<>();
	this.lastCompletedCheckpointId = -1L;
	this.backendUID = UUID.randomUUID();
	checkArgument(writeBatchSize >= 0, "Write batch size have to be no negative.");
	this.writeBatchSize = writeBatchSize;
}
 
Example #18
Source File: RocksDBNoneRestoreOperation.java    From flink with Apache License 2.0 5 votes vote down vote up
public RocksDBNoneRestoreOperation(
	KeyGroupRange keyGroupRange,
	int keyGroupPrefixBytes,
	int numberOfTransferringThreads,
	CloseableRegistry cancelStreamRegistry,
	ClassLoader userCodeClassLoader,
	Map<String, RocksDbKvStateInfo> kvStateInformation,
	StateSerializerProvider<K> keySerializerProvider,
	File instanceBasePath,
	File instanceRocksDBPath,
	DBOptions dbOptions,
	Function<String, ColumnFamilyOptions> columnFamilyOptionsFactory,
	RocksDBNativeMetricOptions nativeMetricOptions,
	MetricGroup metricGroup,
	@Nonnull Collection<KeyedStateHandle> restoreStateHandles,
	@Nonnull RocksDbTtlCompactFiltersManager ttlCompactFiltersManager
) {
	super(keyGroupRange,
		keyGroupPrefixBytes,
		numberOfTransferringThreads,
		cancelStreamRegistry,
		userCodeClassLoader,
		kvStateInformation,
		keySerializerProvider,
		instanceBasePath,
		instanceRocksDBPath,
		dbOptions,
		columnFamilyOptionsFactory,
		nativeMetricOptions,
		metricGroup,
		restoreStateHandles,
		ttlCompactFiltersManager);
}
 
Example #19
Source File: RocksFullSnapshotStrategy.java    From flink with Apache License 2.0 5 votes vote down vote up
private MetaData(
	RocksDbKvStateInfo rocksDbKvStateInfo,
	StateSnapshotTransformer<byte[]> stateSnapshotTransformer) {

	this.rocksDbKvStateInfo = rocksDbKvStateInfo;
	this.stateSnapshotTransformer = stateSnapshotTransformer;
}
 
Example #20
Source File: RocksFullSnapshotStrategy.java    From flink with Apache License 2.0 5 votes vote down vote up
SnapshotAsynchronousPartCallable(
	@Nonnull SupplierWithException<CheckpointStreamWithResultProvider, Exception> checkpointStreamSupplier,
	@Nonnull ResourceGuard.Lease dbLease,
	@Nonnull Snapshot snapshot,
	@Nonnull List<StateMetaInfoSnapshot> stateMetaInfoSnapshots,
	@Nonnull List<RocksDbKvStateInfo> metaDataCopy,
	@Nonnull String logPathString) {

	this.checkpointStreamSupplier = checkpointStreamSupplier;
	this.dbLease = dbLease;
	this.snapshot = snapshot;
	this.stateMetaInfoSnapshots = stateMetaInfoSnapshots;
	this.metaData = fillMetaData(metaDataCopy);
	this.logPathString = logPathString;
}
 
Example #21
Source File: RocksFullSnapshotStrategy.java    From flink with Apache License 2.0 5 votes vote down vote up
private static List<MetaData> fillMetaData(
	List<RocksDbKvStateInfo> metaDataCopy) {
	List<MetaData> metaData = new ArrayList<>(metaDataCopy.size());
	for (RocksDbKvStateInfo rocksDbKvStateInfo : metaDataCopy) {
		StateSnapshotTransformer<byte[]> stateSnapshotTransformer = null;
		if (rocksDbKvStateInfo.metaInfo instanceof RegisteredKeyValueStateBackendMetaInfo) {
			stateSnapshotTransformer = ((RegisteredKeyValueStateBackendMetaInfo<?, ?>) rocksDbKvStateInfo.metaInfo).
				getStateSnapshotTransformFactory().createForSerializedState().orElse(null);
		}
		metaData.add(new MetaData(rocksDbKvStateInfo, stateSnapshotTransformer));
	}
	return metaData;
}
 
Example #22
Source File: AbstractRocksDBRestoreOperation.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
protected AbstractRocksDBRestoreOperation(
	KeyGroupRange keyGroupRange,
	int keyGroupPrefixBytes,
	int numberOfTransferringThreads,
	CloseableRegistry cancelStreamRegistry,
	ClassLoader userCodeClassLoader,
	Map<String, RocksDbKvStateInfo> kvStateInformation,
	StateSerializerProvider<K> keySerializerProvider,
	File instanceBasePath,
	File instanceRocksDBPath,
	DBOptions dbOptions,
	Function<String, ColumnFamilyOptions> columnFamilyOptionsFactory,
	RocksDBNativeMetricOptions nativeMetricOptions,
	MetricGroup metricGroup,
	@Nonnull Collection<KeyedStateHandle> stateHandles,
	@Nonnull RocksDbTtlCompactFiltersManager ttlCompactFiltersManager) {
	this.keyGroupRange = keyGroupRange;
	this.keyGroupPrefixBytes = keyGroupPrefixBytes;
	this.numberOfTransferringThreads = numberOfTransferringThreads;
	this.cancelStreamRegistry = cancelStreamRegistry;
	this.userCodeClassLoader = userCodeClassLoader;
	this.kvStateInformation = kvStateInformation;
	this.keySerializerProvider = keySerializerProvider;
	this.instanceBasePath = instanceBasePath;
	this.instanceRocksDBPath = instanceRocksDBPath;
	this.dbPath = instanceRocksDBPath.getAbsolutePath();
	this.dbOptions = dbOptions;
	this.columnFamilyOptionsFactory = columnFamilyOptionsFactory;
	this.nativeMetricOptions = nativeMetricOptions;
	this.metricGroup = metricGroup;
	this.restoreStateHandles = stateHandles;
	this.ttlCompactFiltersManager = ttlCompactFiltersManager;
	this.columnFamilyHandles = new ArrayList<>(1);
	this.columnFamilyDescriptors = Collections.emptyList();
}
 
Example #23
Source File: RocksFullSnapshotStrategy.java    From flink with Apache License 2.0 5 votes vote down vote up
private static List<MetaData> fillMetaData(
	List<RocksDbKvStateInfo> metaDataCopy) {
	List<MetaData> metaData = new ArrayList<>(metaDataCopy.size());
	for (RocksDbKvStateInfo rocksDbKvStateInfo : metaDataCopy) {
		StateSnapshotTransformer<byte[]> stateSnapshotTransformer = null;
		if (rocksDbKvStateInfo.metaInfo instanceof RegisteredKeyValueStateBackendMetaInfo) {
			stateSnapshotTransformer = ((RegisteredKeyValueStateBackendMetaInfo<?, ?>) rocksDbKvStateInfo.metaInfo).
				getStateSnapshotTransformFactory().createForSerializedState().orElse(null);
		}
		metaData.add(new MetaData(rocksDbKvStateInfo, stateSnapshotTransformer));
	}
	return metaData;
}
 
Example #24
Source File: RocksFullSnapshotStrategy.java    From flink with Apache License 2.0 5 votes vote down vote up
SnapshotAsynchronousPartCallable(
	@Nonnull SupplierWithException<CheckpointStreamWithResultProvider, Exception> checkpointStreamSupplier,
	@Nonnull ResourceGuard.Lease dbLease,
	@Nonnull Snapshot snapshot,
	@Nonnull List<StateMetaInfoSnapshot> stateMetaInfoSnapshots,
	@Nonnull List<RocksDbKvStateInfo> metaDataCopy,
	@Nonnull String logPathString) {

	this.checkpointStreamSupplier = checkpointStreamSupplier;
	this.dbLease = dbLease;
	this.snapshot = snapshot;
	this.stateMetaInfoSnapshots = stateMetaInfoSnapshots;
	this.metaData = fillMetaData(metaDataCopy);
	this.logPathString = logPathString;
}
 
Example #25
Source File: RocksIncrementalSnapshotStrategy.java    From flink with Apache License 2.0 5 votes vote down vote up
public RocksIncrementalSnapshotStrategy(
	@Nonnull RocksDB db,
	@Nonnull ResourceGuard rocksDBResourceGuard,
	@Nonnull TypeSerializer<K> keySerializer,
	@Nonnull LinkedHashMap<String, RocksDbKvStateInfo> kvStateInformation,
	@Nonnull KeyGroupRange keyGroupRange,
	@Nonnegative int keyGroupPrefixBytes,
	@Nonnull LocalRecoveryConfig localRecoveryConfig,
	@Nonnull CloseableRegistry cancelStreamRegistry,
	@Nonnull File instanceBasePath,
	@Nonnull UUID backendUID,
	@Nonnull SortedMap<Long, Set<StateHandleID>> materializedSstFiles,
	long lastCompletedCheckpointId,
	int numberOfTransferingThreads) {

	super(
		DESCRIPTION,
		db,
		rocksDBResourceGuard,
		keySerializer,
		kvStateInformation,
		keyGroupRange,
		keyGroupPrefixBytes,
		localRecoveryConfig,
		cancelStreamRegistry);

	this.instanceBasePath = instanceBasePath;
	this.backendUID = backendUID;
	this.materializedSstFiles = materializedSstFiles;
	this.lastCompletedCheckpointId = lastCompletedCheckpointId;
	this.stateUploader = new RocksDBStateUploader(numberOfTransferingThreads);
	this.localDirectoryName = backendUID.toString().replaceAll("[\\-]", "");
}
 
Example #26
Source File: RocksDBNoneRestoreOperation.java    From flink with Apache License 2.0 5 votes vote down vote up
public RocksDBNoneRestoreOperation(
	KeyGroupRange keyGroupRange,
	int keyGroupPrefixBytes,
	int numberOfTransferringThreads,
	CloseableRegistry cancelStreamRegistry,
	ClassLoader userCodeClassLoader,
	Map<String, RocksDbKvStateInfo> kvStateInformation,
	StateSerializerProvider<K> keySerializerProvider,
	File instanceBasePath,
	File instanceRocksDBPath,
	DBOptions dbOptions,
	Function<String, ColumnFamilyOptions> columnFamilyOptionsFactory,
	RocksDBNativeMetricOptions nativeMetricOptions,
	MetricGroup metricGroup,
	@Nonnull Collection<KeyedStateHandle> restoreStateHandles,
	@Nonnull RocksDbTtlCompactFiltersManager ttlCompactFiltersManager
) {
	super(keyGroupRange,
		keyGroupPrefixBytes,
		numberOfTransferringThreads,
		cancelStreamRegistry,
		userCodeClassLoader,
		kvStateInformation,
		keySerializerProvider,
		instanceBasePath,
		instanceRocksDBPath,
		dbOptions,
		columnFamilyOptionsFactory,
		nativeMetricOptions,
		metricGroup,
		restoreStateHandles,
		ttlCompactFiltersManager);
}
 
Example #27
Source File: RocksDBIncrementalRestoreOperation.java    From flink with Apache License 2.0 5 votes vote down vote up
public RocksDBIncrementalRestoreOperation(
	String operatorIdentifier,
	KeyGroupRange keyGroupRange,
	int keyGroupPrefixBytes,
	int numberOfTransferringThreads,
	CloseableRegistry cancelStreamRegistry,
	ClassLoader userCodeClassLoader,
	Map<String, RocksDbKvStateInfo> kvStateInformation,
	StateSerializerProvider<K> keySerializerProvider,
	File instanceBasePath,
	File instanceRocksDBPath,
	DBOptions dbOptions,
	Function<String, ColumnFamilyOptions> columnFamilyOptionsFactory,
	RocksDBNativeMetricOptions nativeMetricOptions,
	MetricGroup metricGroup,
	@Nonnull Collection<KeyedStateHandle> restoreStateHandles,
	@Nonnull RocksDbTtlCompactFiltersManager ttlCompactFiltersManager) {
	super(keyGroupRange,
		keyGroupPrefixBytes,
		numberOfTransferringThreads,
		cancelStreamRegistry,
		userCodeClassLoader,
		kvStateInformation,
		keySerializerProvider,
		instanceBasePath,
		instanceRocksDBPath,
		dbOptions,
		columnFamilyOptionsFactory,
		nativeMetricOptions,
		metricGroup,
		restoreStateHandles,
		ttlCompactFiltersManager);
	this.operatorIdentifier = operatorIdentifier;
	this.restoredSstFiles = new TreeMap<>();
	this.lastCompletedCheckpointId = -1L;
	this.backendUID = UUID.randomUUID();
}
 
Example #28
Source File: RocksDBFullRestoreOperation.java    From flink with Apache License 2.0 5 votes vote down vote up
public RocksDBFullRestoreOperation(
	KeyGroupRange keyGroupRange,
	int keyGroupPrefixBytes,
	int numberOfTransferringThreads,
	CloseableRegistry cancelStreamRegistry,
	ClassLoader userCodeClassLoader,
	Map<String, RocksDbKvStateInfo> kvStateInformation,
	StateSerializerProvider<K> keySerializerProvider,
	File instanceBasePath,
	File instanceRocksDBPath,
	DBOptions dbOptions,
	Function<String, ColumnFamilyOptions> columnFamilyOptionsFactory,
	RocksDBNativeMetricOptions nativeMetricOptions,
	MetricGroup metricGroup,
	@Nonnull Collection<KeyedStateHandle> restoreStateHandles,
	@Nonnull RocksDbTtlCompactFiltersManager ttlCompactFiltersManager) {
	super(
		keyGroupRange,
		keyGroupPrefixBytes,
		numberOfTransferringThreads,
		cancelStreamRegistry,
		userCodeClassLoader,
		kvStateInformation,
		keySerializerProvider,
		instanceBasePath,
		instanceRocksDBPath,
		dbOptions,
		columnFamilyOptionsFactory,
		nativeMetricOptions,
		metricGroup,
		restoreStateHandles,
		ttlCompactFiltersManager);
}
 
Example #29
Source File: AbstractRocksDBRestoreOperation.java    From flink with Apache License 2.0 5 votes vote down vote up
protected AbstractRocksDBRestoreOperation(
	KeyGroupRange keyGroupRange,
	int keyGroupPrefixBytes,
	int numberOfTransferringThreads,
	CloseableRegistry cancelStreamRegistry,
	ClassLoader userCodeClassLoader,
	Map<String, RocksDbKvStateInfo> kvStateInformation,
	StateSerializerProvider<K> keySerializerProvider,
	File instanceBasePath,
	File instanceRocksDBPath,
	DBOptions dbOptions,
	Function<String, ColumnFamilyOptions> columnFamilyOptionsFactory,
	RocksDBNativeMetricOptions nativeMetricOptions,
	MetricGroup metricGroup,
	@Nonnull Collection<KeyedStateHandle> stateHandles,
	@Nonnull RocksDbTtlCompactFiltersManager ttlCompactFiltersManager) {
	this.keyGroupRange = keyGroupRange;
	this.keyGroupPrefixBytes = keyGroupPrefixBytes;
	this.numberOfTransferringThreads = numberOfTransferringThreads;
	this.cancelStreamRegistry = cancelStreamRegistry;
	this.userCodeClassLoader = userCodeClassLoader;
	this.kvStateInformation = kvStateInformation;
	this.keySerializerProvider = keySerializerProvider;
	this.instanceBasePath = instanceBasePath;
	this.instanceRocksDBPath = instanceRocksDBPath;
	this.dbPath = instanceRocksDBPath.getAbsolutePath();
	this.dbOptions = dbOptions;
	this.columnFamilyOptionsFactory = columnFamilyOptionsFactory;
	this.nativeMetricOptions = nativeMetricOptions;
	this.metricGroup = metricGroup;
	this.restoreStateHandles = stateHandles;
	this.ttlCompactFiltersManager = ttlCompactFiltersManager;
	this.columnFamilyHandles = new ArrayList<>(1);
	this.columnFamilyDescriptors = Collections.emptyList();
}
 
Example #30
Source File: RocksFullSnapshotStrategy.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
private MetaData(
	RocksDbKvStateInfo rocksDbKvStateInfo,
	StateSnapshotTransformer<byte[]> stateSnapshotTransformer) {

	this.rocksDbKvStateInfo = rocksDbKvStateInfo;
	this.stateSnapshotTransformer = stateSnapshotTransformer;
}