Java Code Examples for org.apache.flink.runtime.state.KeyedBackendSerializationProxy#getStateMetaInfoSnapshots()

The following examples show how to use org.apache.flink.runtime.state.KeyedBackendSerializationProxy#getStateMetaInfoSnapshots() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: RocksDBFullRestoreOperation.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up 
/**
 * Restore the KV-state / ColumnFamily meta data for all key-groups referenced by the current state handle.
 */
private void restoreKVStateMetaData() throws IOException, StateMigrationException {
	KeyedBackendSerializationProxy<K> serializationProxy = readMetaData(currentStateHandleInView);

	this.keygroupStreamCompressionDecorator = serializationProxy.isUsingKeyGroupCompression() ?
		SnappyStreamCompressionDecorator.INSTANCE : UncompressedStreamCompressionDecorator.INSTANCE;

	List<StateMetaInfoSnapshot> restoredMetaInfos =
		serializationProxy.getStateMetaInfoSnapshots();
	currentStateHandleKVStateColumnFamilies = new ArrayList<>(restoredMetaInfos.size());

	for (StateMetaInfoSnapshot restoredMetaInfo : restoredMetaInfos) {
		RocksDbKvStateInfo registeredStateCFHandle =
			getOrRegisterStateColumnFamilyHandle(null, restoredMetaInfo);
		currentStateHandleKVStateColumnFamilies.add(registeredStateCFHandle.columnFamilyHandle);
	}
}
Example 2
Source File: RocksDBIncrementalRestoreOperation.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up 
private void restoreFromLocalState(IncrementalLocalKeyedStateHandle localKeyedStateHandle) throws Exception {
	KeyedBackendSerializationProxy<K> serializationProxy = readMetaData(localKeyedStateHandle.getMetaDataState());
	List<StateMetaInfoSnapshot> stateMetaInfoSnapshots = serializationProxy.getStateMetaInfoSnapshots();
	columnFamilyDescriptors = createAndRegisterColumnFamilyDescriptors(stateMetaInfoSnapshots, true);
	columnFamilyHandles = new ArrayList<>(columnFamilyDescriptors.size() + 1);

	Path restoreSourcePath = localKeyedStateHandle.getDirectoryStateHandle().getDirectory();

	LOG.debug("Restoring keyed backend uid in operator {} from incremental snapshot to {}.",
		operatorIdentifier, backendUID);

	if (!instanceRocksDBPath.mkdirs()) {
		String errMsg = "Could not create RocksDB data directory: " + instanceBasePath.getAbsolutePath();
		LOG.error(errMsg);
		throw new IOException(errMsg);
	}

	restoreInstanceDirectoryFromPath(restoreSourcePath, dbPath);

	openDB();

	registerColumnFamilyHandles(stateMetaInfoSnapshots);
}
Example 3
Source File: RocksDBFullRestoreOperation.java    From flink with Apache License 2.0 6 votes vote down vote up 
/**
 * Restore the KV-state / ColumnFamily meta data for all key-groups referenced by the current state handle.
 */
private void restoreKVStateMetaData() throws IOException, StateMigrationException {
	KeyedBackendSerializationProxy<K> serializationProxy = readMetaData(currentStateHandleInView);

	this.keygroupStreamCompressionDecorator = serializationProxy.isUsingKeyGroupCompression() ?
		SnappyStreamCompressionDecorator.INSTANCE : UncompressedStreamCompressionDecorator.INSTANCE;

	List<StateMetaInfoSnapshot> restoredMetaInfos =
		serializationProxy.getStateMetaInfoSnapshots();
	currentStateHandleKVStateColumnFamilies = new ArrayList<>(restoredMetaInfos.size());

	for (StateMetaInfoSnapshot restoredMetaInfo : restoredMetaInfos) {
		RocksDbKvStateInfo registeredStateCFHandle =
			getOrRegisterStateColumnFamilyHandle(null, restoredMetaInfo);
		currentStateHandleKVStateColumnFamilies.add(registeredStateCFHandle.columnFamilyHandle);
	}
}
Example 4
Source File: RocksDBIncrementalRestoreOperation.java    From flink with Apache License 2.0 6 votes vote down vote up 
private void restoreFromLocalState(IncrementalLocalKeyedStateHandle localKeyedStateHandle) throws Exception {
	KeyedBackendSerializationProxy<K> serializationProxy = readMetaData(localKeyedStateHandle.getMetaDataState());
	List<StateMetaInfoSnapshot> stateMetaInfoSnapshots = serializationProxy.getStateMetaInfoSnapshots();
	columnFamilyDescriptors = createAndRegisterColumnFamilyDescriptors(stateMetaInfoSnapshots, true);
	columnFamilyHandles = new ArrayList<>(columnFamilyDescriptors.size() + 1);

	Path restoreSourcePath = localKeyedStateHandle.getDirectoryStateHandle().getDirectory();

	LOG.debug("Restoring keyed backend uid in operator {} from incremental snapshot to {}.",
		operatorIdentifier, backendUID);

	if (!instanceRocksDBPath.mkdirs()) {
		String errMsg = "Could not create RocksDB data directory: " + instanceBasePath.getAbsolutePath();
		LOG.error(errMsg);
		throw new IOException(errMsg);
	}

	restoreInstanceDirectoryFromPath(restoreSourcePath, dbPath);

	openDB();

	registerColumnFamilyHandles(stateMetaInfoSnapshots);
}
Example 5
Source File: RocksDBFullRestoreOperation.java    From flink with Apache License 2.0 6 votes vote down vote up 
/**
 * Restore the KV-state / ColumnFamily meta data for all key-groups referenced by the current state handle.
 */
private void restoreKVStateMetaData() throws IOException, StateMigrationException {
	KeyedBackendSerializationProxy<K> serializationProxy = readMetaData(currentStateHandleInView);

	this.keygroupStreamCompressionDecorator = serializationProxy.isUsingKeyGroupCompression() ?
		SnappyStreamCompressionDecorator.INSTANCE : UncompressedStreamCompressionDecorator.INSTANCE;

	List<StateMetaInfoSnapshot> restoredMetaInfos =
		serializationProxy.getStateMetaInfoSnapshots();
	currentStateHandleKVStateColumnFamilies = new ArrayList<>(restoredMetaInfos.size());

	for (StateMetaInfoSnapshot restoredMetaInfo : restoredMetaInfos) {
		RocksDbKvStateInfo registeredStateCFHandle =
			getOrRegisterStateColumnFamilyHandle(null, restoredMetaInfo);
		currentStateHandleKVStateColumnFamilies.add(registeredStateCFHandle.columnFamilyHandle);
	}
}
Example 6
Source File: RocksDBIncrementalRestoreOperation.java    From flink with Apache License 2.0 6 votes vote down vote up 
private void restoreFromLocalState(IncrementalLocalKeyedStateHandle localKeyedStateHandle) throws Exception {
	KeyedBackendSerializationProxy<K> serializationProxy = readMetaData(localKeyedStateHandle.getMetaDataState());
	List<StateMetaInfoSnapshot> stateMetaInfoSnapshots = serializationProxy.getStateMetaInfoSnapshots();
	columnFamilyDescriptors = createAndRegisterColumnFamilyDescriptors(stateMetaInfoSnapshots, true);
	columnFamilyHandles = new ArrayList<>(columnFamilyDescriptors.size() + 1);

	Path restoreSourcePath = localKeyedStateHandle.getDirectoryStateHandle().getDirectory();

	LOG.debug("Restoring keyed backend uid in operator {} from incremental snapshot to {}.",
		operatorIdentifier, backendUID);

	if (!instanceRocksDBPath.mkdirs()) {
		String errMsg = "Could not create RocksDB data directory: " + instanceBasePath.getAbsolutePath();
		LOG.error(errMsg);
		throw new IOException(errMsg);
	}

	restoreInstanceDirectoryFromPath(restoreSourcePath, dbPath);

	openDB();

	registerColumnFamilyHandles(stateMetaInfoSnapshots);
}
Example 7
Source File: RocksDBIncrementalRestoreOperation.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up 
private RestoredDBInstance restoreDBInstanceFromStateHandle(
	IncrementalRemoteKeyedStateHandle restoreStateHandle,
	Path temporaryRestoreInstancePath) throws Exception {

	try (RocksDBStateDownloader rocksDBStateDownloader =
			new RocksDBStateDownloader(numberOfTransferringThreads)) {
		rocksDBStateDownloader.transferAllStateDataToDirectory(
			restoreStateHandle,
			temporaryRestoreInstancePath,
			cancelStreamRegistry);
	}

	KeyedBackendSerializationProxy<K> serializationProxy = readMetaData(restoreStateHandle.getMetaStateHandle());
	// read meta data
	List<StateMetaInfoSnapshot> stateMetaInfoSnapshots = serializationProxy.getStateMetaInfoSnapshots();

	List<ColumnFamilyDescriptor> columnFamilyDescriptors =
		createAndRegisterColumnFamilyDescriptors(stateMetaInfoSnapshots, false);

	List<ColumnFamilyHandle> columnFamilyHandles =
		new ArrayList<>(stateMetaInfoSnapshots.size() + 1);

	RocksDB restoreDb = RocksDBOperationUtils.openDB(
		temporaryRestoreInstancePath.getPath(),
		columnFamilyDescriptors,
		columnFamilyHandles,
		RocksDBOperationUtils.createColumnFamilyOptions(columnFamilyOptionsFactory, "default"),
		dbOptions);

	return new RestoredDBInstance(restoreDb, columnFamilyHandles, columnFamilyDescriptors, stateMetaInfoSnapshots);
}
Example 8
Source File: RocksDBIncrementalRestoreOperation.java    From flink with Apache License 2.0 5 votes vote down vote up 
private RestoredDBInstance restoreDBInstanceFromStateHandle(
	IncrementalRemoteKeyedStateHandle restoreStateHandle,
	Path temporaryRestoreInstancePath) throws Exception {

	try (RocksDBStateDownloader rocksDBStateDownloader =
			new RocksDBStateDownloader(numberOfTransferringThreads)) {
		rocksDBStateDownloader.transferAllStateDataToDirectory(
			restoreStateHandle,
			temporaryRestoreInstancePath,
			cancelStreamRegistry);
	}

	KeyedBackendSerializationProxy<K> serializationProxy = readMetaData(restoreStateHandle.getMetaStateHandle());
	// read meta data
	List<StateMetaInfoSnapshot> stateMetaInfoSnapshots = serializationProxy.getStateMetaInfoSnapshots();

	List<ColumnFamilyDescriptor> columnFamilyDescriptors =
		createAndRegisterColumnFamilyDescriptors(stateMetaInfoSnapshots, false);

	List<ColumnFamilyHandle> columnFamilyHandles =
		new ArrayList<>(stateMetaInfoSnapshots.size() + 1);

	RocksDB restoreDb = RocksDBOperationUtils.openDB(
		temporaryRestoreInstancePath.getPath(),
		columnFamilyDescriptors,
		columnFamilyHandles,
		RocksDBOperationUtils.createColumnFamilyOptions(columnFamilyOptionsFactory, "default"),
		dbOptions);

	return new RestoredDBInstance(restoreDb, columnFamilyHandles, columnFamilyDescriptors, stateMetaInfoSnapshots);
}
Example 9
Source File: StateMetadataUtils.java    From bravo with Apache License 2.0 5 votes vote down vote up 
@SuppressWarnings("unchecked")
public static <T> Optional<TypeSerializer<T>> getSerializer(KeyedBackendSerializationProxy<?> proxy,
		String stateName) {

	for (StateMetaInfoSnapshot snapshot : proxy.getStateMetaInfoSnapshots()) {
		if (snapshot.getName().equals(stateName)) {
			return Optional
					.of((TypeSerializer<T>) snapshot
							.getTypeSerializerSnapshot(CommonSerializerKeys.VALUE_SERIALIZER)
							.restoreSerializer());
		}
	}

	return Optional.empty();
}
Example 10
Source File: StateMetadataUtils.java    From bravo with Apache License 2.0 5 votes vote down vote up 
public static Map<Integer, String> getStateIdMapping(KeyedBackendSerializationProxy<?> proxy) {
	Map<Integer, String> stateIdMapping = new HashMap<>();

	int stateId = 0;
	for (StateMetaInfoSnapshot snapshot : proxy.getStateMetaInfoSnapshots()) {
		stateIdMapping.put(stateId, snapshot.getName());
		stateId++;
	}

	return stateIdMapping;
}
Example 11
Source File: RocksDBIncrementalRestoreOperation.java    From flink with Apache License 2.0 5 votes vote down vote up 
private RestoredDBInstance restoreDBInstanceFromStateHandle(
	IncrementalRemoteKeyedStateHandle restoreStateHandle,
	Path temporaryRestoreInstancePath) throws Exception {

	try (RocksDBStateDownloader rocksDBStateDownloader =
			new RocksDBStateDownloader(numberOfTransferringThreads)) {
		rocksDBStateDownloader.transferAllStateDataToDirectory(
			restoreStateHandle,
			temporaryRestoreInstancePath,
			cancelStreamRegistry);
	}

	KeyedBackendSerializationProxy<K> serializationProxy = readMetaData(restoreStateHandle.getMetaStateHandle());
	// read meta data
	List<StateMetaInfoSnapshot> stateMetaInfoSnapshots = serializationProxy.getStateMetaInfoSnapshots();

	List<ColumnFamilyDescriptor> columnFamilyDescriptors =
		createAndRegisterColumnFamilyDescriptors(stateMetaInfoSnapshots, false);

	List<ColumnFamilyHandle> columnFamilyHandles =
		new ArrayList<>(stateMetaInfoSnapshots.size() + 1);

	RocksDB restoreDb = RocksDBOperationUtils.openDB(
		temporaryRestoreInstancePath.toString(),
		columnFamilyDescriptors,
		columnFamilyHandles,
		RocksDBOperationUtils.createColumnFamilyOptions(columnFamilyOptionsFactory, "default"),
		dbOptions);

	return new RestoredDBInstance(restoreDb, columnFamilyHandles, columnFamilyDescriptors, stateMetaInfoSnapshots);
}
Example 12
Source File: HeapRestoreOperation.java    From Flink-CEPplus with Apache License 2.0 4 votes vote down vote up 
@Override
public Void restore() throws Exception {

	final Map<Integer, StateMetaInfoSnapshot> kvStatesById = new HashMap<>();
	registeredKVStates.clear();
	registeredPQStates.clear();

	boolean keySerializerRestored = false;

	for (KeyedStateHandle keyedStateHandle : restoreStateHandles) {

		if (keyedStateHandle == null) {
			continue;
		}

		if (!(keyedStateHandle instanceof KeyGroupsStateHandle)) {
			throw new IllegalStateException("Unexpected state handle type, " +
				"expected: " + KeyGroupsStateHandle.class +
				", but found: " + keyedStateHandle.getClass());
		}

		KeyGroupsStateHandle keyGroupsStateHandle = (KeyGroupsStateHandle) keyedStateHandle;
		FSDataInputStream fsDataInputStream = keyGroupsStateHandle.openInputStream();
		cancelStreamRegistry.registerCloseable(fsDataInputStream);

		try {
			DataInputViewStreamWrapper inView = new DataInputViewStreamWrapper(fsDataInputStream);

			KeyedBackendSerializationProxy<K> serializationProxy =
				new KeyedBackendSerializationProxy<>(userCodeClassLoader);

			serializationProxy.read(inView);

			if (!keySerializerRestored) {
				// check for key serializer compatibility; this also reconfigures the
				// key serializer to be compatible, if it is required and is possible
				TypeSerializerSchemaCompatibility<K> keySerializerSchemaCompat =
					keySerializerProvider.setPreviousSerializerSnapshotForRestoredState(serializationProxy.getKeySerializerSnapshot());
				if (keySerializerSchemaCompat.isCompatibleAfterMigration() || keySerializerSchemaCompat.isIncompatible()) {
					throw new StateMigrationException("The new key serializer must be compatible.");
				}

				keySerializerRestored = true;
			}

			List<StateMetaInfoSnapshot> restoredMetaInfos =
				serializationProxy.getStateMetaInfoSnapshots();

			createOrCheckStateForMetaInfo(restoredMetaInfos, kvStatesById);

			readStateHandleStateData(
				fsDataInputStream,
				inView,
				keyGroupsStateHandle.getGroupRangeOffsets(),
				kvStatesById, restoredMetaInfos.size(),
				serializationProxy.getReadVersion(),
				serializationProxy.isUsingKeyGroupCompression());
		} finally {
			if (cancelStreamRegistry.unregisterCloseable(fsDataInputStream)) {
				IOUtils.closeQuietly(fsDataInputStream);
			}
		}
	}
	return null;
}
Example 13
Source File: HeapRestoreOperation.java    From flink with Apache License 2.0 4 votes vote down vote up 
@Override
public Void restore() throws Exception {

	final Map<Integer, StateMetaInfoSnapshot> kvStatesById = new HashMap<>();
	registeredKVStates.clear();
	registeredPQStates.clear();

	boolean keySerializerRestored = false;

	for (KeyedStateHandle keyedStateHandle : restoreStateHandles) {

		if (keyedStateHandle == null) {
			continue;
		}

		if (!(keyedStateHandle instanceof KeyGroupsStateHandle)) {
			throw new IllegalStateException("Unexpected state handle type, " +
				"expected: " + KeyGroupsStateHandle.class +
				", but found: " + keyedStateHandle.getClass());
		}

		KeyGroupsStateHandle keyGroupsStateHandle = (KeyGroupsStateHandle) keyedStateHandle;
		FSDataInputStream fsDataInputStream = keyGroupsStateHandle.openInputStream();
		cancelStreamRegistry.registerCloseable(fsDataInputStream);

		try {
			DataInputViewStreamWrapper inView = new DataInputViewStreamWrapper(fsDataInputStream);

			KeyedBackendSerializationProxy<K> serializationProxy =
				new KeyedBackendSerializationProxy<>(userCodeClassLoader);

			serializationProxy.read(inView);

			if (!keySerializerRestored) {
				// check for key serializer compatibility; this also reconfigures the
				// key serializer to be compatible, if it is required and is possible
				TypeSerializerSchemaCompatibility<K> keySerializerSchemaCompat =
					keySerializerProvider.setPreviousSerializerSnapshotForRestoredState(serializationProxy.getKeySerializerSnapshot());
				if (keySerializerSchemaCompat.isCompatibleAfterMigration() || keySerializerSchemaCompat.isIncompatible()) {
					throw new StateMigrationException("The new key serializer must be compatible.");
				}

				keySerializerRestored = true;
			}

			List<StateMetaInfoSnapshot> restoredMetaInfos =
				serializationProxy.getStateMetaInfoSnapshots();

			createOrCheckStateForMetaInfo(restoredMetaInfos, kvStatesById);

			readStateHandleStateData(
				fsDataInputStream,
				inView,
				keyGroupsStateHandle.getGroupRangeOffsets(),
				kvStatesById, restoredMetaInfos.size(),
				serializationProxy.getReadVersion(),
				serializationProxy.isUsingKeyGroupCompression());
		} finally {
			if (cancelStreamRegistry.unregisterCloseable(fsDataInputStream)) {
				IOUtils.closeQuietly(fsDataInputStream);
			}
		}
	}
	return null;
}
Example 14
Source File: HeapRestoreOperation.java    From flink with Apache License 2.0 4 votes vote down vote up 
@Override
public Void restore() throws Exception {

	registeredKVStates.clear();
	registeredPQStates.clear();

	boolean keySerializerRestored = false;

	for (KeyedStateHandle keyedStateHandle : restoreStateHandles) {

		if (keyedStateHandle == null) {
			continue;
		}

		if (!(keyedStateHandle instanceof KeyGroupsStateHandle)) {
			throw new IllegalStateException("Unexpected state handle type, " +
				"expected: " + KeyGroupsStateHandle.class +
				", but found: " + keyedStateHandle.getClass());
		}

		KeyGroupsStateHandle keyGroupsStateHandle = (KeyGroupsStateHandle) keyedStateHandle;
		FSDataInputStream fsDataInputStream = keyGroupsStateHandle.openInputStream();
		cancelStreamRegistry.registerCloseable(fsDataInputStream);

		try {
			DataInputViewStreamWrapper inView = new DataInputViewStreamWrapper(fsDataInputStream);

			KeyedBackendSerializationProxy<K> serializationProxy =
				new KeyedBackendSerializationProxy<>(userCodeClassLoader);

			serializationProxy.read(inView);

			if (!keySerializerRestored) {
				// check for key serializer compatibility; this also reconfigures the
				// key serializer to be compatible, if it is required and is possible
				TypeSerializerSchemaCompatibility<K> keySerializerSchemaCompat =
					keySerializerProvider.setPreviousSerializerSnapshotForRestoredState(serializationProxy.getKeySerializerSnapshot());
				if (keySerializerSchemaCompat.isCompatibleAfterMigration() || keySerializerSchemaCompat.isIncompatible()) {
					throw new StateMigrationException("The new key serializer must be compatible.");
				}

				keySerializerRestored = true;
			}

			List<StateMetaInfoSnapshot> restoredMetaInfos =
				serializationProxy.getStateMetaInfoSnapshots();

			final Map<Integer, StateMetaInfoSnapshot> kvStatesById = new HashMap<>();

			createOrCheckStateForMetaInfo(restoredMetaInfos, kvStatesById);

			readStateHandleStateData(
				fsDataInputStream,
				inView,
				keyGroupsStateHandle.getGroupRangeOffsets(),
				kvStatesById, restoredMetaInfos.size(),
				serializationProxy.getReadVersion(),
				serializationProxy.isUsingKeyGroupCompression());
		} finally {
			if (cancelStreamRegistry.unregisterCloseable(fsDataInputStream)) {
				IOUtils.closeQuietly(fsDataInputStream);
			}
		}
	}
	return null;
}