org.rocksdb.ColumnFamilyDescriptor Java Examples

/**
 * Tests that {@link RocksDBWriteBatchWrapper} flushes after the memory consumed exceeds the preconfigured value.
 */
@Test
public void testWriteBatchWrapperFlushAfterMemorySizeExceed() throws Exception {
	try (RocksDB db = RocksDB.open(folder.newFolder().getAbsolutePath());
		WriteOptions options = new WriteOptions().setDisableWAL(true);
		ColumnFamilyHandle handle = db.createColumnFamily(new ColumnFamilyDescriptor("test".getBytes()));
		RocksDBWriteBatchWrapper writeBatchWrapper = new RocksDBWriteBatchWrapper(db, options, 200, 50)) {

		long initBatchSize = writeBatchWrapper.getDataSize();
		byte[] dummy = new byte[6];
		ThreadLocalRandom.current().nextBytes(dummy);
		// will add 1 + 1 + 1 + 6 + 1 + 6 = 16 bytes for each KV
		// format is [handleType|kvType|keyLen|key|valueLen|value]
		// more information please ref write_batch.cc in RocksDB
		writeBatchWrapper.put(handle, dummy, dummy);
		assertEquals(initBatchSize + 16, writeBatchWrapper.getDataSize());
		writeBatchWrapper.put(handle, dummy, dummy);
		assertEquals(initBatchSize + 32, writeBatchWrapper.getDataSize());
		writeBatchWrapper.put(handle, dummy, dummy);
		// will flush all, then an empty write batch
		assertEquals(initBatchSize, writeBatchWrapper.getDataSize());
	}
}
 

/**
 * Creates a column descriptor for sate column family.
 *
 * <p>Sets TTL compaction filter if {@code ttlCompactFiltersManager} is not {@code null}.
 */
public static ColumnFamilyDescriptor createColumnFamilyDescriptor(
	RegisteredStateMetaInfoBase metaInfoBase,
	Function<String, ColumnFamilyOptions> columnFamilyOptionsFactory,
	@Nullable RocksDbTtlCompactFiltersManager ttlCompactFiltersManager) {

	ColumnFamilyOptions options = createColumnFamilyOptions(columnFamilyOptionsFactory, metaInfoBase.getName());
	if (ttlCompactFiltersManager != null) {
		ttlCompactFiltersManager.setAndRegisterCompactFilterIfStateTtl(metaInfoBase, options);
	}
	byte[] nameBytes = metaInfoBase.getName().getBytes(ConfigConstants.DEFAULT_CHARSET);
	Preconditions.checkState(!Arrays.equals(RocksDB.DEFAULT_COLUMN_FAMILY, nameBytes),
		"The chosen state name 'default' collides with the name of the default column family!");

	return new ColumnFamilyDescriptor(nameBytes, options);
}
 

/**
 * Creates a column descriptor for sate column family.
 *
 * <p>Sets TTL compaction filter if {@code ttlCompactFiltersManager} is not {@code null}.
 */
public static ColumnFamilyDescriptor createColumnFamilyDescriptor(
	RegisteredStateMetaInfoBase metaInfoBase,
	Function<String, ColumnFamilyOptions> columnFamilyOptionsFactory,
	@Nullable RocksDbTtlCompactFiltersManager ttlCompactFiltersManager) {

	ColumnFamilyOptions options = createColumnFamilyOptions(columnFamilyOptionsFactory, metaInfoBase.getName());
	if (ttlCompactFiltersManager != null) {
		ttlCompactFiltersManager.setAndRegisterCompactFilterIfStateTtl(metaInfoBase, options);
	}
	byte[] nameBytes = metaInfoBase.getName().getBytes(ConfigConstants.DEFAULT_CHARSET);
	Preconditions.checkState(!Arrays.equals(RocksDB.DEFAULT_COLUMN_FAMILY, nameBytes),
		"The chosen state name 'default' collides with the name of the default column family!");

	return new ColumnFamilyDescriptor(nameBytes, options);
}
 

@Test
public void readFromFileInvalidConfig() throws IOException {
  final List<String> families =
      Arrays.asList(StringUtils.bytes2String(RocksDB.DEFAULT_COLUMN_FAMILY),
          "First", "Second", "Third",
          "Fourth", "Fifth",
          "Sixth");
  final List<ColumnFamilyDescriptor> columnFamilyDescriptors =
      new ArrayList<>();
  for (String family : families) {
    columnFamilyDescriptors.add(
        new ColumnFamilyDescriptor(family.getBytes(StandardCharsets.UTF_8),
            new ColumnFamilyOptions()));
  }

  final DBOptions options = DBConfigFromFile.readFromFile("badfile.db.ini",
      columnFamilyDescriptors);

  // This has to return a Null, since we have config defined for badfile.db
  Assert.assertNull(options);
}
 

@Test
public void readFromFile() throws IOException {
  final List<String> families =
      Arrays.asList(StringUtils.bytes2String(RocksDB.DEFAULT_COLUMN_FAMILY),
          "First", "Second", "Third",
          "Fourth", "Fifth",
          "Sixth");
  final List<ColumnFamilyDescriptor> columnFamilyDescriptors =
      new ArrayList<>();
  for (String family : families) {
    columnFamilyDescriptors.add(
        new ColumnFamilyDescriptor(family.getBytes(StandardCharsets.UTF_8),
            new ColumnFamilyOptions()));
  }

  final DBOptions options = DBConfigFromFile.readFromFile(DB_FILE,
      columnFamilyDescriptors);

  // Some Random Values Defined in the test.db.ini, we verify that we are
  // able to get values that are defined in the test.db.ini.
  Assert.assertNotNull(options);
  Assert.assertEquals(551615L, options.maxManifestFileSize());
  Assert.assertEquals(1000L, options.keepLogFileNum());
  Assert.assertEquals(1048576, options.writableFileMaxBufferSize());
}
 

/**
 * This method recreates and registers all {@link ColumnFamilyDescriptor} from Flink's state meta data snapshot.
 */
private List<ColumnFamilyDescriptor> createAndRegisterColumnFamilyDescriptors(
	List<StateMetaInfoSnapshot> stateMetaInfoSnapshots,
	boolean registerTtlCompactFilter) {

	List<ColumnFamilyDescriptor> columnFamilyDescriptors =
		new ArrayList<>(stateMetaInfoSnapshots.size());

	for (StateMetaInfoSnapshot stateMetaInfoSnapshot : stateMetaInfoSnapshots) {
		RegisteredStateMetaInfoBase metaInfoBase =
			RegisteredStateMetaInfoBase.fromMetaInfoSnapshot(stateMetaInfoSnapshot);
		ColumnFamilyDescriptor columnFamilyDescriptor = RocksDBOperationUtils.createColumnFamilyDescriptor(
			metaInfoBase, columnFamilyOptionsFactory, registerTtlCompactFilter ? ttlCompactFiltersManager : null);
		columnFamilyDescriptors.add(columnFamilyDescriptor);
	}
	return columnFamilyDescriptors;
}
 

/**
 * Helper to load managed column family descriptors.
 */
private List<ColumnFamilyDescriptor> loadManagedColumnFamilies(DBOptions dbOptions) throws RocksDBException {
  final List<ColumnFamilyDescriptor> managedColumnFamilies = new ArrayList<>();
  final Options options = new Options(dbOptions, new ColumnFamilyOptions());
  List<byte[]> existing = RocksDB.listColumnFamilies(options, rocksDBBasePath);

  if (existing.isEmpty()) {
    LOG.info("No column family found. Loading default");
    managedColumnFamilies.add(getColumnFamilyDescriptor(RocksDB.DEFAULT_COLUMN_FAMILY));
  } else {
    LOG.info("Loading column families :" + existing.stream().map(String::new).collect(Collectors.toList()));
    managedColumnFamilies
        .addAll(existing.stream().map(RocksDBDAO::getColumnFamilyDescriptor).collect(Collectors.toList()));
  }
  return managedColumnFamilies;
}
 

/**
 * This method recreates and registers all {@link ColumnFamilyDescriptor} from Flink's state meta data snapshot.
 */
private List<ColumnFamilyDescriptor> createAndRegisterColumnFamilyDescriptors(
	List<StateMetaInfoSnapshot> stateMetaInfoSnapshots,
	boolean registerTtlCompactFilter) {

	List<ColumnFamilyDescriptor> columnFamilyDescriptors =
		new ArrayList<>(stateMetaInfoSnapshots.size());

	for (StateMetaInfoSnapshot stateMetaInfoSnapshot : stateMetaInfoSnapshots) {
		RegisteredStateMetaInfoBase metaInfoBase =
			RegisteredStateMetaInfoBase.fromMetaInfoSnapshot(stateMetaInfoSnapshot);
		ColumnFamilyDescriptor columnFamilyDescriptor = RocksDBOperationUtils.createColumnFamilyDescriptor(
			metaInfoBase, columnFamilyOptionsFactory, registerTtlCompactFilter ? ttlCompactFiltersManager : null);
		columnFamilyDescriptors.add(columnFamilyDescriptor);
	}
	return columnFamilyDescriptors;
}
 

public Pair<RocksDB, Map<String, ColumnFamilyHandle>> openSupportingIndex(File supportingIndex)
    throws RocksDBException {
  List<FromBrendaDB> instances = BrendaSupportingEntries.allFromBrendaDBInstances();
  List<ColumnFamilyDescriptor> columnFamilyDescriptors = new ArrayList<>(instances.size() + 1);
  columnFamilyDescriptors.add(new ColumnFamilyDescriptor("default".getBytes()));
  for (FromBrendaDB instance : instances) {
    columnFamilyDescriptors.add(new ColumnFamilyDescriptor(instance.getColumnFamilyName().getBytes()));
  }
  List<ColumnFamilyHandle> columnFamilyHandles = new ArrayList<>(columnFamilyDescriptors.size());

  DBOptions dbOptions = new DBOptions();
  dbOptions.setCreateIfMissing(false);
  RocksDB rocksDB = RocksDB.open(dbOptions, supportingIndex.getAbsolutePath(),
      columnFamilyDescriptors, columnFamilyHandles);
  Map<String, ColumnFamilyHandle> columnFamilyHandleMap = new HashMap<>(columnFamilyHandles.size());
  // TODO: can we zip these together more easily w/ Java 8?

  for (int i = 0; i < columnFamilyDescriptors.size(); i++) {
    ColumnFamilyDescriptor cfd = columnFamilyDescriptors.get(i);
    ColumnFamilyHandle cfh = columnFamilyHandles.get(i);
    columnFamilyHandleMap.put(new String(cfd.columnFamilyName(), BrendaSupportingEntries.UTF8), cfh);
  }

  return Pair.of(rocksDB, columnFamilyHandleMap);
}
 

@Override
protected void before() throws Throwable {
	this.temporaryFolder = new TemporaryFolder();
	this.temporaryFolder.create();
	final File rocksFolder = temporaryFolder.newFolder();
	this.dbOptions = optionsFactory.createDBOptions(PredefinedOptions.DEFAULT.createDBOptions()).
		setCreateIfMissing(true);
	this.columnFamilyOptions = optionsFactory.createColumnOptions(PredefinedOptions.DEFAULT.createColumnOptions());
	this.writeOptions = new WriteOptions();
	this.writeOptions.disableWAL();
	this.readOptions = new ReadOptions();
	this.columnFamilyHandles = new ArrayList<>(1);
	this.rocksDB = RocksDB.open(
		dbOptions,
		rocksFolder.getAbsolutePath(),
		Collections.singletonList(new ColumnFamilyDescriptor("default".getBytes(), columnFamilyOptions)),
		columnFamilyHandles);
	this.batchWrapper = new RocksDBWriteBatchWrapper(rocksDB, writeOptions);
}
 

/**
 * Create a new rocks DB at a particular location on disk.
 * @param pathToIndex A path to the directory where the index will be created.
 * @param columnFamilies Column families to create in the DB.
 * @param <T> A type (probably an enum) that represents a set of column families.
 * @return A DB and map of column family labels (as T) to enums.
 * @throws RocksDBException
 */
public static <T extends ColumnFamilyEnumeration<T>> RocksDBAndHandles<T> createNewRocksDB(
    File pathToIndex, T[] columnFamilies) throws RocksDBException {
  RocksDB db = null; // Not auto-closable.
  Map<T, ColumnFamilyHandle> columnFamilyHandles = new HashMap<>();

  db = RocksDB.open(ROCKS_DB_CREATE_OPTIONS, pathToIndex.getAbsolutePath());

  for (T cf : columnFamilies) {
    LOGGER.info("Creating column family %s", cf.getName());
    ColumnFamilyHandle cfh =
        db.createColumnFamily(new ColumnFamilyDescriptor(cf.getName().getBytes(UTF8)));
    columnFamilyHandles.put(cf, cfh);
  }

  return new RocksDBAndHandles<T>(db, columnFamilyHandles);
}
 

/**
 * Creates a column descriptor for sate column family.
 *
 * <p>Sets TTL compaction filter if {@code ttlCompactFiltersManager} is not {@code null}.
 */
public static ColumnFamilyDescriptor createColumnFamilyDescriptor(
	RegisteredStateMetaInfoBase metaInfoBase,
	Function<String, ColumnFamilyOptions> columnFamilyOptionsFactory,
	@Nullable RocksDbTtlCompactFiltersManager ttlCompactFiltersManager) {

	ColumnFamilyOptions options = createColumnFamilyOptions(columnFamilyOptionsFactory, metaInfoBase.getName());
	if (ttlCompactFiltersManager != null) {
		ttlCompactFiltersManager.setAndRegisterCompactFilterIfStateTtl(metaInfoBase, options);
	}
	byte[] nameBytes = metaInfoBase.getName().getBytes(ConfigConstants.DEFAULT_CHARSET);
	Preconditions.checkState(!Arrays.equals(RocksDB.DEFAULT_COLUMN_FAMILY, nameBytes),
		"The chosen state name 'default' collides with the name of the default column family!");

	return new ColumnFamilyDescriptor(nameBytes, options);
}
 

protected static Pair<RocksDB, Map<COLUMN_FAMILIES, ColumnFamilyHandle>> createNewRocksDB(File pathToIndex)
    throws RocksDBException {
  RocksDB db = null; // Not auto-closable.
  Map<COLUMN_FAMILIES, ColumnFamilyHandle> columnFamilyHandles = new HashMap<>();

  Options options = ROCKS_DB_CREATE_OPTIONS;
  System.out.println("Opening index at " + pathToIndex.getAbsolutePath());
  db = RocksDB.open(options, pathToIndex.getAbsolutePath());

  for (COLUMN_FAMILIES cf : COLUMN_FAMILIES.values()) {
    LOGGER.info("Creating column family %s", cf.getName());
    ColumnFamilyHandle cfh =
        db.createColumnFamily(new ColumnFamilyDescriptor(cf.getName().getBytes(UTF8)));
    columnFamilyHandles.put(cf, cfh);
  }

  return Pair.of(db, columnFamilyHandles);
}
 

public RocksDBStore(String name, ColumnFamilyDescriptor family, ColumnFamilyHandle handle, RocksDB db, int stripes,
                    MetaManager metaManager) {
  super();
  this.family = family;
  this.name = name;
  this.db = db;
  this.parallel = stripes;
  this.handle = handle;
  this.sharedLocks = new AutoCloseableLock[stripes];
  this.exclusiveLocks = new AutoCloseableLock[stripes];
  this.metaManager = metaManager;

  for (int i = 0; i < stripes; i++) {
    ReadWriteLock core = new ReentrantReadWriteLock();
    sharedLocks[i] = new AutoCloseableLock(core.readLock());
    exclusiveLocks[i] = new AutoCloseableLock(core.writeLock());
  }

  registerMetrics();
}
 

/**
 * Tests that {@link RocksDBWriteBatchWrapper} flushes after the kv count exceeds the preconfigured value.
 */
@Test
public void testWriteBatchWrapperFlushAfterCountExceed() throws Exception {
	try (RocksDB db = RocksDB.open(folder.newFolder().getAbsolutePath());
		WriteOptions options = new WriteOptions().setDisableWAL(true);
		ColumnFamilyHandle handle = db.createColumnFamily(new ColumnFamilyDescriptor("test".getBytes()));
		RocksDBWriteBatchWrapper writeBatchWrapper = new RocksDBWriteBatchWrapper(db, options, 100, 50000)) {
		long initBatchSize = writeBatchWrapper.getDataSize();
		byte[] dummy = new byte[2];
		ThreadLocalRandom.current().nextBytes(dummy);
		for (int i = 1; i < 100; ++i) {
			writeBatchWrapper.put(handle, dummy, dummy);
			// each kv consumes 8 bytes
			assertEquals(initBatchSize + 8 * i, writeBatchWrapper.getDataSize());
		}
		writeBatchWrapper.put(handle, dummy, dummy);
		assertEquals(initBatchSize, writeBatchWrapper.getDataSize());
	}
}
 

/**
 * This method recreates and registers all {@link ColumnFamilyDescriptor} from Flink's state meta data snapshot.
 */
private List<ColumnFamilyDescriptor> createAndRegisterColumnFamilyDescriptors(
	List<StateMetaInfoSnapshot> stateMetaInfoSnapshots,
	boolean registerTtlCompactFilter) {

	List<ColumnFamilyDescriptor> columnFamilyDescriptors =
		new ArrayList<>(stateMetaInfoSnapshots.size());

	for (StateMetaInfoSnapshot stateMetaInfoSnapshot : stateMetaInfoSnapshots) {
		RegisteredStateMetaInfoBase metaInfoBase =
			RegisteredStateMetaInfoBase.fromMetaInfoSnapshot(stateMetaInfoSnapshot);
		ColumnFamilyDescriptor columnFamilyDescriptor = RocksDBOperationUtils.createColumnFamilyDescriptor(
			metaInfoBase, columnFamilyOptionsFactory, registerTtlCompactFilter ? ttlCompactFiltersManager : null);
		columnFamilyDescriptors.add(columnFamilyDescriptor);
	}
	return columnFamilyDescriptors;
}
 

@Override
protected void before() throws Throwable {
	this.temporaryFolder = new TemporaryFolder();
	this.temporaryFolder.create();
	final File rocksFolder = temporaryFolder.newFolder();
	this.dbOptions = optionsFactory.createDBOptions(PredefinedOptions.DEFAULT.createDBOptions()).
		setCreateIfMissing(true);
	this.columnFamilyOptions = optionsFactory.createColumnOptions(PredefinedOptions.DEFAULT.createColumnOptions());
	this.writeOptions = new WriteOptions();
	this.writeOptions.disableWAL();
	this.readOptions = new ReadOptions();
	this.columnFamilyHandles = new ArrayList<>(1);
	this.rocksDB = RocksDB.open(
		dbOptions,
		rocksFolder.getAbsolutePath(),
		Collections.singletonList(new ColumnFamilyDescriptor("default".getBytes(), columnFamilyOptions)),
		columnFamilyHandles);
	this.batchWrapper = new RocksDBWriteBatchWrapper(rocksDB, writeOptions);
}
 

@SuppressWarnings("resource")
public static KnowledgeBase getInstance(final String kbDir, final String kbMetadataPathname, final boolean readOnly) throws RocksDBException, ClassNotFoundException, IOException {
	final boolean metadataExists = new File(kbMetadataPathname).exists();
	final boolean kbDirExists = new File(kbDir).exists();
	if (metadataExists != kbDirExists) throw new IllegalArgumentException("Either both or none of the knowledge-base directory and metadata must exist");

	RocksDB.loadLibrary();
	final ColumnFamilyOptions cfOptions = new ColumnFamilyOptions().setCompressionType(CompressionType.LZ4_COMPRESSION);
	final DBOptions dbOptions = new DBOptions().setCreateIfMissing(true).setCreateMissingColumnFamilies(true);
	final List<ColumnFamilyDescriptor> cfDescriptors = Arrays.asList(new ColumnFamilyDescriptor(RocksDB.DEFAULT_COLUMN_FAMILY, cfOptions), new ColumnFamilyDescriptor(GID2URI, cfOptions), new ColumnFamilyDescriptor(URI2GID, cfOptions));

	final List<ColumnFamilyHandle> columnFamilyHandles = new ArrayList<>();
	final RocksDB db = readOnly ? RocksDB.openReadOnly(dbOptions, kbDir, cfDescriptors, columnFamilyHandles) : RocksDB.open(dbOptions, kbDir, cfDescriptors, columnFamilyHandles);

	final KnowledgeBase kb;
	if (metadataExists) {
		kb = (KnowledgeBase) BinIO.loadObject(kbMetadataPathname);
		kb.readOnly = readOnly;
		kb.callGraphDB = db;
		kb.defaultHandle = columnFamilyHandles.get(0);
		kb.gid2uriFamilyHandle = columnFamilyHandles.get(1);
		kb.uri2gidFamilyHandle = columnFamilyHandles.get(2);
	} else kb = new KnowledgeBase(db, columnFamilyHandles.get(0), columnFamilyHandles.get(1), columnFamilyHandles.get(2), kbMetadataPathname, readOnly);
	return kb;
}
 

@Override
public synchronized void createTable(String table) throws RocksDBException {
    if (this.cfs.containsKey(table)) {
        return;
    }

    this.checkValid();

    // Should we use options.setCreateMissingColumnFamilies() to create CF
    ColumnFamilyDescriptor cfd = new ColumnFamilyDescriptor(encode(table));
    ColumnFamilyOptions options = cfd.getOptions();
    initOptions(this.config(), null, null, options, options);
    this.cfs.put(table, new CFHandle(this.rocksdb.createColumnFamily(cfd)));

    ingestExternalFile();
}
 

public RocksDBCheckpointIterator(IncrementalKeyedStateHandle handle, FilterFunction<String> stateFilter,
		String localPath) {
	this.localPath = localPath;
	this.cancelStreamRegistry = new CloseableRegistry();
	List<StateMetaInfoSnapshot> stateMetaInfoSnapshots = StateMetadataUtils
			.getKeyedBackendSerializationProxy(handle.getMetaStateHandle()).getStateMetaInfoSnapshots();

	stateColumnFamilyHandles = new ArrayList<>(stateMetaInfoSnapshots.size() + 1);
	List<ColumnFamilyDescriptor> stateColumnFamilyDescriptors = createAndRegisterColumnFamilyDescriptors(
			stateMetaInfoSnapshots);
	try {
		transferAllStateDataToDirectory(handle, new Path(localPath));
		this.db = openDB(localPath, stateColumnFamilyDescriptors, stateColumnFamilyHandles);
		createColumnIterators(stateFilter, stateMetaInfoSnapshots);
	} catch (Exception e) {
		throw new IllegalStateException(e);
	}
}
 

private RocksDB openDB(String path, List<ColumnFamilyDescriptor> stateColumnFamilyDescriptors,
		List<ColumnFamilyHandle> stateColumnFamilyHandles) throws IOException {

	List<ColumnFamilyDescriptor> columnFamilyDescriptors = new ArrayList<>(1 + stateColumnFamilyDescriptors.size());

	// we add the required descriptor for the default CF in FIRST position,
	// see
	// https://github.com/facebook/rocksdb/wiki/RocksJava-Basics#opening-a-database-with-column-families
	columnFamilyDescriptors.add(new ColumnFamilyDescriptor(RocksDB.DEFAULT_COLUMN_FAMILY, colOptions));
	columnFamilyDescriptors.addAll(stateColumnFamilyDescriptors);

	try {
		return RocksDB.open(
				dbOptions,
				Preconditions.checkNotNull(path),
				columnFamilyDescriptors,
				stateColumnFamilyHandles);
	} catch (RocksDBException e) {
		throw new IOException("Error while opening RocksDB instance.", e);
	}
}
 

@Override
protected void before() throws Throwable {
	this.temporaryFolder = new TemporaryFolder();
	this.temporaryFolder.create();
	final File rocksFolder = temporaryFolder.newFolder();
	this.dbOptions = optionsFactory.createDBOptions(
		PredefinedOptions.DEFAULT.createDBOptions(handlesToClose), handlesToClose).setCreateIfMissing(true);
	this.columnFamilyOptions = optionsFactory.createColumnOptions(
		PredefinedOptions.DEFAULT.createColumnOptions(handlesToClose), handlesToClose);
	this.writeOptions = new WriteOptions();
	this.writeOptions.disableWAL();
	this.readOptions = RocksDBOperationUtils.createTotalOrderSeekReadOptions();
	this.columnFamilyHandles = new ArrayList<>(1);
	this.rocksDB = RocksDB.open(
		dbOptions,
		rocksFolder.getAbsolutePath(),
		Collections.singletonList(new ColumnFamilyDescriptor("default".getBytes(), columnFamilyOptions)),
		columnFamilyHandles);
	this.batchWrapper = new RocksDBWriteBatchWrapper(rocksDB, writeOptions);
}
 

private RestoredDBInstance restoreDBInstanceFromStateHandle(
	IncrementalRemoteKeyedStateHandle restoreStateHandle,
	Path temporaryRestoreInstancePath) throws Exception {

	try (RocksDBStateDownloader rocksDBStateDownloader =
			new RocksDBStateDownloader(numberOfTransferringThreads)) {
		rocksDBStateDownloader.transferAllStateDataToDirectory(
			restoreStateHandle,
			temporaryRestoreInstancePath,
			cancelStreamRegistry);
	}

	KeyedBackendSerializationProxy<K> serializationProxy = readMetaData(restoreStateHandle.getMetaStateHandle());
	// read meta data
	List<StateMetaInfoSnapshot> stateMetaInfoSnapshots = serializationProxy.getStateMetaInfoSnapshots();

	List<ColumnFamilyDescriptor> columnFamilyDescriptors =
		createAndRegisterColumnFamilyDescriptors(stateMetaInfoSnapshots, false);

	List<ColumnFamilyHandle> columnFamilyHandles =
		new ArrayList<>(stateMetaInfoSnapshots.size() + 1);

	RocksDB restoreDb = RocksDBOperationUtils.openDB(
		temporaryRestoreInstancePath.toString(),
		columnFamilyDescriptors,
		columnFamilyHandles,
		RocksDBOperationUtils.createColumnFamilyOptions(columnFamilyOptionsFactory, "default"),
		dbOptions);

	return new RestoredDBInstance(restoreDb, columnFamilyHandles, columnFamilyDescriptors, stateMetaInfoSnapshots);
}
 

/**
 * Constructor of RocksDao (Database Access Object).
 *
 * @param dbDir Directory where RocksDB data will be stored
 * @throws RocksDBException if there is an error loading or opening RocksDB instance
 */
public RocksDao(final String dbDir) throws RocksDBException {
    RocksDB.loadLibrary();
    final ColumnFamilyOptions cfOptions = new ColumnFamilyOptions();
    final DBOptions dbOptions = new DBOptions()
            .setCreateIfMissing(true)
            .setCreateMissingColumnFamilies(true);
    final List<ColumnFamilyDescriptor> cfDescriptors = Collections.singletonList(
            new ColumnFamilyDescriptor(RocksDB.DEFAULT_COLUMN_FAMILY, cfOptions));
    final List<ColumnFamilyHandle> columnFamilyHandles = new ArrayList<>();
    this.rocksDb = RocksDB.open(dbOptions, dbDir, cfDescriptors, columnFamilyHandles);
    this.defaultHandle = columnFamilyHandles.get(0);
    initKryo();
}
 

private RocksDBStore newRocksDBStore(String name, ColumnFamilyDescriptor columnFamilyDescriptor,
                                     ColumnFamilyHandle handle) {
  final RocksMetaManager rocksManager;
  if (BLOB_WHITELIST.contains(name)) {
    rocksManager = new RocksMetaManager(baseDirectory, name, FILTER_SIZE_IN_BYTES);
  } else {
    rocksManager = new RocksMetaManager(baseDirectory, name, Long.MAX_VALUE);
  }
  return new RocksDBStore(name, columnFamilyDescriptor, handle, db, stripeCount, rocksManager);
}
 

private ByteStore newStore(String name) throws RocksDBException {
  if (inMemory) {
    return new MapStore(name);
  } else {
    final ColumnFamilyDescriptor columnFamilyDescriptor = new ColumnFamilyDescriptor(name.getBytes(UTF_8));
    ColumnFamilyHandle handle = db.createColumnFamily(columnFamilyDescriptor);
    handleIdToNameMap.put(handle.getID(), name);
    metadataManager.createEntry(name, false);

    return newRocksDBStore(name, columnFamilyDescriptor, handle);
  }
}
 

public static ColumnFamilyHandle createCF(final String name) {
    try {
        ColumnFamilyHandle cfh = DB.createColumnFamily(new ColumnFamilyDescriptor(name.getBytes(Charsets.UTF_8)));
        return cfh;
    } catch (RocksDBException e) {
        LOGGER.error("error while createCF, msg:{}", e.getMessage(), e);
        return null;
    }
}
 

private void openRocksDB(final DBOptions dbOptions,
                         final ColumnFamilyOptions columnFamilyOptions) {
    final List<ColumnFamilyDescriptor> columnFamilyDescriptors
        = Collections.singletonList(new ColumnFamilyDescriptor(RocksDB.DEFAULT_COLUMN_FAMILY, columnFamilyOptions));
    final List<ColumnFamilyHandle> columnFamilies = new ArrayList<>(columnFamilyDescriptors.size());

    try {
        db = RocksDB.open(dbOptions, dbDir.getAbsolutePath(), columnFamilyDescriptors, columnFamilies);
        dbAccessor = new SingleColumnFamilyAccessor(columnFamilies.get(0));
    } catch (final RocksDBException e) {
        throw new CacheInitializationException("Error opening store " + name + " at location " + dbDir.toString(), e);
    }
}
 

private static ColumnFamilyHandle createColumnFamily(ColumnFamilyDescriptor columnDescriptor, RocksDB db) {
	try {
		return db.createColumnFamily(columnDescriptor);
	} catch (RocksDBException e) {
		IOUtils.closeQuietly(columnDescriptor.getOptions());
		throw new FlinkRuntimeException("Error creating ColumnFamilyHandle.", e);
	}
}
 

private RestoredDBInstance(
	@Nonnull RocksDB db,
	@Nonnull List<ColumnFamilyHandle> columnFamilyHandles,
	@Nonnull List<ColumnFamilyDescriptor> columnFamilyDescriptors,
	@Nonnull List<StateMetaInfoSnapshot> stateMetaInfoSnapshots) {
	this.db = db;
	this.defaultColumnFamilyHandle = columnFamilyHandles.remove(0);
	this.columnFamilyHandles = columnFamilyHandles;
	this.columnFamilyDescriptors = columnFamilyDescriptors;
	this.stateMetaInfoSnapshots = stateMetaInfoSnapshots;
	this.readOptions = RocksDBOperationUtils.createTotalOrderSeekReadOptions();
}