org.rocksdb.ColumnFamilyOptions Java Examples

/**
 * Get a new default ColumnFamilyOptions or a copy of the exist
 * ColumnFamilyOptions.  Users should call ColumnFamilyOptions#close()
 * to release resources themselves.
 *
 * @param cls the key of ColumnFamilyOptions
 * @return new default ColumnFamilyOptions or a copy of the exist
 * ColumnFamilyOptions
 */
public static ColumnFamilyOptions getRocksDBColumnFamilyOptions(final Class<?> cls) {
    Requires.requireNonNull(cls, "cls");
    ColumnFamilyOptions opts = columnFamilyOptionsTable.get(cls.getName());
    if (opts == null) {
        final ColumnFamilyOptions newOpts = getDefaultRocksDBColumnFamilyOptions();
        opts = columnFamilyOptionsTable.putIfAbsent(cls.getName(), newOpts);
        if (opts == null) {
            opts = newOpts;
        } else {
            newOpts.close();
        }
    }
    // NOTE: This does a shallow copy, which means comparator, merge_operator,
    // compaction_filter, compaction_filter_factory and other pointers will be
    // cloned!
    return new ColumnFamilyOptions(checkInvalid(opts));
}
 

@Test
public void testPredefinedOptions() throws Exception {
	String checkpointPath = tempFolder.newFolder().toURI().toString();
	RocksDBStateBackend rocksDbBackend = new RocksDBStateBackend(checkpointPath);

	// verify that we would use PredefinedOptions.DEFAULT by default.
	assertEquals(PredefinedOptions.DEFAULT, rocksDbBackend.getPredefinedOptions());

	// verify that user could configure predefined options via flink-conf.yaml
	Configuration configuration = new Configuration();
	configuration.setString(RocksDBOptions.PREDEFINED_OPTIONS, PredefinedOptions.FLASH_SSD_OPTIMIZED.name());
	rocksDbBackend = new RocksDBStateBackend(checkpointPath);
	rocksDbBackend = rocksDbBackend.configure(configuration, getClass().getClassLoader());
	assertEquals(PredefinedOptions.FLASH_SSD_OPTIMIZED, rocksDbBackend.getPredefinedOptions());

	// verify that predefined options could be set programmatically and override pre-configured one.
	rocksDbBackend.setPredefinedOptions(PredefinedOptions.SPINNING_DISK_OPTIMIZED);
	assertEquals(PredefinedOptions.SPINNING_DISK_OPTIMIZED, rocksDbBackend.getPredefinedOptions());

	try (ColumnFamilyOptions colCreated = rocksDbBackend.getColumnOptions()) {
		assertEquals(CompactionStyle.LEVEL, colCreated.compactionStyle());
	}
}
 

/**
 * Guard that {@link RocksDBResourceContainer#getColumnOptions()} shares the same {@link Cache} instance
 * if the {@link RocksDBResourceContainer} instance is initiated with {@link OpaqueMemoryResource}.
 *
 * @throws Exception if unexpected error happened.
 */
@Test
public void testGetColumnFamilyOptionsWithSharedResources() throws Exception {
	final int optionNumber = 20;
	OpaqueMemoryResource<RocksDBSharedResources> sharedResources = getSharedResources();
	RocksDBResourceContainer container =
		new RocksDBResourceContainer(PredefinedOptions.DEFAULT, null, sharedResources);
	HashSet<Cache> caches = new HashSet<>();
	for (int i = 0; i < optionNumber; i++) {
		ColumnFamilyOptions columnOptions = container.getColumnOptions();
		Cache cache = getBlockCache(columnOptions);
		caches.add(cache);
	}
	assertThat(caches.size(), is(1));
	assertThat(caches.iterator().next(),
		is(sharedResources.getResourceHandle().getCache()));
	container.close();
}
 

@Test
public void testPredefinedOptions() throws Exception {
	String checkpointPath = tempFolder.newFolder().toURI().toString();
	RocksDBStateBackend rocksDbBackend = new RocksDBStateBackend(checkpointPath);

	// verify that we would use PredefinedOptions.DEFAULT by default.
	assertEquals(PredefinedOptions.DEFAULT, rocksDbBackend.getPredefinedOptions());

	// verify that user could configure predefined options via flink-conf.yaml
	Configuration configuration = new Configuration();
	configuration.setString(RocksDBOptions.PREDEFINED_OPTIONS, PredefinedOptions.FLASH_SSD_OPTIMIZED.name());
	rocksDbBackend = new RocksDBStateBackend(checkpointPath);
	rocksDbBackend = rocksDbBackend.configure(configuration, getClass().getClassLoader());
	assertEquals(PredefinedOptions.FLASH_SSD_OPTIMIZED, rocksDbBackend.getPredefinedOptions());

	// verify that predefined options could be set programmatically and override pre-configured one.
	rocksDbBackend.setPredefinedOptions(PredefinedOptions.SPINNING_DISK_OPTIMIZED);
	assertEquals(PredefinedOptions.SPINNING_DISK_OPTIMIZED, rocksDbBackend.getPredefinedOptions());

	try (ColumnFamilyOptions colCreated = rocksDbBackend.getColumnOptions()) {
		assertEquals(CompactionStyle.LEVEL, colCreated.compactionStyle());
	}
}
 

RocksDBPriorityQueueSetFactory(
	KeyGroupRange keyGroupRange,
	int keyGroupPrefixBytes,
	int numberOfKeyGroups,
	Map<String, RocksDBKeyedStateBackend.RocksDbKvStateInfo> kvStateInformation,
	RocksDB db,
	RocksDBWriteBatchWrapper writeBatchWrapper,
	RocksDBNativeMetricMonitor nativeMetricMonitor,
	Function<String, ColumnFamilyOptions> columnFamilyOptionsFactory) {
	this.keyGroupRange = keyGroupRange;
	this.keyGroupPrefixBytes = keyGroupPrefixBytes;
	this.numberOfKeyGroups = numberOfKeyGroups;
	this.kvStateInformation = kvStateInformation;
	this.db = db;
	this.writeBatchWrapper = writeBatchWrapper;
	this.nativeMetricMonitor = nativeMetricMonitor;
	this.columnFamilyOptionsFactory = columnFamilyOptionsFactory;
	this.sharedElementOutView = new DataOutputSerializer(128);
	this.sharedElementInView = new DataInputDeserializer();
}
 

@Test
public void testPathExceptionOnWindows() throws Exception {
	assumeTrue(OperatingSystem.isWindows());

	final File folder = TMP_DIR.newFolder();
	final File rocksDir = new File(folder, getLongString(247 - folder.getAbsolutePath().length()));

	Files.createDirectories(rocksDir.toPath());

	try (DBOptions dbOptions = new DBOptions().setCreateIfMissing(true);
		ColumnFamilyOptions colOptions = new ColumnFamilyOptions()) {

		RocksDB rocks = RocksDBOperationUtils.openDB(
				rocksDir.getAbsolutePath(),
				Collections.emptyList(),
				Collections.emptyList(),
				colOptions, dbOptions);
		rocks.close();

		// do not provoke a test failure if this passes, because some setups may actually
		// support long paths, in which case: great!
	}
	catch (IOException e) {
		assertThat(e.getMessage(), containsString("longer than the directory path length limit for Windows"));
	}
}
 

@Before
public void setUp() throws Exception {
  options = new DBOptions();
  options.setCreateIfMissing(true);
  options.setCreateMissingColumnFamilies(true);

  Statistics statistics = new Statistics();
  statistics.setStatsLevel(StatsLevel.ALL);
  options = options.setStatistics(statistics);

  Set<TableConfig> configSet = new HashSet<>();
  for(String name : families) {
    TableConfig newConfig = new TableConfig(name, new ColumnFamilyOptions());
    configSet.add(newConfig);
  }
  rdbStore = new RDBStore(folder.newFolder(), options, configSet);
}
 

@Test
public void readFromFile() throws IOException {
  final List<String> families =
      Arrays.asList(StringUtils.bytes2String(RocksDB.DEFAULT_COLUMN_FAMILY),
          "First", "Second", "Third",
          "Fourth", "Fifth",
          "Sixth");
  final List<ColumnFamilyDescriptor> columnFamilyDescriptors =
      new ArrayList<>();
  for (String family : families) {
    columnFamilyDescriptors.add(
        new ColumnFamilyDescriptor(family.getBytes(StandardCharsets.UTF_8),
            new ColumnFamilyOptions()));
  }

  final DBOptions options = DBConfigFromFile.readFromFile(DB_FILE,
      columnFamilyDescriptors);

  // Some Random Values Defined in the test.db.ini, we verify that we are
  // able to get values that are defined in the test.db.ini.
  Assert.assertNotNull(options);
  Assert.assertEquals(551615L, options.maxManifestFileSize());
  Assert.assertEquals(1000L, options.keepLogFileNum());
  Assert.assertEquals(1048576, options.writableFileMaxBufferSize());
}
 

@Before
public void setUp() throws Exception {
  options = new DBOptions();
  options.setCreateIfMissing(true);
  options.setCreateMissingColumnFamilies(true);

  Statistics statistics = new Statistics();
  statistics.setStatsLevel(StatsLevel.ALL);
  options = options.setStatistics(statistics);
  configSet = new HashSet<>();
  for(String name : families) {
    TableConfig newConfig = new TableConfig(name, new ColumnFamilyOptions());
    configSet.add(newConfig);
  }
  rdbStore = new RDBStore(folder.newFolder(), options, configSet);
}
 

@Test
public void readFromFileInvalidConfig() throws IOException {
  final List<String> families =
      Arrays.asList(StringUtils.bytes2String(RocksDB.DEFAULT_COLUMN_FAMILY),
          "First", "Second", "Third",
          "Fourth", "Fifth",
          "Sixth");
  final List<ColumnFamilyDescriptor> columnFamilyDescriptors =
      new ArrayList<>();
  for (String family : families) {
    columnFamilyDescriptors.add(
        new ColumnFamilyDescriptor(family.getBytes(StandardCharsets.UTF_8),
            new ColumnFamilyOptions()));
  }

  final DBOptions options = DBConfigFromFile.readFromFile("badfile.db.ini",
      columnFamilyDescriptors);

  // This has to return a Null, since we have config defined for badfile.db
  Assert.assertNull(options);
}
 

@Before
public void setUp() throws Exception {
  options = new DBOptions();
  options.setCreateIfMissing(true);
  options.setCreateMissingColumnFamilies(true);

  Statistics statistics = new Statistics();
  statistics.setStatsLevel(StatsLevel.ALL);
  options = options.setStatistics(statistics);

  Set<TableConfig> configSet = new HashSet<>();
  for (String name : families) {
    TableConfig newConfig = new TableConfig(name, new ColumnFamilyOptions());
    configSet.add(newConfig);
  }
  rdbStore = new RDBStore(folder.newFolder(), options, configSet);

  codecRegistry = new CodecRegistry();

}
 

RocksDBPriorityQueueSetFactory(
	KeyGroupRange keyGroupRange,
	int keyGroupPrefixBytes,
	int numberOfKeyGroups,
	Map<String, RocksDBKeyedStateBackend.RocksDbKvStateInfo> kvStateInformation,
	RocksDB db,
	ReadOptions readOptions,
	RocksDBWriteBatchWrapper writeBatchWrapper,
	RocksDBNativeMetricMonitor nativeMetricMonitor,
	Function<String, ColumnFamilyOptions> columnFamilyOptionsFactory) {
	this.keyGroupRange = keyGroupRange;
	this.keyGroupPrefixBytes = keyGroupPrefixBytes;
	this.numberOfKeyGroups = numberOfKeyGroups;
	this.kvStateInformation = kvStateInformation;
	this.db = db;
	this.readOptions = readOptions;
	this.writeBatchWrapper = writeBatchWrapper;
	this.nativeMetricMonitor = nativeMetricMonitor;
	this.columnFamilyOptionsFactory = columnFamilyOptionsFactory;
	this.sharedElementOutView = new DataOutputSerializer(128);
	this.sharedElementInView = new DataInputDeserializer();
}
 

public void setAndRegisterCompactFilterIfStateTtl(
	@Nonnull RegisteredStateMetaInfoBase metaInfoBase,
	@Nonnull ColumnFamilyOptions options) {

	if (enableTtlCompactionFilter && metaInfoBase instanceof RegisteredKeyValueStateBackendMetaInfo) {
		RegisteredKeyValueStateBackendMetaInfo kvMetaInfoBase = (RegisteredKeyValueStateBackendMetaInfo) metaInfoBase;
		if (TtlStateFactory.TtlSerializer.isTtlStateSerializer(kvMetaInfoBase.getStateSerializer())) {
			createAndSetCompactFilterFactory(metaInfoBase.getName(), options);
		}
	}
}
 

protected AbstractRocksDBRestoreOperation(
	KeyGroupRange keyGroupRange,
	int keyGroupPrefixBytes,
	int numberOfTransferringThreads,
	CloseableRegistry cancelStreamRegistry,
	ClassLoader userCodeClassLoader,
	Map<String, RocksDbKvStateInfo> kvStateInformation,
	StateSerializerProvider<K> keySerializerProvider,
	File instanceBasePath,
	File instanceRocksDBPath,
	DBOptions dbOptions,
	Function<String, ColumnFamilyOptions> columnFamilyOptionsFactory,
	RocksDBNativeMetricOptions nativeMetricOptions,
	MetricGroup metricGroup,
	@Nonnull Collection<KeyedStateHandle> stateHandles,
	@Nonnull RocksDbTtlCompactFiltersManager ttlCompactFiltersManager) {
	this.keyGroupRange = keyGroupRange;
	this.keyGroupPrefixBytes = keyGroupPrefixBytes;
	this.numberOfTransferringThreads = numberOfTransferringThreads;
	this.cancelStreamRegistry = cancelStreamRegistry;
	this.userCodeClassLoader = userCodeClassLoader;
	this.kvStateInformation = kvStateInformation;
	this.keySerializerProvider = keySerializerProvider;
	this.instanceBasePath = instanceBasePath;
	this.instanceRocksDBPath = instanceRocksDBPath;
	this.dbPath = instanceRocksDBPath.getAbsolutePath();
	this.dbOptions = dbOptions;
	this.columnFamilyOptionsFactory = columnFamilyOptionsFactory;
	this.nativeMetricOptions = nativeMetricOptions;
	this.metricGroup = metricGroup;
	this.restoreStateHandles = stateHandles;
	this.ttlCompactFiltersManager = ttlCompactFiltersManager;
	this.columnFamilyHandles = new ArrayList<>(1);
	this.columnFamilyDescriptors = Collections.emptyList();
}
 

private static ColumnFamilyOptions createColumnFamilyOptions(
    final RocksDbConfiguration configuration) {
  return new ColumnFamilyOptions()
      .setCompressionType(configuration.getCompressionType())
      .setBottommostCompressionType(configuration.getBottomMostCompressionType())
      .setTableFormatConfig(createBlockBasedTableConfig(configuration));
}
 

private static List<ColumnFamilyDescriptor> createColumnFamilyDescriptors(
    final Class<? extends Schema> schema, final ColumnFamilyOptions columnFamilyOptions) {
  List<ColumnFamilyDescriptor> columnDescriptors =
      Schema.streamColumns(schema)
          .map(
              col -> new ColumnFamilyDescriptor(col.getId().toArrayUnsafe(), columnFamilyOptions))
          .collect(Collectors.toList());
  columnDescriptors.add(
      new ColumnFamilyDescriptor(Schema.DEFAULT_COLUMN_ID.toArrayUnsafe(), columnFamilyOptions));
  return columnDescriptors;
}
 

private void openRocksDB(final DBOptions dbOptions,
                         final ColumnFamilyOptions columnFamilyOptions) {
    final List<ColumnFamilyDescriptor> columnFamilyDescriptors
        = Collections.singletonList(new ColumnFamilyDescriptor(RocksDB.DEFAULT_COLUMN_FAMILY, columnFamilyOptions));
    final List<ColumnFamilyHandle> columnFamilies = new ArrayList<>(columnFamilyDescriptors.size());

    try {
        db = RocksDB.open(dbOptions, dbDir.getAbsolutePath(), columnFamilyDescriptors, columnFamilies);
        dbAccessor = new SingleColumnFamilyAccessor(columnFamilies.get(0));
    } catch (final RocksDBException e) {
        throw new CacheInitializationException("Error opening store " + name + " at location " + dbDir.toString(), e);
    }
}
 

public RocksDBFullRestoreOperation(
	KeyGroupRange keyGroupRange,
	int keyGroupPrefixBytes,
	int numberOfTransferringThreads,
	CloseableRegistry cancelStreamRegistry,
	ClassLoader userCodeClassLoader,
	Map<String, RocksDbKvStateInfo> kvStateInformation,
	StateSerializerProvider<K> keySerializerProvider,
	File instanceBasePath,
	File instanceRocksDBPath,
	DBOptions dbOptions,
	Function<String, ColumnFamilyOptions> columnFamilyOptionsFactory,
	RocksDBNativeMetricOptions nativeMetricOptions,
	MetricGroup metricGroup,
	@Nonnull Collection<KeyedStateHandle> restoreStateHandles,
	@Nonnull RocksDbTtlCompactFiltersManager ttlCompactFiltersManager,
	@Nonnegative long writeBatchSize) {
	super(
		keyGroupRange,
		keyGroupPrefixBytes,
		numberOfTransferringThreads,
		cancelStreamRegistry,
		userCodeClassLoader,
		kvStateInformation,
		keySerializerProvider,
		instanceBasePath,
		instanceRocksDBPath,
		dbOptions,
		columnFamilyOptionsFactory,
		nativeMetricOptions,
		metricGroup,
		restoreStateHandles,
		ttlCompactFiltersManager);
	checkArgument(writeBatchSize >= 0, "Write batch size have to be no negative.");
	this.writeBatchSize = writeBatchSize;
}
 

public static RocksDB openDB(
	String path,
	List<ColumnFamilyDescriptor> stateColumnFamilyDescriptors,
	List<ColumnFamilyHandle> stateColumnFamilyHandles,
	ColumnFamilyOptions columnFamilyOptions,
	DBOptions dbOptions) throws IOException {
	List<ColumnFamilyDescriptor> columnFamilyDescriptors =
		new ArrayList<>(1 + stateColumnFamilyDescriptors.size());

	// we add the required descriptor for the default CF in FIRST position, see
	// https://github.com/facebook/rocksdb/wiki/RocksJava-Basics#opening-a-database-with-column-families
	columnFamilyDescriptors.add(new ColumnFamilyDescriptor(RocksDB.DEFAULT_COLUMN_FAMILY, columnFamilyOptions));
	columnFamilyDescriptors.addAll(stateColumnFamilyDescriptors);

	RocksDB dbRef;

	try {
		dbRef = RocksDB.open(
			Preconditions.checkNotNull(dbOptions),
			Preconditions.checkNotNull(path),
			columnFamilyDescriptors,
			stateColumnFamilyHandles);
	} catch (RocksDBException e) {
		IOUtils.closeQuietly(columnFamilyOptions);
		columnFamilyDescriptors.forEach((cfd) -> IOUtils.closeQuietly(cfd.getOptions()));

		// improve error reporting on Windows
		throwExceptionIfPathLengthExceededOnWindows(path, e);

		throw new IOException("Error while opening RocksDB instance.", e);
	}

	// requested + default CF
	Preconditions.checkState(1 + stateColumnFamilyDescriptors.size() == stateColumnFamilyHandles.size(),
		"Not all requested column family handles have been created");
	return dbRef;
}
 

public RocksDBIncrementalRestoreOperation(
	String operatorIdentifier,
	KeyGroupRange keyGroupRange,
	int keyGroupPrefixBytes,
	int numberOfTransferringThreads,
	CloseableRegistry cancelStreamRegistry,
	ClassLoader userCodeClassLoader,
	Map<String, RocksDbKvStateInfo> kvStateInformation,
	StateSerializerProvider<K> keySerializerProvider,
	File instanceBasePath,
	File instanceRocksDBPath,
	DBOptions dbOptions,
	Function<String, ColumnFamilyOptions> columnFamilyOptionsFactory,
	RocksDBNativeMetricOptions nativeMetricOptions,
	MetricGroup metricGroup,
	@Nonnull Collection<KeyedStateHandle> restoreStateHandles,
	@Nonnull RocksDbTtlCompactFiltersManager ttlCompactFiltersManager,
	@Nonnegative long writeBatchSize) {
	super(keyGroupRange,
		keyGroupPrefixBytes,
		numberOfTransferringThreads,
		cancelStreamRegistry,
		userCodeClassLoader,
		kvStateInformation,
		keySerializerProvider,
		instanceBasePath,
		instanceRocksDBPath,
		dbOptions,
		columnFamilyOptionsFactory,
		nativeMetricOptions,
		metricGroup,
		restoreStateHandles,
		ttlCompactFiltersManager);
	this.operatorIdentifier = operatorIdentifier;
	this.restoredSstFiles = new TreeMap<>();
	this.lastCompletedCheckpointId = -1L;
	this.backendUID = UUID.randomUUID();
	checkArgument(writeBatchSize >= 0, "Write batch size have to be no negative.");
	this.writeBatchSize = writeBatchSize;
}
 

@Override
public void close() {
	List<ColumnFamilyOptions> columnFamilyOptions = new ArrayList<>(columnFamilyDescriptors.size() + 1);
	columnFamilyDescriptors.forEach((cfd) -> columnFamilyOptions.add(cfd.getOptions()));
	RocksDBOperationUtils.addColumnFamilyOptionsToCloseLater(columnFamilyOptions, defaultColumnFamilyHandle);
	IOUtils.closeQuietly(defaultColumnFamilyHandle);
	IOUtils.closeAllQuietly(columnFamilyHandles);
	IOUtils.closeQuietly(db);
	IOUtils.closeAllQuietly(columnFamilyOptions);
}
 

public RocksDBIncrementalRestoreOperation(
	String operatorIdentifier,
	KeyGroupRange keyGroupRange,
	int keyGroupPrefixBytes,
	int numberOfTransferringThreads,
	CloseableRegistry cancelStreamRegistry,
	ClassLoader userCodeClassLoader,
	Map<String, RocksDbKvStateInfo> kvStateInformation,
	StateSerializerProvider<K> keySerializerProvider,
	File instanceBasePath,
	File instanceRocksDBPath,
	DBOptions dbOptions,
	Function<String, ColumnFamilyOptions> columnFamilyOptionsFactory,
	RocksDBNativeMetricOptions nativeMetricOptions,
	MetricGroup metricGroup,
	@Nonnull Collection<KeyedStateHandle> restoreStateHandles,
	@Nonnull RocksDbTtlCompactFiltersManager ttlCompactFiltersManager) {
	super(keyGroupRange,
		keyGroupPrefixBytes,
		numberOfTransferringThreads,
		cancelStreamRegistry,
		userCodeClassLoader,
		kvStateInformation,
		keySerializerProvider,
		instanceBasePath,
		instanceRocksDBPath,
		dbOptions,
		columnFamilyOptionsFactory,
		nativeMetricOptions,
		metricGroup,
		restoreStateHandles,
		ttlCompactFiltersManager);
	this.operatorIdentifier = operatorIdentifier;
	this.restoredSstFiles = new TreeMap<>();
	this.lastCompletedCheckpointId = -1L;
	this.backendUID = UUID.randomUUID();
}
 

public void prepareRocksDB() throws Exception {
	String dbPath = new File(tempFolder.newFolder(), DB_INSTANCE_DIR_STRING).getAbsolutePath();
	ColumnFamilyOptions columnOptions = optionsContainer.getColumnOptions();

	ArrayList<ColumnFamilyHandle> columnFamilyHandles = new ArrayList<>(1);
	db = RocksDBOperationUtils.openDB(dbPath, Collections.emptyList(),
		columnFamilyHandles, columnOptions, optionsContainer.getDbOptions());
	defaultCFHandle = columnFamilyHandles.remove(0);
}
 

protected AbstractRocksDBRestoreOperation(
	KeyGroupRange keyGroupRange,
	int keyGroupPrefixBytes,
	int numberOfTransferringThreads,
	CloseableRegistry cancelStreamRegistry,
	ClassLoader userCodeClassLoader,
	Map<String, RocksDbKvStateInfo> kvStateInformation,
	StateSerializerProvider<K> keySerializerProvider,
	File instanceBasePath,
	File instanceRocksDBPath,
	DBOptions dbOptions,
	Function<String, ColumnFamilyOptions> columnFamilyOptionsFactory,
	RocksDBNativeMetricOptions nativeMetricOptions,
	MetricGroup metricGroup,
	@Nonnull Collection<KeyedStateHandle> stateHandles,
	@Nonnull RocksDbTtlCompactFiltersManager ttlCompactFiltersManager) {
	this.keyGroupRange = keyGroupRange;
	this.keyGroupPrefixBytes = keyGroupPrefixBytes;
	this.numberOfTransferringThreads = numberOfTransferringThreads;
	this.cancelStreamRegistry = cancelStreamRegistry;
	this.userCodeClassLoader = userCodeClassLoader;
	this.kvStateInformation = kvStateInformation;
	this.keySerializerProvider = keySerializerProvider;
	this.instanceBasePath = instanceBasePath;
	this.instanceRocksDBPath = instanceRocksDBPath;
	this.dbPath = instanceRocksDBPath.getAbsolutePath();
	this.dbOptions = dbOptions;
	this.columnFamilyOptionsFactory = columnFamilyOptionsFactory;
	this.nativeMetricOptions = nativeMetricOptions;
	this.metricGroup = metricGroup;
	this.restoreStateHandles = stateHandles;
	this.ttlCompactFiltersManager = ttlCompactFiltersManager;
	this.columnFamilyHandles = new ArrayList<>(1);
	this.columnFamilyDescriptors = Collections.emptyList();
}
 

public static void addColumnFamilyOptionsToCloseLater(
	List<ColumnFamilyOptions> columnFamilyOptions, ColumnFamilyHandle columnFamilyHandle) {
	try {
		if (columnFamilyHandle != null && columnFamilyHandle.getDescriptor() != null) {
			columnFamilyOptions.add(columnFamilyHandle.getDescriptor().getOptions());
		}
	} catch (RocksDBException e) {
		// ignore
	}
}
 

private void createAndSetCompactFilterFactory(String stateName, @Nonnull ColumnFamilyOptions options) {

		FlinkCompactionFilterFactory compactionFilterFactory =
			new FlinkCompactionFilterFactory(new TimeProviderWrapper(ttlTimeProvider), createRocksDbNativeLogger());
		//noinspection resource
		options.setCompactionFilterFactory(compactionFilterFactory);
		compactionFilterFactories.put(stateName, compactionFilterFactory);
	}
 

/**
 * Creates a state info from a new meta info to use with a k/v state.
 *
 * <p>Creates the column family for the state.
 * Sets TTL compaction filter if {@code ttlCompactFiltersManager} is not {@code null}.
 */
public static RocksDBKeyedStateBackend.RocksDbKvStateInfo createStateInfo(
	RegisteredStateMetaInfoBase metaInfoBase,
	RocksDB db,
	Function<String, ColumnFamilyOptions> columnFamilyOptionsFactory,
	@Nullable RocksDbTtlCompactFiltersManager ttlCompactFiltersManager) {

	ColumnFamilyDescriptor columnFamilyDescriptor = createColumnFamilyDescriptor(
		metaInfoBase, columnFamilyOptionsFactory, ttlCompactFiltersManager);
	return new RocksDBKeyedStateBackend.RocksDbKvStateInfo(createColumnFamily(columnFamilyDescriptor, db), metaInfoBase);
}
 

public static RocksDB openDB(
	String path,
	List<ColumnFamilyDescriptor> stateColumnFamilyDescriptors,
	List<ColumnFamilyHandle> stateColumnFamilyHandles,
	ColumnFamilyOptions columnFamilyOptions,
	DBOptions dbOptions) throws IOException {
	List<ColumnFamilyDescriptor> columnFamilyDescriptors =
		new ArrayList<>(1 + stateColumnFamilyDescriptors.size());

	// we add the required descriptor for the default CF in FIRST position, see
	// https://github.com/facebook/rocksdb/wiki/RocksJava-Basics#opening-a-database-with-column-families
	columnFamilyDescriptors.add(new ColumnFamilyDescriptor(RocksDB.DEFAULT_COLUMN_FAMILY, columnFamilyOptions));
	columnFamilyDescriptors.addAll(stateColumnFamilyDescriptors);

	RocksDB dbRef;

	try {
		dbRef = RocksDB.open(
			Preconditions.checkNotNull(dbOptions),
			Preconditions.checkNotNull(path),
			columnFamilyDescriptors,
			stateColumnFamilyHandles);
	} catch (RocksDBException e) {
		IOUtils.closeQuietly(columnFamilyOptions);
		columnFamilyDescriptors.forEach((cfd) -> IOUtils.closeQuietly(cfd.getOptions()));
		throw new IOException("Error while opening RocksDB instance.", e);
	}

	// requested + default CF
	Preconditions.checkState(1 + stateColumnFamilyDescriptors.size() == stateColumnFamilyHandles.size(),
		"Not all requested column family handles have been created");
	return dbRef;
}
 

private boolean initAndLoad(final ConfigurationManager confManager) throws RocksDBException {
    this.hasLoadFirstLogIndex = false;
    this.firstLogIndex = 1;
    final List<ColumnFamilyDescriptor> columnFamilyDescriptors = new ArrayList<>();
    final ColumnFamilyOptions cfOption = createColumnFamilyOptions();
    this.cfOptions.add(cfOption);
    // Column family to store configuration log entry.
    columnFamilyDescriptors.add(new ColumnFamilyDescriptor("Configuration".getBytes(), cfOption));
    // Default column family to store user data log entry.
    columnFamilyDescriptors.add(new ColumnFamilyDescriptor(RocksDB.DEFAULT_COLUMN_FAMILY, cfOption));

    openDB(columnFamilyDescriptors);
    load(confManager);
    return onInitLoaded();
}
 

@Override
public void shutdown() {
    this.writeLock.lock();
    try {
        // The shutdown order is matter.
        // 1. close column family handles
        closeDB();
        onShutdown();
        // 2. close column family options.
        for (final ColumnFamilyOptions opt : this.cfOptions) {
            opt.close();
        }
        // 3. close options
        this.dbOptions.close();
        if (this.statistics != null) {
            this.statistics.close();
        }
        this.writeOptions.close();
        this.totalOrderReadOptions.close();
        // 4. help gc.
        this.cfOptions.clear();
        this.dbOptions = null;
        this.statistics = null;
        this.writeOptions = null;
        this.totalOrderReadOptions = null;
        this.defaultHandle = null;
        this.confHandle = null;
        this.db = null;
        LOG.info("DB destroyed, the db path is: {}.", this.path);
    } finally {
        this.writeLock.unlock();
    }
}