org.rocksdb.DBOptions Java Examples

/**
 * Guard that {@link RocksDBResourceContainer#getDbOptions()} shares the same {@link WriteBufferManager} instance
 * if the {@link RocksDBResourceContainer} instance is initiated with {@link OpaqueMemoryResource}.
 *
 * @throws Exception if unexpected error happened.
 */
@Test
public void testGetDbOptionsWithSharedResources() throws Exception {
	final int optionNumber = 20;
	OpaqueMemoryResource<RocksDBSharedResources> sharedResources = getSharedResources();
	RocksDBResourceContainer container =
		new RocksDBResourceContainer(PredefinedOptions.DEFAULT, null, sharedResources);
	HashSet<WriteBufferManager> writeBufferManagers = new HashSet<>();
	for (int i = 0; i < optionNumber; i++) {
		DBOptions dbOptions = container.getDbOptions();
		WriteBufferManager writeBufferManager = getWriteBufferManager(dbOptions);
		writeBufferManagers.add(writeBufferManager);
	}
	assertThat(writeBufferManagers.size(), is(1));
	assertThat(writeBufferManagers.iterator().next(),
		is(sharedResources.getResourceHandle().getWriteBufferManager()));
	container.close();
}
 

@Before
public void setUp() throws Exception {
  options = new DBOptions();
  options.setCreateIfMissing(true);
  options.setCreateMissingColumnFamilies(true);

  Statistics statistics = new Statistics();
  statistics.setStatsLevel(StatsLevel.ALL);
  options = options.setStatistics(statistics);

  Set<TableConfig> configSet = new HashSet<>();
  for (String name : families) {
    TableConfig newConfig = new TableConfig(name, new ColumnFamilyOptions());
    configSet.add(newConfig);
  }
  rdbStore = new RDBStore(folder.newFolder(), options, configSet);

  codecRegistry = new CodecRegistry();

}
 

/**
 * Gets the RocksDB {@link DBOptions} to be used for RocksDB instances.
 */
public DBOptions getDbOptions() {
	// initial options from pre-defined profile
	DBOptions opt = predefinedOptions.createDBOptions(handlesToClose);
	handlesToClose.add(opt);

	// add user-defined options factory, if specified
	if (optionsFactory != null) {
		opt = optionsFactory.createDBOptions(opt, handlesToClose);
	}

	// add necessary default options
	opt = opt.setCreateIfMissing(true);

	// if sharedResources is non-null, use the write buffer manager from it.
	if (sharedResources != null) {
		opt.setWriteBufferManager(sharedResources.getResourceHandle().getWriteBufferManager());
	}

	return opt;
}
 

@Before
public void setUp() throws Exception {
  options = new DBOptions();
  options.setCreateIfMissing(true);
  options.setCreateMissingColumnFamilies(true);

  Statistics statistics = new Statistics();
  statistics.setStatsLevel(StatsLevel.ALL);
  options = options.setStatistics(statistics);
  configSet = new HashSet<>();
  for(String name : families) {
    TableConfig newConfig = new TableConfig(name, new ColumnFamilyOptions());
    configSet.add(newConfig);
  }
  rdbStore = new RDBStore(folder.newFolder(), options, configSet);
}
 

@Test
public void readFromFile() throws IOException {
  final List<String> families =
      Arrays.asList(StringUtils.bytes2String(RocksDB.DEFAULT_COLUMN_FAMILY),
          "First", "Second", "Third",
          "Fourth", "Fifth",
          "Sixth");
  final List<ColumnFamilyDescriptor> columnFamilyDescriptors =
      new ArrayList<>();
  for (String family : families) {
    columnFamilyDescriptors.add(
        new ColumnFamilyDescriptor(family.getBytes(StandardCharsets.UTF_8),
            new ColumnFamilyOptions()));
  }

  final DBOptions options = DBConfigFromFile.readFromFile(DB_FILE,
      columnFamilyDescriptors);

  // Some Random Values Defined in the test.db.ini, we verify that we are
  // able to get values that are defined in the test.db.ini.
  Assert.assertNotNull(options);
  Assert.assertEquals(551615L, options.maxManifestFileSize());
  Assert.assertEquals(1000L, options.keepLogFileNum());
  Assert.assertEquals(1048576, options.writableFileMaxBufferSize());
}
 

@Test
public void readFromFileInvalidConfig() throws IOException {
  final List<String> families =
      Arrays.asList(StringUtils.bytes2String(RocksDB.DEFAULT_COLUMN_FAMILY),
          "First", "Second", "Third",
          "Fourth", "Fifth",
          "Sixth");
  final List<ColumnFamilyDescriptor> columnFamilyDescriptors =
      new ArrayList<>();
  for (String family : families) {
    columnFamilyDescriptors.add(
        new ColumnFamilyDescriptor(family.getBytes(StandardCharsets.UTF_8),
            new ColumnFamilyOptions()));
  }

  final DBOptions options = DBConfigFromFile.readFromFile("badfile.db.ini",
      columnFamilyDescriptors);

  // This has to return a Null, since we have config defined for badfile.db
  Assert.assertNull(options);
}
 

@Before
public void setUp() throws Exception {
  options = new DBOptions();
  options.setCreateIfMissing(true);
  options.setCreateMissingColumnFamilies(true);

  Statistics statistics = new Statistics();
  statistics.setStatsLevel(StatsLevel.ALL);
  options = options.setStatistics(statistics);

  Set<TableConfig> configSet = new HashSet<>();
  for(String name : families) {
    TableConfig newConfig = new TableConfig(name, new ColumnFamilyOptions());
    configSet.add(newConfig);
  }
  rdbStore = new RDBStore(folder.newFolder(), options, configSet);
}
 

/**
 * Helper to load managed column family descriptors.
 */
private List<ColumnFamilyDescriptor> loadManagedColumnFamilies(DBOptions dbOptions) throws RocksDBException {
  final List<ColumnFamilyDescriptor> managedColumnFamilies = new ArrayList<>();
  final Options options = new Options(dbOptions, new ColumnFamilyOptions());
  List<byte[]> existing = RocksDB.listColumnFamilies(options, rocksDBBasePath);

  if (existing.isEmpty()) {
    LOG.info("No column family found. Loading default");
    managedColumnFamilies.add(getColumnFamilyDescriptor(RocksDB.DEFAULT_COLUMN_FAMILY));
  } else {
    LOG.info("Loading column families :" + existing.stream().map(String::new).collect(Collectors.toList()));
    managedColumnFamilies
        .addAll(existing.stream().map(RocksDBDAO::getColumnFamilyDescriptor).collect(Collectors.toList()));
  }
  return managedColumnFamilies;
}
 

private void registerMetrics(DBOptions dbOptions) {
  // calling DBOptions.statisticsPtr() will create a Statistics object that will collect various stats from RocksDB and
  // will introduce a 5-10% overhead
  if(!COLLECT_METRICS) {
    return;
  }

  final Statistics statistics = new Statistics();
  statistics.setStatsLevel(StatsLevel.ALL);
  dbOptions.setStatistics(statistics);
  // for now, let's add all ticker stats as gauge metrics
  for (TickerType tickerType : TickerType.values()) {
    if (tickerType == TickerType.TICKER_ENUM_MAX) {
      continue;
    }

    Metrics.newGauge(Metrics.join(METRICS_PREFIX, tickerType.name()), () -> statistics.getTickerCount(tickerType));
  }
  // Note that Statistics also contains various histogram metrics, but those cannot be easily tracked through our metrics
}
 

private static org.rocksdb.Logger createRocksDbNativeLogger() {
	if (LOG.isDebugEnabled()) {
		// options are always needed for org.rocksdb.Logger construction (no other constructor)
		// the logger level gets configured from the options in native code
		try (DBOptions opts = new DBOptions().setInfoLogLevel(InfoLogLevel.DEBUG_LEVEL)) {
			return new org.rocksdb.Logger(opts) {
				@Override
				protected void log(InfoLogLevel infoLogLevel, String logMsg) {
					LOG.debug("RocksDB filter native code log: " + logMsg);
				}
			};
		}
	} else {
		return null;
	}
}
 

@SuppressWarnings("resource")
public static KnowledgeBase getInstance(final String kbDir, final String kbMetadataPathname, final boolean readOnly) throws RocksDBException, ClassNotFoundException, IOException {
	final boolean metadataExists = new File(kbMetadataPathname).exists();
	final boolean kbDirExists = new File(kbDir).exists();
	if (metadataExists != kbDirExists) throw new IllegalArgumentException("Either both or none of the knowledge-base directory and metadata must exist");

	RocksDB.loadLibrary();
	final ColumnFamilyOptions cfOptions = new ColumnFamilyOptions().setCompressionType(CompressionType.LZ4_COMPRESSION);
	final DBOptions dbOptions = new DBOptions().setCreateIfMissing(true).setCreateMissingColumnFamilies(true);
	final List<ColumnFamilyDescriptor> cfDescriptors = Arrays.asList(new ColumnFamilyDescriptor(RocksDB.DEFAULT_COLUMN_FAMILY, cfOptions), new ColumnFamilyDescriptor(GID2URI, cfOptions), new ColumnFamilyDescriptor(URI2GID, cfOptions));

	final List<ColumnFamilyHandle> columnFamilyHandles = new ArrayList<>();
	final RocksDB db = readOnly ? RocksDB.openReadOnly(dbOptions, kbDir, cfDescriptors, columnFamilyHandles) : RocksDB.open(dbOptions, kbDir, cfDescriptors, columnFamilyHandles);

	final KnowledgeBase kb;
	if (metadataExists) {
		kb = (KnowledgeBase) BinIO.loadObject(kbMetadataPathname);
		kb.readOnly = readOnly;
		kb.callGraphDB = db;
		kb.defaultHandle = columnFamilyHandles.get(0);
		kb.gid2uriFamilyHandle = columnFamilyHandles.get(1);
		kb.uri2gidFamilyHandle = columnFamilyHandles.get(2);
	} else kb = new KnowledgeBase(db, columnFamilyHandles.get(0), columnFamilyHandles.get(1), columnFamilyHandles.get(2), kbMetadataPathname, readOnly);
	return kb;
}
 

@Test
public void testPathExceptionOnWindows() throws Exception {
	assumeTrue(OperatingSystem.isWindows());

	final File folder = TMP_DIR.newFolder();
	final File rocksDir = new File(folder, getLongString(247 - folder.getAbsolutePath().length()));

	Files.createDirectories(rocksDir.toPath());

	try (DBOptions dbOptions = new DBOptions().setCreateIfMissing(true);
		ColumnFamilyOptions colOptions = new ColumnFamilyOptions()) {

		RocksDB rocks = RocksDBOperationUtils.openDB(
				rocksDir.getAbsolutePath(),
				Collections.emptyList(),
				Collections.emptyList(),
				colOptions, dbOptions);
		rocks.close();

		// do not provoke a test failure if this passes, because some setups may actually
		// support long paths, in which case: great!
	}
	catch (IOException e) {
		assertThat(e.getMessage(), containsString("longer than the directory path length limit for Windows"));
	}
}
 

public Pair<RocksDB, Map<String, ColumnFamilyHandle>> openSupportingIndex(File supportingIndex)
    throws RocksDBException {
  List<FromBrendaDB> instances = BrendaSupportingEntries.allFromBrendaDBInstances();
  List<ColumnFamilyDescriptor> columnFamilyDescriptors = new ArrayList<>(instances.size() + 1);
  columnFamilyDescriptors.add(new ColumnFamilyDescriptor("default".getBytes()));
  for (FromBrendaDB instance : instances) {
    columnFamilyDescriptors.add(new ColumnFamilyDescriptor(instance.getColumnFamilyName().getBytes()));
  }
  List<ColumnFamilyHandle> columnFamilyHandles = new ArrayList<>(columnFamilyDescriptors.size());

  DBOptions dbOptions = new DBOptions();
  dbOptions.setCreateIfMissing(false);
  RocksDB rocksDB = RocksDB.open(dbOptions, supportingIndex.getAbsolutePath(),
      columnFamilyDescriptors, columnFamilyHandles);
  Map<String, ColumnFamilyHandle> columnFamilyHandleMap = new HashMap<>(columnFamilyHandles.size());
  // TODO: can we zip these together more easily w/ Java 8?

  for (int i = 0; i < columnFamilyDescriptors.size(); i++) {
    ColumnFamilyDescriptor cfd = columnFamilyDescriptors.get(i);
    ColumnFamilyHandle cfh = columnFamilyHandles.get(i);
    columnFamilyHandleMap.put(new String(cfd.columnFamilyName(), BrendaSupportingEntries.UTF8), cfh);
  }

  return Pair.of(rocksDB, columnFamilyHandleMap);
}
 

private static org.rocksdb.Logger createRocksDbNativeLogger() {
	if (LOG.isDebugEnabled()) {
		// options are always needed for org.rocksdb.Logger construction (no other constructor)
		// the logger level gets configured from the options in native code
		try (DBOptions opts = new DBOptions().setInfoLogLevel(InfoLogLevel.DEBUG_LEVEL)) {
			return new org.rocksdb.Logger(opts) {
				@Override
				protected void log(InfoLogLevel infoLogLevel, String logMsg) {
					LOG.debug("RocksDB filter native code log: " + logMsg);
				}
			};
		}
	} else {
		return null;
	}
}
 

/**
 * Builds a DBStore instance and returns that.
 *
 * @return DBStore
 */
public DBStore build() throws IOException {
  if(StringUtil.isBlank(dbname) || (dbPath == null)) {
    LOG.error("Required Parameter missing.");
    throw new IOException("Required parameter is missing. Please make sure "
        + "sure Path and DB name is provided.");
  }
  processDBProfile();
  processTables();
  DBOptions options = getDbProfile();

  WriteOptions writeOptions = new WriteOptions();
  writeOptions.setSync(rocksDBConfiguration.getSyncOption());


  File dbFile = getDBFile();
  if (!dbFile.getParentFile().exists()) {
    throw new IOException("The DB destination directory should exist.");
  }
  return new RDBStore(dbFile, options, writeOptions, tables, registry);
}
 

protected void initRocksDb() {
    RocksDbOptionsFactory optionFactory = new RocksDbOptionsFactory.Defaults();
    Options options = optionFactory.createOptions(null);
    DBOptions dbOptions = optionFactory.createDbOptions(null);
    ColumnFamilyOptions cfOptions = optionFactory.createColumnFamilyOptions(null);
    String optionsFactoryClass = (String) conf.get(ConfigExtension.ROCKSDB_OPTIONS_FACTORY_CLASS);
    if (optionsFactoryClass != null) {
        RocksDbOptionsFactory udfOptionFactory = (RocksDbOptionsFactory) Utils.newInstance(optionsFactoryClass);
        options = udfOptionFactory.createOptions(options);
        dbOptions = udfOptionFactory.createDbOptions(dbOptions);
        cfOptions = udfOptionFactory.createColumnFamilyOptions(cfOptions);
    }

    try {
        ttlTimeSec = ConfigExtension.getStateTtlTime(conf);
        if (ttlTimeSec > 0)
            rocksDb = TtlDB.open(options, rocksDbDir, ttlTimeSec, false);
        else
            rocksDb = RocksDB.open(options, rocksDbDir);
        // enable compaction
        rocksDb.compactRange();
        LOG.info("Finish the initialization of RocksDB");
    } catch (RocksDBException e) {
        LOG.error("Failed to open rocksdb located at " + rocksDbDir, e);
        throw new RuntimeException(e.getMessage());
    }

    lastCheckpointFiles = new HashSet<String>();
    lastCleanTime = System.currentTimeMillis();
    lastSuccessBatchId = -1;
}
 

public RocksDBFullRestoreOperation(
	KeyGroupRange keyGroupRange,
	int keyGroupPrefixBytes,
	int numberOfTransferringThreads,
	CloseableRegistry cancelStreamRegistry,
	ClassLoader userCodeClassLoader,
	Map<String, RocksDbKvStateInfo> kvStateInformation,
	StateSerializerProvider<K> keySerializerProvider,
	File instanceBasePath,
	File instanceRocksDBPath,
	DBOptions dbOptions,
	Function<String, ColumnFamilyOptions> columnFamilyOptionsFactory,
	RocksDBNativeMetricOptions nativeMetricOptions,
	MetricGroup metricGroup,
	@Nonnull Collection<KeyedStateHandle> restoreStateHandles,
	@Nonnull RocksDbTtlCompactFiltersManager ttlCompactFiltersManager,
	@Nonnegative long writeBatchSize) {
	super(
		keyGroupRange,
		keyGroupPrefixBytes,
		numberOfTransferringThreads,
		cancelStreamRegistry,
		userCodeClassLoader,
		kvStateInformation,
		keySerializerProvider,
		instanceBasePath,
		instanceRocksDBPath,
		dbOptions,
		columnFamilyOptionsFactory,
		nativeMetricOptions,
		metricGroup,
		restoreStateHandles,
		ttlCompactFiltersManager);
	checkArgument(writeBatchSize >= 0, "Write batch size have to be no negative.");
	this.writeBatchSize = writeBatchSize;
}
 

/**
 * Open an existing RocksDB index.  Use this after successful index generation to access the map of Pubchem compound
 * ids to synonyms/MeSH ids using the column family CID_TO_SYNONYMS.
 * @param pathToIndex A path to the RocksDB index directory to use.
 * @return
 * @throws RocksDBException
 */
public static Pair<RocksDB, Map<COLUMN_FAMILIES, ColumnFamilyHandle>> openExistingRocksDB(File pathToIndex)
    throws RocksDBException {
  List<ColumnFamilyDescriptor> columnFamilyDescriptors = new ArrayList<>(COLUMN_FAMILIES.values().length + 1);
  // Must also open the "default" family or RocksDB will probably choke.
  columnFamilyDescriptors.add(new ColumnFamilyDescriptor(DEFAULT_ROCKSDB_COLUMN_FAMILY.getBytes()));
  for (COLUMN_FAMILIES family : COLUMN_FAMILIES.values()) {
    columnFamilyDescriptors.add(new ColumnFamilyDescriptor(family.getName().getBytes()));
  }
  List<ColumnFamilyHandle> columnFamilyHandles = new ArrayList<>(columnFamilyDescriptors.size());

  DBOptions dbOptions = ROCKS_DB_OPEN_OPTIONS;
  dbOptions.setCreateIfMissing(false);
  RocksDB rocksDB = RocksDB.open(dbOptions, pathToIndex.getAbsolutePath(),
      columnFamilyDescriptors, columnFamilyHandles);
  Map<COLUMN_FAMILIES, ColumnFamilyHandle> columnFamilyHandleMap = new HashMap<>(COLUMN_FAMILIES.values().length);
  // TODO: can we zip these together more easily w/ Java 8?

  for (int i = 0; i < columnFamilyDescriptors.size(); i++) {
    ColumnFamilyDescriptor cfd = columnFamilyDescriptors.get(i);
    ColumnFamilyHandle cfh = columnFamilyHandles.get(i);
    String familyName = new String(cfd.columnFamilyName(), UTF8);
    COLUMN_FAMILIES descriptorFamily = COLUMN_FAMILIES.getFamilyByName(familyName);
    if (descriptorFamily == null) {
      if (!DEFAULT_ROCKSDB_COLUMN_FAMILY.equals(familyName)) {
        String msg = String.format("Found unexpected family name '%s' when trying to open RocksDB at %s",
            familyName, pathToIndex.getAbsolutePath());
        LOGGER.error(msg);
        // Crash if we don't recognize the contents of this DB.
        throw new RuntimeException(msg);
      }
      // Just skip this column family if it doesn't map to something we know but is expected.
      continue;
    }

    columnFamilyHandleMap.put(descriptorFamily, cfh);
  }

  return Pair.of(rocksDB, columnFamilyHandleMap);
}
 

public static RocksDB openDB(
	String path,
	List<ColumnFamilyDescriptor> stateColumnFamilyDescriptors,
	List<ColumnFamilyHandle> stateColumnFamilyHandles,
	ColumnFamilyOptions columnFamilyOptions,
	DBOptions dbOptions) throws IOException {
	List<ColumnFamilyDescriptor> columnFamilyDescriptors =
		new ArrayList<>(1 + stateColumnFamilyDescriptors.size());

	// we add the required descriptor for the default CF in FIRST position, see
	// https://github.com/facebook/rocksdb/wiki/RocksJava-Basics#opening-a-database-with-column-families
	columnFamilyDescriptors.add(new ColumnFamilyDescriptor(RocksDB.DEFAULT_COLUMN_FAMILY, columnFamilyOptions));
	columnFamilyDescriptors.addAll(stateColumnFamilyDescriptors);

	RocksDB dbRef;

	try {
		dbRef = RocksDB.open(
			Preconditions.checkNotNull(dbOptions),
			Preconditions.checkNotNull(path),
			columnFamilyDescriptors,
			stateColumnFamilyHandles);
	} catch (RocksDBException e) {
		IOUtils.closeQuietly(columnFamilyOptions);
		columnFamilyDescriptors.forEach((cfd) -> IOUtils.closeQuietly(cfd.getOptions()));

		// improve error reporting on Windows
		throwExceptionIfPathLengthExceededOnWindows(path, e);

		throw new IOException("Error while opening RocksDB instance.", e);
	}

	// requested + default CF
	Preconditions.checkState(1 + stateColumnFamilyDescriptors.size() == stateColumnFamilyHandles.size(),
		"Not all requested column family handles have been created");
	return dbRef;
}
 

public void init(String[] cfs) throws Exception {
    logger.debug("Checking streaming dict local store for {} at {}.", cubeName, String.join(", ", cfs));
    if (!dictPath.exists() && dictPath.mkdirs()) {
        logger.warn("Create {} failed.", dictPath);
    }
    // maybe following options is naive, should improve in the future
    try (DBOptions options = new DBOptions()
            .setCreateIfMissing(true)
            .setCreateMissingColumnFamilies(true)
            .setMaxBackgroundCompactions(5)
            .setWritableFileMaxBufferSize(400 * SizeUnit.KB)) {
        String dataPath = dictPath.getAbsolutePath() + "/data";
        List<ColumnFamilyDescriptor> columnFamilyDescriptorList = new ArrayList<>();
        List<ColumnFamilyHandle> columnFamilyHandleList = new ArrayList<>(); // to be fill in
        for (String family : cfs) {
            ColumnFamilyDescriptor columnFamilyDescriptor = new ColumnFamilyDescriptor(
                    family.getBytes(StandardCharsets.UTF_8));
            columnFamilyDescriptorList.add(columnFamilyDescriptor);
        }
        logger.debug("Try to open rocksdb {}.", dataPath);
        db = RocksDB.open(options, dataPath, columnFamilyDescriptorList, columnFamilyHandleList);
        Preconditions.checkNotNull(db, "RocksDB cannot created for some reasons.");
        for (int i = 0; i < columnFamilyHandleList.size(); i++) {
            columnFamilyHandleMap.put(new ByteArray(cfs[i].getBytes(StandardCharsets.UTF_8)),
                    columnFamilyHandleList.get(i));
        }
    } catch (Exception e) {
        logger.error("Init rocks db failed.", e);
        throw e;
    }
    logger.debug("Init local dict succeed.");
}
 

protected AbstractRocksDBRestoreOperation(
	KeyGroupRange keyGroupRange,
	int keyGroupPrefixBytes,
	int numberOfTransferringThreads,
	CloseableRegistry cancelStreamRegistry,
	ClassLoader userCodeClassLoader,
	Map<String, RocksDbKvStateInfo> kvStateInformation,
	StateSerializerProvider<K> keySerializerProvider,
	File instanceBasePath,
	File instanceRocksDBPath,
	DBOptions dbOptions,
	Function<String, ColumnFamilyOptions> columnFamilyOptionsFactory,
	RocksDBNativeMetricOptions nativeMetricOptions,
	MetricGroup metricGroup,
	@Nonnull Collection<KeyedStateHandle> stateHandles,
	@Nonnull RocksDbTtlCompactFiltersManager ttlCompactFiltersManager) {
	this.keyGroupRange = keyGroupRange;
	this.keyGroupPrefixBytes = keyGroupPrefixBytes;
	this.numberOfTransferringThreads = numberOfTransferringThreads;
	this.cancelStreamRegistry = cancelStreamRegistry;
	this.userCodeClassLoader = userCodeClassLoader;
	this.kvStateInformation = kvStateInformation;
	this.keySerializerProvider = keySerializerProvider;
	this.instanceBasePath = instanceBasePath;
	this.instanceRocksDBPath = instanceRocksDBPath;
	this.dbPath = instanceRocksDBPath.getAbsolutePath();
	this.dbOptions = dbOptions;
	this.columnFamilyOptionsFactory = columnFamilyOptionsFactory;
	this.nativeMetricOptions = nativeMetricOptions;
	this.metricGroup = metricGroup;
	this.restoreStateHandles = stateHandles;
	this.ttlCompactFiltersManager = ttlCompactFiltersManager;
	this.columnFamilyHandles = new ArrayList<>(1);
	this.columnFamilyDescriptors = Collections.emptyList();
}
 

@Override
public DBOptions createDBOptions(DBOptions currentOptions, Collection<AutoCloseable> handlesToClose) {
	if (isOptionConfigured(MAX_BACKGROUND_THREADS)) {
		currentOptions.setIncreaseParallelism(getMaxBackgroundThreads());
	}

	if (isOptionConfigured(MAX_OPEN_FILES)) {
		currentOptions.setMaxOpenFiles(getMaxOpenFiles());
	}

	return currentOptions;
}
 

@Test
public void testPredefinedOptionsEnum() {
	for (PredefinedOptions o : PredefinedOptions.values()) {
		try (DBOptions opt = o.createDBOptions()) {
			assertNotNull(opt);
		}
	}
}
 

@Test
public void testSetConfigurableOptions() throws Exception  {
	String checkpointPath = tempFolder.newFolder().toURI().toString();
	RocksDBStateBackend rocksDbBackend = new RocksDBStateBackend(checkpointPath);

	assertNull(rocksDbBackend.getOptions());

	DefaultConfigurableOptionsFactory customizedOptions = new DefaultConfigurableOptionsFactory()
		.setMaxBackgroundThreads(4)
		.setMaxOpenFiles(-1)
		.setCompactionStyle(CompactionStyle.LEVEL)
		.setUseDynamicLevelSize(true)
		.setTargetFileSizeBase("4MB")
		.setMaxSizeLevelBase("128 mb")
		.setWriteBufferSize("128 MB")
		.setMaxWriteBufferNumber(4)
		.setMinWriteBufferNumberToMerge(3)
		.setBlockSize("64KB")
		.setBlockCacheSize("512mb");

	rocksDbBackend.setOptions(customizedOptions);

	try (DBOptions dbOptions = rocksDbBackend.getDbOptions()) {
		assertEquals(-1, dbOptions.maxOpenFiles());
	}

	try (ColumnFamilyOptions columnOptions = rocksDbBackend.getColumnOptions()) {
		assertEquals(CompactionStyle.LEVEL, columnOptions.compactionStyle());
		assertTrue(columnOptions.levelCompactionDynamicLevelBytes());
		assertEquals(4 * SizeUnit.MB, columnOptions.targetFileSizeBase());
		assertEquals(128 * SizeUnit.MB, columnOptions.maxBytesForLevelBase());
		assertEquals(4, columnOptions.maxWriteBufferNumber());
		assertEquals(3, columnOptions.minWriteBufferNumberToMerge());

		BlockBasedTableConfig tableConfig = (BlockBasedTableConfig) columnOptions.tableFormatConfig();
		assertEquals(64 * SizeUnit.KB, tableConfig.blockSize());
		assertEquals(512 * SizeUnit.MB, tableConfig.blockCacheSize());
	}
}
 

public void initDb(List<Integer> list) throws Exception {
    LOG.info("Begin to init rocksDB of {}", rootDir);

    DBOptions dbOptions = null;

    List<ColumnFamilyDescriptor> columnFamilyNames = new ArrayList<>();
    columnFamilyNames.add(new ColumnFamilyDescriptor(RocksDB.DEFAULT_COLUMN_FAMILY));
    for (Integer timeout : list) {
        columnFamilyNames.add(new ColumnFamilyDescriptor(String.valueOf(timeout).getBytes()));
    }

    List<Integer> ttlValues = new ArrayList<>();
    // Default column family with infinite TTL
    // NOTE that the first must be 0, RocksDB.java API has this limitation
    ttlValues.add(0);
    // new column family with list second ttl
    ttlValues.addAll(list);

    try {
        dbOptions = new DBOptions().setCreateMissingColumnFamilies(true).setCreateIfMissing(true);
        List<ColumnFamilyHandle> columnFamilyHandleList = new ArrayList<>();
        ttlDB = TtlDB.open(dbOptions, rootDir, columnFamilyNames, columnFamilyHandleList, ttlValues, false);
        for (int i = 0; i < ttlValues.size(); i++) {
            windowHandlers.put(ttlValues.get(i), columnFamilyHandleList.get(i));
        }
        LOG.info("Successfully init rocksDB of {}", rootDir);
    } finally {

        if (dbOptions != null) {
            dbOptions.dispose();
        }
    }
}
 

/**
 * Gets the RocksDB {@link DBOptions} to be used for all RocksDB instances.
 */
public DBOptions getDbOptions() {
	// initial options from pre-defined profile
	DBOptions opt = getPredefinedOptions().createDBOptions();

	// add user-defined options factory, if specified
	if (optionsFactory != null) {
		opt = optionsFactory.createDBOptions(opt);
	}

	// add necessary default options
	opt = opt.setCreateIfMissing(true);

	return opt;
}
 

public RocksDBNoneRestoreOperation(
	KeyGroupRange keyGroupRange,
	int keyGroupPrefixBytes,
	int numberOfTransferringThreads,
	CloseableRegistry cancelStreamRegistry,
	ClassLoader userCodeClassLoader,
	Map<String, RocksDbKvStateInfo> kvStateInformation,
	StateSerializerProvider<K> keySerializerProvider,
	File instanceBasePath,
	File instanceRocksDBPath,
	DBOptions dbOptions,
	Function<String, ColumnFamilyOptions> columnFamilyOptionsFactory,
	RocksDBNativeMetricOptions nativeMetricOptions,
	MetricGroup metricGroup,
	@Nonnull Collection<KeyedStateHandle> restoreStateHandles,
	@Nonnull RocksDbTtlCompactFiltersManager ttlCompactFiltersManager
) {
	super(keyGroupRange,
		keyGroupPrefixBytes,
		numberOfTransferringThreads,
		cancelStreamRegistry,
		userCodeClassLoader,
		kvStateInformation,
		keySerializerProvider,
		instanceBasePath,
		instanceRocksDBPath,
		dbOptions,
		columnFamilyOptionsFactory,
		nativeMetricOptions,
		metricGroup,
		restoreStateHandles,
		ttlCompactFiltersManager);
}
 

@Override
public DBOptions createDBOptions(DBOptions currentOptions) {
	if (isOptionConfigured(MAX_BACKGROUND_THREADS)) {
		currentOptions.setIncreaseParallelism(getMaxBackgroundThreads());
	}

	if (isOptionConfigured(MAX_OPEN_FILES)) {
		currentOptions.setMaxOpenFiles(getMaxOpenFiles());
	}

	return currentOptions;
}
 

public RocksDBIncrementalRestoreOperation(
	String operatorIdentifier,
	KeyGroupRange keyGroupRange,
	int keyGroupPrefixBytes,
	int numberOfTransferringThreads,
	CloseableRegistry cancelStreamRegistry,
	ClassLoader userCodeClassLoader,
	Map<String, RocksDbKvStateInfo> kvStateInformation,
	StateSerializerProvider<K> keySerializerProvider,
	File instanceBasePath,
	File instanceRocksDBPath,
	DBOptions dbOptions,
	Function<String, ColumnFamilyOptions> columnFamilyOptionsFactory,
	RocksDBNativeMetricOptions nativeMetricOptions,
	MetricGroup metricGroup,
	@Nonnull Collection<KeyedStateHandle> restoreStateHandles,
	@Nonnull RocksDbTtlCompactFiltersManager ttlCompactFiltersManager) {
	super(keyGroupRange,
		keyGroupPrefixBytes,
		numberOfTransferringThreads,
		cancelStreamRegistry,
		userCodeClassLoader,
		kvStateInformation,
		keySerializerProvider,
		instanceBasePath,
		instanceRocksDBPath,
		dbOptions,
		columnFamilyOptionsFactory,
		nativeMetricOptions,
		metricGroup,
		restoreStateHandles,
		ttlCompactFiltersManager);
	this.operatorIdentifier = operatorIdentifier;
	this.restoredSstFiles = new TreeMap<>();
	this.lastCompletedCheckpointId = -1L;
	this.backendUID = UUID.randomUUID();
}
 

protected AbstractRocksDBRestoreOperation(
	KeyGroupRange keyGroupRange,
	int keyGroupPrefixBytes,
	int numberOfTransferringThreads,
	CloseableRegistry cancelStreamRegistry,
	ClassLoader userCodeClassLoader,
	Map<String, RocksDbKvStateInfo> kvStateInformation,
	StateSerializerProvider<K> keySerializerProvider,
	File instanceBasePath,
	File instanceRocksDBPath,
	DBOptions dbOptions,
	Function<String, ColumnFamilyOptions> columnFamilyOptionsFactory,
	RocksDBNativeMetricOptions nativeMetricOptions,
	MetricGroup metricGroup,
	@Nonnull Collection<KeyedStateHandle> stateHandles,
	@Nonnull RocksDbTtlCompactFiltersManager ttlCompactFiltersManager) {
	this.keyGroupRange = keyGroupRange;
	this.keyGroupPrefixBytes = keyGroupPrefixBytes;
	this.numberOfTransferringThreads = numberOfTransferringThreads;
	this.cancelStreamRegistry = cancelStreamRegistry;
	this.userCodeClassLoader = userCodeClassLoader;
	this.kvStateInformation = kvStateInformation;
	this.keySerializerProvider = keySerializerProvider;
	this.instanceBasePath = instanceBasePath;
	this.instanceRocksDBPath = instanceRocksDBPath;
	this.dbPath = instanceRocksDBPath.getAbsolutePath();
	this.dbOptions = dbOptions;
	this.columnFamilyOptionsFactory = columnFamilyOptionsFactory;
	this.nativeMetricOptions = nativeMetricOptions;
	this.metricGroup = metricGroup;
	this.restoreStateHandles = stateHandles;
	this.ttlCompactFiltersManager = ttlCompactFiltersManager;
	this.columnFamilyHandles = new ArrayList<>(1);
	this.columnFamilyDescriptors = Collections.emptyList();
}