Java Code Examples for org.rocksdb.Options#setCreateIfMissing()

RocksDBStoreImpl(String path, int ttl, int maxCompactions) {
    try {
        ensureDirectoryExists(path);

        final Options options = new Options();
        options.setCreateIfMissing(true);
        options.setMaxBackgroundCompactions(maxCompactions);
        options.setMaxOpenFiles(2);//RocksDB 会将打开的 SST 文件句柄缓存这，这样下次访问的时候就可以直接使用，而不需要重新在打开。
        options.setWriteBufferSize(4 * MB_BYTE);//4M, memtable 的最大 size
        options.setMaxWriteBufferNumber(4);//最大 memtable 的个数
        options.setLevel0FileNumCompactionTrigger(4);//当有4个未进行Compact的文件时，达到触发Compact的条件

        this.rocksDB = TtlDB.open(options, path, ttl, false);
        LOG.info("open rocks db success, path:{}, ttl:{}", path, ttl);

    } catch (Exception e) {
        LOG.error("open rocks db error, path:{}, ttl:{}", path, ttl, e);
        throw new RuntimeException(e);
    }
}
 

private static TtlDB getTtlDb() throws Exception {
    final String path = "/home/test/rocsksdb";
    ensureDirectoryExists(path);
    final Options options = new Options();
    final int ttl = (int) TimeUnit.HOURS.toSeconds(1);
    options.setCreateIfMissing(true);
    options.setMaxBackgroundCompactions(2);
    options.setMaxOpenFiles(2);//RocksDB 会将打开的 SST 文件句柄缓存这，这样下次访问的时候就可以直接使用，而不需要重新在打开。
    options.setWriteBufferSize(4194304);//4M, memtable 的最大 size
    options.setMaxWriteBufferNumber(4);//最大 memtable 的个数
    options.setLevel0FileNumCompactionTrigger(2);//当有4个未进行Compact的文件时，达到触发Compact的条件
    //options.setMaxCompactionBytes(0);
    RemoveEmptyValueCompactionFilter filter = new RemoveEmptyValueCompactionFilter();
    final TtlDB ttlDB = TtlDB.open(options, path, ttl, false);
    return ttlDB;
}
 

@Override
public BlockHeaderStorage newBlockHeaderStorage(String path) {
    RocksDBBlockHeaderStorage rocksDBBlockHeaderStorage = new RocksDBBlockHeaderStorage();
    Options options = new Options();
    options.setCreateIfMissing(true);
    options.setCreateMissingColumnFamilies(true);

    String dbPath = basePath + "/" + path;
    try {
        File dir = new File(dbPath);
        if (!dir.exists()) {
            dir.mkdirs();
        } else {
            if (!dir.isDirectory()) {
                logger.error("File {} exists and isn't dir", dbPath);
            }
        }

        RocksDB rocksDB = RocksDB.open(options, dbPath);
        rocksDBBlockHeaderStorage.setRocksDB(rocksDB);
    } catch (RocksDBException e) {
        logger.error("RocksDB open failed", e);
    }
    return rocksDBBlockHeaderStorage;
}
 

public RocksDBStoreImpl(final DynamicConfig config) {
    final String path = config.getString(ROCKS_DB_PATH_CONFIG_KEY);
    final int ttl = config.getInt(ROCKS_DB_TTL_CONFIG_KEY, DEFAULT_ROCKS_DB_TTL);

    File file = new File(path);
    if (!file.exists() || !file.isDirectory()) {
        if (!file.mkdirs()) {
            throw new RuntimeException("Failed to create RocksDB dir.");
        }
    }

    try {
        final Options options = new Options();
        options.setCreateIfMissing(true);
        this.rocksDB = TtlDB.open(options, path, ttl, false);
        LOG.info("open rocks db success, path:{}, ttl:{}", path, ttl);
    } catch (Exception e) {
        LOG.error("open rocks db error, path:{}, ttl:{}", path, ttl, e);
        throw new RuntimeException(e);
    }
}
 

@Override
public void readSnapshot(String snapshotDir) {
    try {
        // copy snapshot dir to data dir
        if (db != null) {
            db.close();
        }
        String dataDir = raftDataDir + File.separator + "rocksdb_data";
        File dataFile = new File(dataDir);
        if (dataFile.exists()) {
            FileUtils.deleteDirectory(dataFile);
        }
        File snapshotFile = new File(snapshotDir);
        if (snapshotFile.exists()) {
            FileUtils.copyDirectory(snapshotFile, dataFile);
        }
        // open rocksdb data dir
        Options options = new Options();
        options.setCreateIfMissing(true);
        db = RocksDB.open(options, dataDir);
    } catch (Exception ex) {
        LOG.warn("meet exception, msg={}", ex.getMessage());
    }
}
 

@Override
@SuppressWarnings("PMD.CloseResource")
public void setup(final BenchmarkParams b) throws IOException {
  super.setup(b);
  wkb = new UnsafeBuffer(new byte[keySize]);
  wvb = new UnsafeBuffer(new byte[valSize]);
  loadLibrary();
  final Options options = new Options();
  options.setCreateIfMissing(true);
  options.setCompressionType(NO_COMPRESSION);
  try {
    db = open(options, tmp.getAbsolutePath());
  } catch (final RocksDBException ex) {
    throw new IOException(ex);
  }
}
 

private static KeyValueStore<ByteArray, byte[]> createStore(String name) {
  final Options options = new Options();
  options.setCreateIfMissing(true);

  RocksDbKeyValueStore rocksStore =
      new RocksDbKeyValueStore(
          new File(System.getProperty("java.io.tmpdir") + "/" + name),
          options,
          new MapConfig(),
          false,
          "beamStore",
          new WriteOptions(),
          new FlushOptions(),
          new KeyValueStoreMetrics("beamStore", new MetricsRegistryMap()));

  return new SerializedKeyValueStore<>(
      rocksStore,
      new ByteArraySerdeFactory.ByteArraySerde(),
      new ByteSerde(),
      new SerializedKeyValueStoreMetrics("beamStore", new MetricsRegistryMap()));
}
 

private Options setupRocksDbOptions() {
    Options options = new Options();

    options.setCreateIfMissing(true);
    options.setUseFsync(false);
    options.setCompressionType(
            enableDbCompression
                    ? CompressionType.LZ4_COMPRESSION
                    : CompressionType.NO_COMPRESSION);

    options.setBottommostCompressionType(CompressionType.ZLIB_COMPRESSION);
    options.setMinWriteBufferNumberToMerge(MIN_WRITE_BUFFER_NUMBER_TOMERGE);
    options.setLevel0StopWritesTrigger(LEVEL0_STOP_WRITES_TRIGGER);
    options.setLevel0SlowdownWritesTrigger(LEVEL0_SLOWDOWN_WRITES_TRIGGER);
    options.setAtomicFlush(true);
    options.setWriteBufferSize(this.writeBufferSize);
    options.setRandomAccessMaxBufferSize(this.readBufferSize);
    options.setParanoidChecks(true);
    options.setMaxOpenFiles(this.maxOpenFiles);
    options.setTableFormatConfig(setupBlockBasedTableConfig());
    options.setDisableAutoCompactions(false);
    options.setIncreaseParallelism(max(1, Runtime.getRuntime().availableProcessors() / 2));

    options.setLevelCompactionDynamicLevelBytes(true);
    options.setMaxBackgroundCompactions(MAX_BACKGROUND_COMPACTIONS);
    options.setMaxBackgroundFlushes(MAX_BACKGROUND_FLUSHES);
    options.setBytesPerSync(BYTES_PER_SYNC);
    options.setCompactionPriority(CompactionPriority.MinOverlappingRatio);
    options.optimizeLevelStyleCompaction(OPTIMIZE_LEVEL_STYLE_COMPACTION);

    return options;
}
 

protected RocksDB openRocksDB(String path)
  throws Exception
{

  Options options = new Options();

  options.setIncreaseParallelism(16);
  options.setCreateIfMissing(true);
  options.setAllowMmapReads(true);
  options.setKeepLogFileNum(5);
  //options.setAllowMmapWrites(true);

  return RocksDB.open(options, path);
}
 

@Test
public void testIterate() throws Exception {
  Config config = new MapConfig();
  Options options = new Options();
  options.setCreateIfMissing(true);

  File dbDir = new File(System.getProperty("java.io.tmpdir") + "/dbStore" + System.currentTimeMillis());
  RocksDbKeyValueStore store = new RocksDbKeyValueStore(dbDir, options, config, false, "dbStore",
      new WriteOptions(), new FlushOptions(), new KeyValueStoreMetrics("dbStore", new MetricsRegistryMap()));

  ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
  String prefix = "prefix";
  for (int i = 0; i < 100; i++) {
    store.put(genKey(outputStream, prefix, i), genValue());
  }

  byte[] firstKey = genKey(outputStream, prefix, 0);
  byte[] lastKey = genKey(outputStream, prefix, 1000);
  KeyValueSnapshot<byte[], byte[]> snapshot = store.snapshot(firstKey, lastKey);
  // Make sure the cached Iterable won't change when new elements are added
  store.put(genKey(outputStream, prefix, 200), genValue());
  KeyValueIterator<byte[], byte[]> iterator = snapshot.iterator();
  assertTrue(Iterators.size(iterator) == 100);
  iterator.close();
  List<Integer> keys = new ArrayList<>();
  KeyValueIterator<byte[], byte[]> iterator2 = snapshot.iterator();
  while (iterator2.hasNext()) {
    Entry<byte[], byte[]> entry = iterator2.next();
    int key = Ints.fromByteArray(Arrays.copyOfRange(entry.getKey(), prefix.getBytes().length, entry.getKey().length));
    keys.add(key);
  }
  assertEquals(keys, IntStream.rangeClosed(0, 99).boxed().collect(Collectors.toList()));
  iterator2.close();

  outputStream.close();
  snapshot.close();
  store.close();
}
 

@Before
public void setup() {

  Config config = new MapConfig();
  Options options = new Options();
  options.setCreateIfMissing(true);

  File dbDir = new File(System.getProperty("java.io.tmpdir") + "/dbStore" + System.currentTimeMillis());
  RocksDbKeyValueStore kvStore = new RocksDbKeyValueStore(dbDir, options, config, false, "dbStore",
      new WriteOptions(), new FlushOptions(), new KeyValueStoreMetrics("dbStore", new MetricsRegistryMap()));
  KeyValueStore<String, String> serializedStore =
      new SerializedKeyValueStore<>(kvStore, stringSerde, stringSerde, serializedKeyValueStoreMetrics);
  store = new NullSafeKeyValueStore<>(serializedStore);
}
 

public RocksdbMap(String name, StorageDataType keyType, StorageDataType valueType, RocksdbStorage storage) {
    super(name, keyType, valueType, storage);

    Options options = new Options();
    options.setCreateIfMissing(true);
    BlockBasedTableConfig config = new BlockBasedTableConfig();
    options.setTableFormatConfig(config);
    dbPath = storage.getStoragePath() + File.separator + name;
    try {
        db = RocksDB.open(options, dbPath);
    } catch (RocksDBException e) {
        throw ioe(e, "Failed to open " + dbPath);
    }
    setMaxKey(lastKey());
}
 

public JRocksDB(Config config, EventLog log)
  throws Exception
{
  super(config);

  this.log = log;

  config.require("rocksdb_path");

  String path = config.get("rocksdb_path");

  RocksDB.loadLibrary();
  Options options = new Options();

  options.setIncreaseParallelism(16);
  options.setCreateIfMissing(true);
  options.setAllowMmapReads(true);
  //options.setAllowMmapWrites(true);

  sharedWriteOptions = new WriteOptions();
  sharedWriteOptions.setDisableWAL(true);
  sharedWriteOptions.setSync(false);

  db = RocksDB.open(options, path);

  open();
}
 

@Test
public void testPerf() throws Exception {
  Config config = new MapConfig();
  Options options = new Options();
  options.setCreateIfMissing(true);

  File dbDir = new File(System.getProperty("java.io.tmpdir") + "/dbStore" + System.currentTimeMillis());
  RocksDbKeyValueStore store = new RocksDbKeyValueStore(dbDir, options, config, false, "dbStore",
      new WriteOptions(), new FlushOptions(), new KeyValueStoreMetrics("dbStore", new MetricsRegistryMap()));

  ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
  String prefix = "this is the key prefix";
  Random r = new Random();
  for (int i = 0; i < 100000; i++) {
    store.put(genKey(outputStream, prefix, r.nextInt()), genValue());
  }

  byte[] firstKey = genKey(outputStream, prefix, 0);
  byte[] lastKey = genKey(outputStream, prefix, Integer.MAX_VALUE);

  long start;

  start = System.currentTimeMillis();
  KeyValueIterator<byte[], byte[]> iterator1 = store.range(firstKey, lastKey);
  long rangeTime = System.currentTimeMillis() - start;
  start = System.currentTimeMillis();
  Iterators.size(iterator1);
  long rangeIterTime = System.currentTimeMillis() - start;
  System.out.println("range iter create time: " + rangeTime + ", iterate time: " + rangeIterTime);
  iterator1.close();
  // Please comment out range query part in order to do an accurate perf test for snapshot
  start = System.currentTimeMillis();
  KeyValueSnapshot<byte[], byte[]> snapshot = store.snapshot(firstKey, lastKey);
  KeyValueIterator<byte[], byte[]> iterator2 = snapshot.iterator();
  long snapshotTime = System.currentTimeMillis() - start;
  start = System.currentTimeMillis();
  Iterators.size(iterator2);
  long snapshotIterTime = System.currentTimeMillis() - start;
  System.out.println("snapshot iter create time: " + snapshotTime + ", iterate time: " + snapshotIterTime);
  iterator2.close();
  snapshot.close();
  store.close();
}
 

private synchronized Index openDb(String collection, boolean createAllowed) throws IOException {
    if (!isValidCollectionName(collection)) {
        throw new IllegalArgumentException("Invalid collection name");
    }
    Index index = indexes.get(collection);
    if (index != null) {
        return index;
    }
    File path = new File(dataDir, collection);
    if (!createAllowed && !path.isDirectory()) {
        return null;
    }

    try {
        Options options = new Options();
        options.setCreateIfMissing(createAllowed);
        configureColumnFamily(options);

        /*
         * https://github.com/facebook/rocksdb/wiki/Memory-usage-in-RocksDB "If you're
         * certain that Get() will mostly find a key you're looking for, you can set
         * options.optimize_filters_for_hits = true. With this option turned on, we will
         * not build bloom filters on the last level, which contains 90% of the
         * database. Thus, the memory usage for bloom filters will be 10X less. You will
         * pay one IO for each Get() that doesn't find data in the database, though."
         *
         * We expect a low miss rate (~17% per Alex). This setting should greatly reduce
         * memory usage.
         */
        options.setOptimizeFiltersForHits(true);

        // this one doesn't seem to be used? see dbOptions.setMaxOpenFiles()
        options.setMaxOpenFiles(256);

        DBOptions dbOptions = new DBOptions();
        dbOptions.setCreateIfMissing(createAllowed);
        dbOptions.setMaxBackgroundCompactions(Math.min(8, Runtime.getRuntime().availableProcessors()));
        dbOptions.setAvoidFlushDuringRecovery(true);

        // if not null, replication data will be available this far back in
        // time (in seconds)
        if (replicationWindow != null) {
            dbOptions.setWalTtlSeconds(replicationWindow);
        }

        dbOptions.setMaxOpenFiles(maxOpenSstFiles);

        ColumnFamilyOptions cfOptions = new ColumnFamilyOptions();
        configureColumnFamily(cfOptions);

        List<ColumnFamilyDescriptor> cfDescriptors;
        if (FeatureFlags.experimentalAccessControl()) {
            cfDescriptors = Arrays.asList(
                    new ColumnFamilyDescriptor(RocksDB.DEFAULT_COLUMN_FAMILY, cfOptions),
                    new ColumnFamilyDescriptor("alias".getBytes(UTF_8), cfOptions),
                    new ColumnFamilyDescriptor("access-rule".getBytes(UTF_8), cfOptions),
                    new ColumnFamilyDescriptor("access-policy".getBytes(UTF_8), cfOptions)
            );
        } else {
            cfDescriptors = Arrays.asList(
                    new ColumnFamilyDescriptor(RocksDB.DEFAULT_COLUMN_FAMILY, cfOptions),
                    new ColumnFamilyDescriptor("alias".getBytes(UTF_8), cfOptions));
        }

        createColumnFamiliesIfNotExists(options, dbOptions, path.toString(), cfDescriptors);

        List<ColumnFamilyHandle> cfHandles = new ArrayList<>(cfDescriptors.size());
        RocksDB db = RocksDB.open(dbOptions, path.toString(), cfDescriptors, cfHandles);

        AccessControl accessControl = null;
        if (FeatureFlags.experimentalAccessControl()) {
            accessControl = new AccessControl(db, cfHandles.get(2), cfHandles.get(3));
        }

        index = new Index(collection, db, cfHandles.get(0), cfHandles.get(1), accessControl, scanCap, canonicalizer);
        indexes.put(collection, index);
        return index;
    } catch (RocksDBException e) {
        throw new IOException(e);
    }
}
 

public static void main(String[] args) {
    Map conf = JStormHelper.LoadConf(args[0]);
    putNum = JStormUtils.parseInt(conf.get("put.number"), 100);
    isFlush = JStormUtils.parseBoolean(conf.get("is.flush"), true);
    isCheckpoint = JStormUtils.parseBoolean(conf.get("is.checkpoint"), true);
    sleepTime = JStormUtils.parseInt(conf.get("sleep.time"), 5000);
    compactionInterval = JStormUtils.parseInt(conf.get("compaction.interval"), 30000);
    flushInterval = JStormUtils.parseInt(conf.get("flush.interval"), 3000);
    isCompaction = JStormUtils.parseBoolean(conf.get("is.compaction"), true);
    fileSizeBase = JStormUtils.parseLong(conf.get("file.size.base"), 10 * SizeUnit.KB);
    levelNum = JStormUtils.parseInt(conf.get("db.level.num"), 1);
    compactionTriggerNum = JStormUtils.parseInt(conf.get("db.compaction.trigger.num"), 4);
    LOG.info("Conf={}", conf);
    
    RocksDB db;
    File file = new File(cpPath);
    file.mkdirs();

    List<ColumnFamilyHandle> columnFamilyHandles = new ArrayList<>();
    try {
        Options options = new Options();
        options.setCreateMissingColumnFamilies(true);
        options.setCreateIfMissing(true);
        options.setTargetFileSizeBase(fileSizeBase);
        options.setMaxBackgroundFlushes(2);
        options.setMaxBackgroundCompactions(2);
        options.setCompactionStyle(CompactionStyle.LEVEL);
        options.setNumLevels(levelNum);
        options.setLevelZeroFileNumCompactionTrigger(compactionTriggerNum);

        DBOptions dbOptions = new DBOptions();
        dbOptions.setCreateMissingColumnFamilies(true);
        dbOptions.setCreateIfMissing(true);
        dbOptions.setMaxBackgroundFlushes(2);
        dbOptions.setMaxBackgroundCompactions(2);
        ColumnFamilyOptions familyOptions = new ColumnFamilyOptions();
        familyOptions.setTargetFileSizeBase(fileSizeBase);
        familyOptions.setCompactionStyle(CompactionStyle.LEVEL);
        familyOptions.setNumLevels(levelNum);
        familyOptions.setLevelZeroFileNumCompactionTrigger(compactionTriggerNum);
        List<byte[]> families = RocksDB.listColumnFamilies(options, dbPath);
        List<ColumnFamilyDescriptor> columnFamilyDescriptors = new ArrayList<>();
        if (families != null) {
            for (byte[] bytes : families) {
                columnFamilyDescriptors.add(new ColumnFamilyDescriptor(bytes, familyOptions));
                LOG.info("Load colum family of {}", new String(bytes));
            }
        }
        
        if (columnFamilyDescriptors.size() > 0) {
            db = RocksDB.open(dbOptions, dbPath, columnFamilyDescriptors, columnFamilyHandles);
        } else {
            db = RocksDB.open(options, dbPath);
        }
    } catch (RocksDBException e) {
        LOG.error("Failed to open db", e);
        return;
    }

    rocksDbTest(db, columnFamilyHandles);
    
    db.close();
}