Java Code Examples for org.rocksdb.Options#setMaxBackgroundCompactions()

RocksDBStoreImpl(String path, int ttl, int maxCompactions) {
    try {
        ensureDirectoryExists(path);

        final Options options = new Options();
        options.setCreateIfMissing(true);
        options.setMaxBackgroundCompactions(maxCompactions);
        options.setMaxOpenFiles(2);//RocksDB 会将打开的 SST 文件句柄缓存这，这样下次访问的时候就可以直接使用，而不需要重新在打开。
        options.setWriteBufferSize(4 * MB_BYTE);//4M, memtable 的最大 size
        options.setMaxWriteBufferNumber(4);//最大 memtable 的个数
        options.setLevel0FileNumCompactionTrigger(4);//当有4个未进行Compact的文件时，达到触发Compact的条件

        this.rocksDB = TtlDB.open(options, path, ttl, false);
        LOG.info("open rocks db success, path:{}, ttl:{}", path, ttl);

    } catch (Exception e) {
        LOG.error("open rocks db error, path:{}, ttl:{}", path, ttl, e);
        throw new RuntimeException(e);
    }
}
 

private static TtlDB getTtlDb() throws Exception {
    final String path = "/home/test/rocsksdb";
    ensureDirectoryExists(path);
    final Options options = new Options();
    final int ttl = (int) TimeUnit.HOURS.toSeconds(1);
    options.setCreateIfMissing(true);
    options.setMaxBackgroundCompactions(2);
    options.setMaxOpenFiles(2);//RocksDB 会将打开的 SST 文件句柄缓存这，这样下次访问的时候就可以直接使用，而不需要重新在打开。
    options.setWriteBufferSize(4194304);//4M, memtable 的最大 size
    options.setMaxWriteBufferNumber(4);//最大 memtable 的个数
    options.setLevel0FileNumCompactionTrigger(2);//当有4个未进行Compact的文件时，达到触发Compact的条件
    //options.setMaxCompactionBytes(0);
    RemoveEmptyValueCompactionFilter filter = new RemoveEmptyValueCompactionFilter();
    final TtlDB ttlDB = TtlDB.open(options, path, ttl, false);
    return ttlDB;
}
 

private Options setupRocksDbOptions() {
    Options options = new Options();

    options.setCreateIfMissing(true);
    options.setUseFsync(false);
    options.setCompressionType(
            enableDbCompression
                    ? CompressionType.LZ4_COMPRESSION
                    : CompressionType.NO_COMPRESSION);

    options.setBottommostCompressionType(CompressionType.ZLIB_COMPRESSION);
    options.setMinWriteBufferNumberToMerge(MIN_WRITE_BUFFER_NUMBER_TOMERGE);
    options.setLevel0StopWritesTrigger(LEVEL0_STOP_WRITES_TRIGGER);
    options.setLevel0SlowdownWritesTrigger(LEVEL0_SLOWDOWN_WRITES_TRIGGER);
    options.setAtomicFlush(true);
    options.setWriteBufferSize(this.writeBufferSize);
    options.setRandomAccessMaxBufferSize(this.readBufferSize);
    options.setParanoidChecks(true);
    options.setMaxOpenFiles(this.maxOpenFiles);
    options.setTableFormatConfig(setupBlockBasedTableConfig());
    options.setDisableAutoCompactions(false);
    options.setIncreaseParallelism(max(1, Runtime.getRuntime().availableProcessors() / 2));

    options.setLevelCompactionDynamicLevelBytes(true);
    options.setMaxBackgroundCompactions(MAX_BACKGROUND_COMPACTIONS);
    options.setMaxBackgroundFlushes(MAX_BACKGROUND_FLUSHES);
    options.setBytesPerSync(BYTES_PER_SYNC);
    options.setCompactionPriority(CompactionPriority.MinOverlappingRatio);
    options.optimizeLevelStyleCompaction(OPTIMIZE_LEVEL_STYLE_COMPACTION);

    return options;
}
 

public static void main(String[] args) {
    Map conf = JStormHelper.LoadConf(args[0]);
    putNum = JStormUtils.parseInt(conf.get("put.number"), 100);
    isFlush = JStormUtils.parseBoolean(conf.get("is.flush"), true);
    isCheckpoint = JStormUtils.parseBoolean(conf.get("is.checkpoint"), true);
    sleepTime = JStormUtils.parseInt(conf.get("sleep.time"), 5000);
    compactionInterval = JStormUtils.parseInt(conf.get("compaction.interval"), 30000);
    flushInterval = JStormUtils.parseInt(conf.get("flush.interval"), 3000);
    isCompaction = JStormUtils.parseBoolean(conf.get("is.compaction"), true);
    fileSizeBase = JStormUtils.parseLong(conf.get("file.size.base"), 10 * SizeUnit.KB);
    levelNum = JStormUtils.parseInt(conf.get("db.level.num"), 1);
    compactionTriggerNum = JStormUtils.parseInt(conf.get("db.compaction.trigger.num"), 4);
    LOG.info("Conf={}", conf);
    
    RocksDB db;
    File file = new File(cpPath);
    file.mkdirs();

    List<ColumnFamilyHandle> columnFamilyHandles = new ArrayList<>();
    try {
        Options options = new Options();
        options.setCreateMissingColumnFamilies(true);
        options.setCreateIfMissing(true);
        options.setTargetFileSizeBase(fileSizeBase);
        options.setMaxBackgroundFlushes(2);
        options.setMaxBackgroundCompactions(2);
        options.setCompactionStyle(CompactionStyle.LEVEL);
        options.setNumLevels(levelNum);
        options.setLevelZeroFileNumCompactionTrigger(compactionTriggerNum);

        DBOptions dbOptions = new DBOptions();
        dbOptions.setCreateMissingColumnFamilies(true);
        dbOptions.setCreateIfMissing(true);
        dbOptions.setMaxBackgroundFlushes(2);
        dbOptions.setMaxBackgroundCompactions(2);
        ColumnFamilyOptions familyOptions = new ColumnFamilyOptions();
        familyOptions.setTargetFileSizeBase(fileSizeBase);
        familyOptions.setCompactionStyle(CompactionStyle.LEVEL);
        familyOptions.setNumLevels(levelNum);
        familyOptions.setLevelZeroFileNumCompactionTrigger(compactionTriggerNum);
        List<byte[]> families = RocksDB.listColumnFamilies(options, dbPath);
        List<ColumnFamilyDescriptor> columnFamilyDescriptors = new ArrayList<>();
        if (families != null) {
            for (byte[] bytes : families) {
                columnFamilyDescriptors.add(new ColumnFamilyDescriptor(bytes, familyOptions));
                LOG.info("Load colum family of {}", new String(bytes));
            }
        }
        
        if (columnFamilyDescriptors.size() > 0) {
            db = RocksDB.open(dbOptions, dbPath, columnFamilyDescriptors, columnFamilyHandles);
        } else {
            db = RocksDB.open(options, dbPath);
        }
    } catch (RocksDBException e) {
        LOG.error("Failed to open db", e);
        return;
    }

    rocksDbTest(db, columnFamilyHandles);
    
    db.close();
}