org.rocksdb.CompressionType Java Examples

@SuppressWarnings("resource")
public static KnowledgeBase getInstance(final String kbDir, final String kbMetadataPathname, final boolean readOnly) throws RocksDBException, ClassNotFoundException, IOException {
	final boolean metadataExists = new File(kbMetadataPathname).exists();
	final boolean kbDirExists = new File(kbDir).exists();
	if (metadataExists != kbDirExists) throw new IllegalArgumentException("Either both or none of the knowledge-base directory and metadata must exist");

	RocksDB.loadLibrary();
	final ColumnFamilyOptions cfOptions = new ColumnFamilyOptions().setCompressionType(CompressionType.LZ4_COMPRESSION);
	final DBOptions dbOptions = new DBOptions().setCreateIfMissing(true).setCreateMissingColumnFamilies(true);
	final List<ColumnFamilyDescriptor> cfDescriptors = Arrays.asList(new ColumnFamilyDescriptor(RocksDB.DEFAULT_COLUMN_FAMILY, cfOptions), new ColumnFamilyDescriptor(GID2URI, cfOptions), new ColumnFamilyDescriptor(URI2GID, cfOptions));

	final List<ColumnFamilyHandle> columnFamilyHandles = new ArrayList<>();
	final RocksDB db = readOnly ? RocksDB.openReadOnly(dbOptions, kbDir, cfDescriptors, columnFamilyHandles) : RocksDB.open(dbOptions, kbDir, cfDescriptors, columnFamilyHandles);

	final KnowledgeBase kb;
	if (metadataExists) {
		kb = (KnowledgeBase) BinIO.loadObject(kbMetadataPathname);
		kb.readOnly = readOnly;
		kb.callGraphDB = db;
		kb.defaultHandle = columnFamilyHandles.get(0);
		kb.gid2uriFamilyHandle = columnFamilyHandles.get(1);
		kb.uri2gidFamilyHandle = columnFamilyHandles.get(2);
	} else kb = new KnowledgeBase(db, columnFamilyHandles.get(0), columnFamilyHandles.get(1), columnFamilyHandles.get(2), kbMetadataPathname, readOnly);
	return kb;
}
 

public RocksDBLookupBuilder(TableDesc tableDesc, String[] keyColumns, String dbPath) {
    this.tableDesc = tableDesc;
    this.encoder = new RocksDBLookupRowEncoder(tableDesc, keyColumns);
    this.dbPath = dbPath;
    this.writeBatchSize = 500;
    this.options = new Options();
    options.setCreateIfMissing(true).setWriteBufferSize(8 * SizeUnit.KB).setMaxWriteBufferNumber(3)
            .setMaxBackgroundCompactions(5).setCompressionType(CompressionType.SNAPPY_COMPRESSION)
            .setCompactionStyle(CompactionStyle.UNIVERSAL);

}
 

private Options setupRocksDbOptions() {
    Options options = new Options();

    options.setCreateIfMissing(true);
    options.setUseFsync(false);
    options.setCompressionType(
            enableDbCompression
                    ? CompressionType.LZ4_COMPRESSION
                    : CompressionType.NO_COMPRESSION);

    options.setBottommostCompressionType(CompressionType.ZLIB_COMPRESSION);
    options.setMinWriteBufferNumberToMerge(MIN_WRITE_BUFFER_NUMBER_TOMERGE);
    options.setLevel0StopWritesTrigger(LEVEL0_STOP_WRITES_TRIGGER);
    options.setLevel0SlowdownWritesTrigger(LEVEL0_SLOWDOWN_WRITES_TRIGGER);
    options.setAtomicFlush(true);
    options.setWriteBufferSize(this.writeBufferSize);
    options.setRandomAccessMaxBufferSize(this.readBufferSize);
    options.setParanoidChecks(true);
    options.setMaxOpenFiles(this.maxOpenFiles);
    options.setTableFormatConfig(setupBlockBasedTableConfig());
    options.setDisableAutoCompactions(false);
    options.setIncreaseParallelism(max(1, Runtime.getRuntime().availableProcessors() / 2));

    options.setLevelCompactionDynamicLevelBytes(true);
    options.setMaxBackgroundCompactions(MAX_BACKGROUND_COMPACTIONS);
    options.setMaxBackgroundFlushes(MAX_BACKGROUND_FLUSHES);
    options.setBytesPerSync(BYTES_PER_SYNC);
    options.setCompactionPriority(CompactionPriority.MinOverlappingRatio);
    options.optimizeLevelStyleCompaction(OPTIMIZE_LEVEL_STYLE_COMPACTION);

    return options;
}
 

public RocksDBLookupBuilder(TableDesc tableDesc, String[] keyColumns, String dbPath) {
    this.tableDesc = tableDesc;
    this.encoder = new RocksDBLookupRowEncoder(tableDesc, keyColumns);
    this.dbPath = dbPath;
    this.writeBatchSize = 500;
    this.options = new Options();
    options.setCreateIfMissing(true).setWriteBufferSize(8 * SizeUnit.KB).setMaxWriteBufferNumber(3)
            .setMaxBackgroundCompactions(5).setCompressionType(CompressionType.SNAPPY_COMPRESSION)
            .setCompactionStyle(CompactionStyle.UNIVERSAL);

}
 

public static Options parse(Properties properties) {
    Options options = new Options();
    options.setCompressionType(CompressionType.LZ4_COMPRESSION)
            .setCompactionStyle(CompactionStyle.LEVEL);

    BlockBasedTableConfig tableOptions = new BlockBasedTableConfig();
    options.setTableFormatConfig(tableOptions);

    for (String key : properties.stringPropertyNames()) {
        String prefix = null;
        Object configInstance = null;

        if (key.startsWith(RocksDBConfigs.OPTIONS_PREFIX)) {
            prefix = RocksDBConfigs.OPTIONS_PREFIX;
            configInstance = options;
        } else if (key.startsWith(RocksDBConfigs.TABLE_OPTIONS_PREFIX)) {
            prefix = RocksDBConfigs.TABLE_OPTIONS_PREFIX;
            configInstance = tableOptions;
        } else {
            continue;
        }

        String fieldKey = key.substring(prefix.length(), key.length());
        String value = properties.getProperty(key);

        try {
            Method setterMethod = findSetterMethod(configInstance.getClass(), fieldKey);
            if (setterMethod == null) {
                logger.warn("parse config error, method not found, key: {}, value: {}", key, value);
                continue;
            }
            setterMethod.invoke(configInstance, PropertyUtils.convert(value, setterMethod.getParameters()[0].getType()));
        } catch (Exception e) {
            logger.error("parse config error, key: {}, value: {}", key, value, e);
        }
    }

    if (properties.containsKey(RocksDBConfigs.FILTER_BITSPER_KEY)) {
        tableOptions.setFilterPolicy(new BloomFilter(
                PropertyUtils.convertInt(properties.getProperty(RocksDBConfigs.FILTER_BITSPER_KEY), 0)));
    }

    return options;
}
 

public static ColumnFamilyOptions getDefaultRocksDBColumnFamilyOptions() {
    final ColumnFamilyOptions opts = new ColumnFamilyOptions();

    // Flushing options:
    // write_buffer_size sets the size of a single mem_table. Once mem_table exceeds
    // this size, it is marked immutable and a new one is created.
    opts.setWriteBufferSize(64 * SizeUnit.MB);

    // Flushing options:
    // max_write_buffer_number sets the maximum number of mem_tables, both active
    // and immutable.  If the active mem_table fills up and the total number of
    // mem_tables is larger than max_write_buffer_number we stall further writes.
    // This may happen if the flush process is slower than the write rate.
    opts.setMaxWriteBufferNumber(3);

    // Flushing options:
    // min_write_buffer_number_to_merge is the minimum number of mem_tables to be
    // merged before flushing to storage. For example, if this option is set to 2,
    // immutable mem_tables are only flushed when there are two of them - a single
    // immutable mem_table will never be flushed.  If multiple mem_tables are merged
    // together, less data may be written to storage since two updates are merged to
    // a single key. However, every Get() must traverse all immutable mem_tables
    // linearly to check if the key is there. Setting this option too high may hurt
    // read performance.
    opts.setMinWriteBufferNumberToMerge(1);

    // Level Style Compaction:
    // level0_file_num_compaction_trigger -- Once level 0 reaches this number of
    // files, L0->L1 compaction is triggered. We can therefore estimate level 0
    // size in stable state as
    // write_buffer_size * min_write_buffer_number_to_merge * level0_file_num_compaction_trigger.
    opts.setLevel0FileNumCompactionTrigger(10);

    // Soft limit on number of level-0 files. We start slowing down writes at this
    // point. A value 0 means that no writing slow down will be triggered by number
    // of files in level-0.
    opts.setLevel0SlowdownWritesTrigger(20);

    // Maximum number of level-0 files.  We stop writes at this point.
    opts.setLevel0StopWritesTrigger(40);

    // Level Style Compaction:
    // max_bytes_for_level_base and max_bytes_for_level_multiplier
    //  -- max_bytes_for_level_base is total size of level 1. As mentioned, we
    // recommend that this be around the size of level 0. Each subsequent level
    // is max_bytes_for_level_multiplier larger than previous one. The default
    // is 10 and we do not recommend changing that.
    opts.setMaxBytesForLevelBase(512 * SizeUnit.MB);

    // Level Style Compaction:
    // target_file_size_base and target_file_size_multiplier
    //  -- Files in level 1 will have target_file_size_base bytes. Each next
    // level's file size will be target_file_size_multiplier bigger than previous
    // one. However, by default target_file_size_multiplier is 1, so files in all
    // L1..LMax levels are equal. Increasing target_file_size_base will reduce total
    // number of database files, which is generally a good thing. We recommend setting
    // target_file_size_base to be max_bytes_for_level_base / 10, so that there are
    // 10 files in level 1.
    opts.setTargetFileSizeBase(64 * SizeUnit.MB);

    // If prefix_extractor is set and memtable_prefix_bloom_size_ratio is not 0,
    // create prefix bloom for memtable with the size of
    // write_buffer_size * memtable_prefix_bloom_size_ratio.
    // If it is larger than 0.25, it is santinized to 0.25.
    opts.setMemtablePrefixBloomSizeRatio(0.125);

    // Seems like the rocksDB jni for Windows doesn't come linked with any of the
    // compression type
    if (!Platform.isWindows()) {
        opts.setCompressionType(CompressionType.LZ4_COMPRESSION) //
            .setCompactionStyle(CompactionStyle.LEVEL) //
            .optimizeLevelStyleCompaction();
    }

    // https://github.com/facebook/rocksdb/pull/5744
    opts.setForceConsistencyChecks(true);

    return opts;
}
 

@Override
public void open() {
    options = new Options().setCreateIfMissing(true);
    options.setStatsDumpPeriodSec(1000000);

    options.setWriteBufferSize(128l * 1024 * 1024);
    options.setMaxWriteBufferNumber(3);
    options.setMaxBackgroundCompactions(20);

    Env env = Env.getDefault();
    env.setBackgroundThreads(20, Env.COMPACTION_POOL);
    options.setEnv(env);

    // max size of L1 10 MB.
    options.setMaxBytesForLevelBase(10485760);
    options.setTargetFileSizeBase(67108864);

    options.setLevel0FileNumCompactionTrigger(4);
    options.setLevel0SlowdownWritesTrigger(6);
    options.setLevel0StopWritesTrigger(12);
    options.setNumLevels(6);
    options.setDeleteObsoleteFilesPeriodMicros(300000000);


    options.setAllowMmapReads(false);
    options.setCompressionType(CompressionType.SNAPPY_COMPRESSION);


    System.out.printf("maxBackgroundCompactions %d \n", options.maxBackgroundCompactions());
    System.out.printf("minWriteBufferNumberToMerge %d \n", options.minWriteBufferNumberToMerge());
    System.out.printf("maxWriteBufferNumberToMaintain %d \n", options.maxWriteBufferNumberToMaintain());


    System.out.printf("level0FileNumCompactionTrigger %d \n", options.level0FileNumCompactionTrigger());
    System.out.printf("maxBytesForLevelBase %d \n", options.maxBytesForLevelBase());
    System.out.printf("maxBytesForLevelMultiplier %f \n", options.maxBytesForLevelMultiplier());
    System.out.printf("targetFileSizeBase %d \n", options.targetFileSizeBase());
    System.out.printf("targetFileSizeMultiplier %d \n", options.targetFileSizeMultiplier());

    List<CompressionType> compressionLevels =
        Arrays.asList(
            CompressionType.NO_COMPRESSION,
            CompressionType.NO_COMPRESSION,
            CompressionType.SNAPPY_COMPRESSION,
            CompressionType.SNAPPY_COMPRESSION,
            CompressionType.SNAPPY_COMPRESSION,
            CompressionType.SNAPPY_COMPRESSION
        );

    options.setCompressionPerLevel(compressionLevels);

    System.out.printf("compressionPerLevel %s \n", options.compressionPerLevel());
    System.out.printf("numLevels %s \n", options.numLevels());

    writeOptions = new WriteOptions();
    writeOptions.setDisableWAL(true);

    System.out.printf("WAL is disabled - %s \n", writeOptions.disableWAL());

    try {
        db = RocksDB.open(options, dbDirectory.getPath());
    } catch (RocksDBException e) {
        e.printStackTrace();
    }

}
 

public CompressionType getCompressionType() {
  return compressionType;
}
 

public CompressionType getBottomMostCompressionType() {
  return bottomMostCompressionType;
}
 

@Override
public CompressionType compressionType() {
    return columnFamilyOptions.compressionType();
}
 

@Override
public Options setCompressionPerLevel(final List<CompressionType> compressionLevels) {
    columnFamilyOptions.setCompressionPerLevel(compressionLevels);
    return this;
}
 

@Override
public List<CompressionType> compressionPerLevel() {
    return columnFamilyOptions.compressionPerLevel();
}
 

@Override
public Options setCompressionType(final CompressionType compressionType) {
    columnFamilyOptions.setCompressionType(compressionType);
    return this;
}
 

@Override
public Options setBottommostCompressionType(final CompressionType bottommostCompressionType) {
    columnFamilyOptions.setBottommostCompressionType(bottommostCompressionType);
    return this;
}
 

@Override
public CompressionType bottommostCompressionType() {
    return columnFamilyOptions.bottommostCompressionType();
}