org.rocksdb.CompactionStyle Java Examples

@Before
public void init() throws IOException {
	rocksDir = tmp.newFolder();

	// ensure the RocksDB library is loaded to a distinct location each retry
	NativeLibraryLoader.getInstance().loadLibrary(rocksDir.getAbsolutePath());

	options = new Options()
			.setCompactionStyle(CompactionStyle.LEVEL)
			.setLevelCompactionDynamicLevelBytes(true)
			.setIncreaseParallelism(4)
			.setUseFsync(false)
			.setMaxOpenFiles(-1)
			.setCreateIfMissing(true)
			.setMergeOperatorName(RocksDBKeyedStateBackend.MERGE_OPERATOR_NAME);

	writeOptions = new WriteOptions()
			.setSync(false)
			.setDisableWAL(true);
}
 

@Test
public void testPredefinedOptions() throws Exception {
	String checkpointPath = tempFolder.newFolder().toURI().toString();
	RocksDBStateBackend rocksDbBackend = new RocksDBStateBackend(checkpointPath);

	// verify that we would use PredefinedOptions.DEFAULT by default.
	assertEquals(PredefinedOptions.DEFAULT, rocksDbBackend.getPredefinedOptions());

	// verify that user could configure predefined options via flink-conf.yaml
	Configuration configuration = new Configuration();
	configuration.setString(RocksDBOptions.PREDEFINED_OPTIONS, PredefinedOptions.FLASH_SSD_OPTIMIZED.name());
	rocksDbBackend = new RocksDBStateBackend(checkpointPath);
	rocksDbBackend = rocksDbBackend.configure(configuration, getClass().getClassLoader());
	assertEquals(PredefinedOptions.FLASH_SSD_OPTIMIZED, rocksDbBackend.getPredefinedOptions());

	// verify that predefined options could be set programmatically and override pre-configured one.
	rocksDbBackend.setPredefinedOptions(PredefinedOptions.SPINNING_DISK_OPTIMIZED);
	assertEquals(PredefinedOptions.SPINNING_DISK_OPTIMIZED, rocksDbBackend.getPredefinedOptions());

	try (ColumnFamilyOptions colCreated = rocksDbBackend.getColumnOptions()) {
		assertEquals(CompactionStyle.LEVEL, colCreated.compactionStyle());
	}
}
 

@Before
public void init() throws IOException {
	rocksDir = tmp.newFolder();

	// ensure the RocksDB library is loaded to a distinct location each retry
	NativeLibraryLoader.getInstance().loadLibrary(rocksDir.getAbsolutePath());

	options = new Options()
			.setCompactionStyle(CompactionStyle.LEVEL)
			.setLevelCompactionDynamicLevelBytes(true)
			.setIncreaseParallelism(4)
			.setUseFsync(false)
			.setMaxOpenFiles(-1)
			.setCreateIfMissing(true)
			.setMergeOperatorName(RocksDBKeyedStateBackend.MERGE_OPERATOR_NAME);

	writeOptions = new WriteOptions()
			.setSync(false)
			.setDisableWAL(true);
}
 

@Test
public void testPredefinedOptions() throws Exception {
	String checkpointPath = tempFolder.newFolder().toURI().toString();
	RocksDBStateBackend rocksDbBackend = new RocksDBStateBackend(checkpointPath);

	// verify that we would use PredefinedOptions.DEFAULT by default.
	assertEquals(PredefinedOptions.DEFAULT, rocksDbBackend.getPredefinedOptions());

	// verify that user could configure predefined options via flink-conf.yaml
	Configuration configuration = new Configuration();
	configuration.setString(RocksDBOptions.PREDEFINED_OPTIONS, PredefinedOptions.FLASH_SSD_OPTIMIZED.name());
	rocksDbBackend = new RocksDBStateBackend(checkpointPath);
	rocksDbBackend = rocksDbBackend.configure(configuration, getClass().getClassLoader());
	assertEquals(PredefinedOptions.FLASH_SSD_OPTIMIZED, rocksDbBackend.getPredefinedOptions());

	// verify that predefined options could be set programmatically and override pre-configured one.
	rocksDbBackend.setPredefinedOptions(PredefinedOptions.SPINNING_DISK_OPTIMIZED);
	assertEquals(PredefinedOptions.SPINNING_DISK_OPTIMIZED, rocksDbBackend.getPredefinedOptions());

	try (ColumnFamilyOptions colCreated = rocksDbBackend.getColumnOptions()) {
		assertEquals(CompactionStyle.LEVEL, colCreated.compactionStyle());
	}
}
 

@Before
public void init() throws IOException {
	rocksDir = tmp.newFolder();

	// ensure the RocksDB library is loaded to a distinct location each retry
	NativeLibraryLoader.getInstance().loadLibrary(rocksDir.getAbsolutePath());

	options = new Options()
			.setCompactionStyle(CompactionStyle.LEVEL)
			.setLevelCompactionDynamicLevelBytes(true)
			.setIncreaseParallelism(4)
			.setUseFsync(false)
			.setMaxOpenFiles(-1)
			.setCreateIfMissing(true)
			.setMergeOperatorName(RocksDBKeyedStateBackend.MERGE_OPERATOR_NAME);

	writeOptions = new WriteOptions()
			.setSync(false)
			.setDisableWAL(true);
}
 

public RocksDBLookupBuilder(TableDesc tableDesc, String[] keyColumns, String dbPath) {
    this.tableDesc = tableDesc;
    this.encoder = new RocksDBLookupRowEncoder(tableDesc, keyColumns);
    this.dbPath = dbPath;
    this.writeBatchSize = 500;
    this.options = new Options();
    options.setCreateIfMissing(true).setWriteBufferSize(8 * SizeUnit.KB).setMaxWriteBufferNumber(3)
            .setMaxBackgroundCompactions(5).setCompressionType(CompressionType.SNAPPY_COMPRESSION)
            .setCompactionStyle(CompactionStyle.UNIVERSAL);

}
 

@Test
public void testSetConfigurableOptions() throws Exception  {
	String checkpointPath = tempFolder.newFolder().toURI().toString();
	RocksDBStateBackend rocksDbBackend = new RocksDBStateBackend(checkpointPath);

	assertNull(rocksDbBackend.getOptions());

	DefaultConfigurableOptionsFactory customizedOptions = new DefaultConfigurableOptionsFactory()
		.setMaxBackgroundThreads(4)
		.setMaxOpenFiles(-1)
		.setCompactionStyle(CompactionStyle.LEVEL)
		.setUseDynamicLevelSize(true)
		.setTargetFileSizeBase("4MB")
		.setMaxSizeLevelBase("128 mb")
		.setWriteBufferSize("128 MB")
		.setMaxWriteBufferNumber(4)
		.setMinWriteBufferNumberToMerge(3)
		.setBlockSize("64KB")
		.setBlockCacheSize("512mb");

	rocksDbBackend.setOptions(customizedOptions);

	try (DBOptions dbOptions = rocksDbBackend.getDbOptions()) {
		assertEquals(-1, dbOptions.maxOpenFiles());
	}

	try (ColumnFamilyOptions columnOptions = rocksDbBackend.getColumnOptions()) {
		assertEquals(CompactionStyle.LEVEL, columnOptions.compactionStyle());
		assertTrue(columnOptions.levelCompactionDynamicLevelBytes());
		assertEquals(4 * SizeUnit.MB, columnOptions.targetFileSizeBase());
		assertEquals(128 * SizeUnit.MB, columnOptions.maxBytesForLevelBase());
		assertEquals(4, columnOptions.maxWriteBufferNumber());
		assertEquals(3, columnOptions.minWriteBufferNumberToMerge());

		BlockBasedTableConfig tableConfig = (BlockBasedTableConfig) columnOptions.tableFormatConfig();
		assertEquals(64 * SizeUnit.KB, tableConfig.blockSize());
		assertEquals(512 * SizeUnit.MB, tableConfig.blockCacheSize());
	}
}
 

@Test
public void testSetConfigurableOptions() throws Exception  {
	DefaultConfigurableOptionsFactory customizedOptions = new DefaultConfigurableOptionsFactory()
		.setMaxBackgroundThreads(4)
		.setMaxOpenFiles(-1)
		.setCompactionStyle(CompactionStyle.LEVEL)
		.setUseDynamicLevelSize(true)
		.setTargetFileSizeBase("4MB")
		.setMaxSizeLevelBase("128 mb")
		.setWriteBufferSize("128 MB")
		.setMaxWriteBufferNumber(4)
		.setMinWriteBufferNumberToMerge(3)
		.setBlockSize("64KB")
		.setBlockCacheSize("512mb");

	try (RocksDBResourceContainer optionsContainer =
			new RocksDBResourceContainer(PredefinedOptions.DEFAULT, customizedOptions)) {

		DBOptions dbOptions = optionsContainer.getDbOptions();
		assertEquals(-1, dbOptions.maxOpenFiles());

		ColumnFamilyOptions columnOptions = optionsContainer.getColumnOptions();
		assertEquals(CompactionStyle.LEVEL, columnOptions.compactionStyle());
		assertTrue(columnOptions.levelCompactionDynamicLevelBytes());
		assertEquals(4 * SizeUnit.MB, columnOptions.targetFileSizeBase());
		assertEquals(128 * SizeUnit.MB, columnOptions.maxBytesForLevelBase());
		assertEquals(4, columnOptions.maxWriteBufferNumber());
		assertEquals(3, columnOptions.minWriteBufferNumberToMerge());

		BlockBasedTableConfig tableConfig = (BlockBasedTableConfig) columnOptions.tableFormatConfig();
		assertEquals(64 * SizeUnit.KB, tableConfig.blockSize());
		assertEquals(512 * SizeUnit.MB, tableConfig.blockCacheSize());
	}
}
 

@Test
public void testSetConfigurableOptions() throws Exception  {
	String checkpointPath = tempFolder.newFolder().toURI().toString();
	RocksDBStateBackend rocksDbBackend = new RocksDBStateBackend(checkpointPath);

	assertNull(rocksDbBackend.getOptions());

	DefaultConfigurableOptionsFactory customizedOptions = new DefaultConfigurableOptionsFactory()
		.setMaxBackgroundThreads(4)
		.setMaxOpenFiles(-1)
		.setCompactionStyle(CompactionStyle.LEVEL)
		.setUseDynamicLevelSize(true)
		.setTargetFileSizeBase("4MB")
		.setMaxSizeLevelBase("128 mb")
		.setWriteBufferSize("128 MB")
		.setMaxWriteBufferNumber(4)
		.setMinWriteBufferNumberToMerge(3)
		.setBlockSize("64KB")
		.setBlockCacheSize("512mb");

	rocksDbBackend.setOptions(customizedOptions);

	try (DBOptions dbOptions = rocksDbBackend.getDbOptions()) {
		assertEquals(-1, dbOptions.maxOpenFiles());
	}

	try (ColumnFamilyOptions columnOptions = rocksDbBackend.getColumnOptions()) {
		assertEquals(CompactionStyle.LEVEL, columnOptions.compactionStyle());
		assertTrue(columnOptions.levelCompactionDynamicLevelBytes());
		assertEquals(4 * SizeUnit.MB, columnOptions.targetFileSizeBase());
		assertEquals(128 * SizeUnit.MB, columnOptions.maxBytesForLevelBase());
		assertEquals(4, columnOptions.maxWriteBufferNumber());
		assertEquals(3, columnOptions.minWriteBufferNumberToMerge());

		BlockBasedTableConfig tableConfig = (BlockBasedTableConfig) columnOptions.tableFormatConfig();
		assertEquals(64 * SizeUnit.KB, tableConfig.blockSize());
		assertEquals(512 * SizeUnit.MB, tableConfig.blockCacheSize());
	}
}
 

public RocksDBLookupBuilder(TableDesc tableDesc, String[] keyColumns, String dbPath) {
    this.tableDesc = tableDesc;
    this.encoder = new RocksDBLookupRowEncoder(tableDesc, keyColumns);
    this.dbPath = dbPath;
    this.writeBatchSize = 500;
    this.options = new Options();
    options.setCreateIfMissing(true).setWriteBufferSize(8 * SizeUnit.KB).setMaxWriteBufferNumber(3)
            .setMaxBackgroundCompactions(5).setCompressionType(CompressionType.SNAPPY_COMPRESSION)
            .setCompactionStyle(CompactionStyle.UNIVERSAL);

}
 

@Test(timeout = 2000)
@RetryOnFailure(times = 3)
public void testRocksDbListStateAPIs() throws Exception {
	final File rocksDir = tmp.newFolder();

	// ensure the RocksDB library is loaded to a distinct location each retry
	NativeLibraryLoader.getInstance().loadLibrary(rocksDir.getAbsolutePath());

	final String key1 = "key1";
	final String key2 = "key2";
	final String value = "abcdefghijklmnopqrstuvwxyz0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ7890654321";

	final byte[] keyBytes1 = key1.getBytes(StandardCharsets.UTF_8);
	final byte[] keyBytes2 = key2.getBytes(StandardCharsets.UTF_8);
	final byte[] valueBytes = value.getBytes(StandardCharsets.UTF_8);

	// The number of values added to ListState. Can be changed for benchmarking
	final int num = 10;

	try (
		final Options options = new Options()
				.setCompactionStyle(CompactionStyle.LEVEL)
				.setLevelCompactionDynamicLevelBytes(true)
				.setIncreaseParallelism(4)
				.setUseFsync(false)
				.setMaxOpenFiles(-1)
				.setCreateIfMissing(true)
				.setMergeOperatorName(RocksDBKeyedStateBackend.MERGE_OPERATOR_NAME);

		final WriteOptions writeOptions = new WriteOptions()
				.setSync(false)
				.setDisableWAL(true);

		final RocksDB rocksDB = RocksDB.open(options, rocksDir.getAbsolutePath())) {

		// ----- add() API -----
		log.info("begin add");

		final long beginInsert1 = System.nanoTime();
		for (int i = 0; i < num; i++) {
			rocksDB.merge(writeOptions, keyBytes1, valueBytes);
		}
		final long endInsert1 = System.nanoTime();

		log.info("end add - duration: {} ns", (endInsert1 - beginInsert1));

		// ----- update() API -----

		List<byte[]> list = new ArrayList<>(num);
		for (int i = 0; i < num; i++) {
			list.add(valueBytes);
		}
		byte[] premerged = merge(list);

		log.info("begin update");

		final long beginInsert2 = System.nanoTime();
		rocksDB.merge(writeOptions, keyBytes2, premerged);
		final long endInsert2 = System.nanoTime();

		log.info("end update - duration: {} ns", (endInsert2 - beginInsert2));
	}
}
 

@Override
public CompactionStyle compactionStyle() {
    return columnFamilyOptions.compactionStyle();
}
 

@Override
public Options setCompactionStyle(final CompactionStyle compactionStyle) {
    columnFamilyOptions.setCompactionStyle(compactionStyle);
    return this;
}
 

private CompactionStyle getCompactionStyle() {
	return CompactionStyle.valueOf(getInternal(COMPACTION_STYLE.key()).toUpperCase());
}
 

public DefaultConfigurableOptionsFactory setCompactionStyle(CompactionStyle compactionStyle) {
	setInternal(COMPACTION_STYLE.key(), compactionStyle.name());
	return this;
}
 

public static void main(String[] args) {
    Map conf = JStormHelper.LoadConf(args[0]);
    putNum = JStormUtils.parseInt(conf.get("put.number"), 100);
    isFlush = JStormUtils.parseBoolean(conf.get("is.flush"), true);
    isCheckpoint = JStormUtils.parseBoolean(conf.get("is.checkpoint"), true);
    sleepTime = JStormUtils.parseInt(conf.get("sleep.time"), 5000);
    compactionInterval = JStormUtils.parseInt(conf.get("compaction.interval"), 30000);
    flushInterval = JStormUtils.parseInt(conf.get("flush.interval"), 3000);
    isCompaction = JStormUtils.parseBoolean(conf.get("is.compaction"), true);
    fileSizeBase = JStormUtils.parseLong(conf.get("file.size.base"), 10 * SizeUnit.KB);
    levelNum = JStormUtils.parseInt(conf.get("db.level.num"), 1);
    compactionTriggerNum = JStormUtils.parseInt(conf.get("db.compaction.trigger.num"), 4);
    LOG.info("Conf={}", conf);
    
    RocksDB db;
    File file = new File(cpPath);
    file.mkdirs();

    List<ColumnFamilyHandle> columnFamilyHandles = new ArrayList<>();
    try {
        Options options = new Options();
        options.setCreateMissingColumnFamilies(true);
        options.setCreateIfMissing(true);
        options.setTargetFileSizeBase(fileSizeBase);
        options.setMaxBackgroundFlushes(2);
        options.setMaxBackgroundCompactions(2);
        options.setCompactionStyle(CompactionStyle.LEVEL);
        options.setNumLevels(levelNum);
        options.setLevelZeroFileNumCompactionTrigger(compactionTriggerNum);

        DBOptions dbOptions = new DBOptions();
        dbOptions.setCreateMissingColumnFamilies(true);
        dbOptions.setCreateIfMissing(true);
        dbOptions.setMaxBackgroundFlushes(2);
        dbOptions.setMaxBackgroundCompactions(2);
        ColumnFamilyOptions familyOptions = new ColumnFamilyOptions();
        familyOptions.setTargetFileSizeBase(fileSizeBase);
        familyOptions.setCompactionStyle(CompactionStyle.LEVEL);
        familyOptions.setNumLevels(levelNum);
        familyOptions.setLevelZeroFileNumCompactionTrigger(compactionTriggerNum);
        List<byte[]> families = RocksDB.listColumnFamilies(options, dbPath);
        List<ColumnFamilyDescriptor> columnFamilyDescriptors = new ArrayList<>();
        if (families != null) {
            for (byte[] bytes : families) {
                columnFamilyDescriptors.add(new ColumnFamilyDescriptor(bytes, familyOptions));
                LOG.info("Load colum family of {}", new String(bytes));
            }
        }
        
        if (columnFamilyDescriptors.size() > 0) {
            db = RocksDB.open(dbOptions, dbPath, columnFamilyDescriptors, columnFamilyHandles);
        } else {
            db = RocksDB.open(options, dbPath);
        }
    } catch (RocksDBException e) {
        LOG.error("Failed to open db", e);
        return;
    }

    rocksDbTest(db, columnFamilyHandles);
    
    db.close();
}
 

@Test
public void testConfigurableOptionsFromConfig() throws Exception {
	Configuration configuration = new Configuration();
	DefaultConfigurableOptionsFactory defaultOptionsFactory = new DefaultConfigurableOptionsFactory();
	assertTrue(defaultOptionsFactory.configure(configuration).getConfiguredOptions().isEmpty());

	// verify illegal configuration
	{
		verifyIllegalArgument(RocksDBConfigurableOptions.MAX_BACKGROUND_THREADS, "-1");
		verifyIllegalArgument(RocksDBConfigurableOptions.MAX_WRITE_BUFFER_NUMBER, "-1");
		verifyIllegalArgument(RocksDBConfigurableOptions.MIN_WRITE_BUFFER_NUMBER_TO_MERGE, "-1");

		verifyIllegalArgument(RocksDBConfigurableOptions.TARGET_FILE_SIZE_BASE, "0KB");
		verifyIllegalArgument(RocksDBConfigurableOptions.MAX_SIZE_LEVEL_BASE, "1BB");
		verifyIllegalArgument(RocksDBConfigurableOptions.WRITE_BUFFER_SIZE, "-1KB");
		verifyIllegalArgument(RocksDBConfigurableOptions.BLOCK_SIZE, "0MB");
		verifyIllegalArgument(RocksDBConfigurableOptions.BLOCK_CACHE_SIZE, "0");

		verifyIllegalArgument(RocksDBConfigurableOptions.USE_DYNAMIC_LEVEL_SIZE, "1");

		verifyIllegalArgument(RocksDBConfigurableOptions.COMPACTION_STYLE, "LEV");
	}

	// verify legal configuration
	{
		configuration.setString(RocksDBConfigurableOptions.COMPACTION_STYLE.key(), "level");
		configuration.setString(RocksDBConfigurableOptions.USE_DYNAMIC_LEVEL_SIZE.key(), "TRUE");
		configuration.setString(RocksDBConfigurableOptions.TARGET_FILE_SIZE_BASE.key(), "8 mb");
		configuration.setString(RocksDBConfigurableOptions.MAX_SIZE_LEVEL_BASE.key(), "128MB");
		configuration.setString(RocksDBConfigurableOptions.MAX_BACKGROUND_THREADS.key(), "4");
		configuration.setString(RocksDBConfigurableOptions.MAX_WRITE_BUFFER_NUMBER.key(), "4");
		configuration.setString(RocksDBConfigurableOptions.MIN_WRITE_BUFFER_NUMBER_TO_MERGE.key(), "2");
		configuration.setString(RocksDBConfigurableOptions.WRITE_BUFFER_SIZE.key(), "64 MB");
		configuration.setString(RocksDBConfigurableOptions.BLOCK_SIZE.key(), "4 kb");
		configuration.setString(RocksDBConfigurableOptions.BLOCK_CACHE_SIZE.key(), "512 mb");

		DefaultConfigurableOptionsFactory optionsFactory = new DefaultConfigurableOptionsFactory();
		optionsFactory.configure(configuration);

		try (RocksDBResourceContainer optionsContainer =
				new RocksDBResourceContainer(PredefinedOptions.DEFAULT, optionsFactory)) {

			DBOptions dbOptions = optionsContainer.getDbOptions();
			assertEquals(-1, dbOptions.maxOpenFiles());

			ColumnFamilyOptions columnOptions = optionsContainer.getColumnOptions();
			assertEquals(CompactionStyle.LEVEL, columnOptions.compactionStyle());
			assertTrue(columnOptions.levelCompactionDynamicLevelBytes());
			assertEquals(8 * SizeUnit.MB, columnOptions.targetFileSizeBase());
			assertEquals(128 * SizeUnit.MB, columnOptions.maxBytesForLevelBase());
			assertEquals(4, columnOptions.maxWriteBufferNumber());
			assertEquals(2, columnOptions.minWriteBufferNumberToMerge());
			assertEquals(64 * SizeUnit.MB, columnOptions.writeBufferSize());

			BlockBasedTableConfig tableConfig = (BlockBasedTableConfig) columnOptions.tableFormatConfig();
			assertEquals(4 * SizeUnit.KB, tableConfig.blockSize());
			assertEquals(512 * SizeUnit.MB, tableConfig.blockCacheSize());
		}
	}
}
 

@Override
public ColumnFamilyOptions createColumnOptions(ColumnFamilyOptions currentOptions, Collection<AutoCloseable> handlesToClose) {
	return currentOptions.setCompactionStyle(CompactionStyle.UNIVERSAL);
}
 

public static Options parse(Properties properties) {
    Options options = new Options();
    options.setCompressionType(CompressionType.LZ4_COMPRESSION)
            .setCompactionStyle(CompactionStyle.LEVEL);

    BlockBasedTableConfig tableOptions = new BlockBasedTableConfig();
    options.setTableFormatConfig(tableOptions);

    for (String key : properties.stringPropertyNames()) {
        String prefix = null;
        Object configInstance = null;

        if (key.startsWith(RocksDBConfigs.OPTIONS_PREFIX)) {
            prefix = RocksDBConfigs.OPTIONS_PREFIX;
            configInstance = options;
        } else if (key.startsWith(RocksDBConfigs.TABLE_OPTIONS_PREFIX)) {
            prefix = RocksDBConfigs.TABLE_OPTIONS_PREFIX;
            configInstance = tableOptions;
        } else {
            continue;
        }

        String fieldKey = key.substring(prefix.length(), key.length());
        String value = properties.getProperty(key);

        try {
            Method setterMethod = findSetterMethod(configInstance.getClass(), fieldKey);
            if (setterMethod == null) {
                logger.warn("parse config error, method not found, key: {}, value: {}", key, value);
                continue;
            }
            setterMethod.invoke(configInstance, PropertyUtils.convert(value, setterMethod.getParameters()[0].getType()));
        } catch (Exception e) {
            logger.error("parse config error, key: {}, value: {}", key, value, e);
        }
    }

    if (properties.containsKey(RocksDBConfigs.FILTER_BITSPER_KEY)) {
        tableOptions.setFilterPolicy(new BloomFilter(
                PropertyUtils.convertInt(properties.getProperty(RocksDBConfigs.FILTER_BITSPER_KEY), 0)));
    }

    return options;
}
 

@Override
public ColumnFamilyOptions createColumnOptions(ColumnFamilyOptions currentOptions) {
	return currentOptions.setCompactionStyle(CompactionStyle.UNIVERSAL);
}
 

@Test
public void testConfigurableOptionsFromConfig() throws IOException {
	Configuration configuration = new Configuration();
	DefaultConfigurableOptionsFactory defaultOptionsFactory = new DefaultConfigurableOptionsFactory();
	assertTrue(defaultOptionsFactory.configure(configuration).getConfiguredOptions().isEmpty());

	// verify illegal configuration
	{
		verifyIllegalArgument(RocksDBConfigurableOptions.MAX_BACKGROUND_THREADS, "-1");
		verifyIllegalArgument(RocksDBConfigurableOptions.MAX_WRITE_BUFFER_NUMBER, "-1");
		verifyIllegalArgument(RocksDBConfigurableOptions.MIN_WRITE_BUFFER_NUMBER_TO_MERGE, "-1");

		verifyIllegalArgument(RocksDBConfigurableOptions.TARGET_FILE_SIZE_BASE, "0KB");
		verifyIllegalArgument(RocksDBConfigurableOptions.MAX_SIZE_LEVEL_BASE, "1BB");
		verifyIllegalArgument(RocksDBConfigurableOptions.WRITE_BUFFER_SIZE, "-1KB");
		verifyIllegalArgument(RocksDBConfigurableOptions.BLOCK_SIZE, "0MB");
		verifyIllegalArgument(RocksDBConfigurableOptions.BLOCK_CACHE_SIZE, "0");

		verifyIllegalArgument(RocksDBConfigurableOptions.USE_DYNAMIC_LEVEL_SIZE, "1");

		verifyIllegalArgument(RocksDBConfigurableOptions.COMPACTION_STYLE, "LEV");
	}

	// verify legal configuration
	{
		configuration.setString(RocksDBConfigurableOptions.COMPACTION_STYLE, "level");
		configuration.setString(RocksDBConfigurableOptions.USE_DYNAMIC_LEVEL_SIZE, "TRUE");
		configuration.setString(RocksDBConfigurableOptions.TARGET_FILE_SIZE_BASE, "8 mb");
		configuration.setString(RocksDBConfigurableOptions.MAX_SIZE_LEVEL_BASE, "128MB");
		configuration.setString(RocksDBConfigurableOptions.MAX_BACKGROUND_THREADS, "4");
		configuration.setString(RocksDBConfigurableOptions.MAX_WRITE_BUFFER_NUMBER, "4");
		configuration.setString(RocksDBConfigurableOptions.MIN_WRITE_BUFFER_NUMBER_TO_MERGE, "2");
		configuration.setString(RocksDBConfigurableOptions.WRITE_BUFFER_SIZE, "64 MB");
		configuration.setString(RocksDBConfigurableOptions.BLOCK_SIZE, "4 kb");
		configuration.setString(RocksDBConfigurableOptions.BLOCK_CACHE_SIZE, "512 mb");

		DefaultConfigurableOptionsFactory optionsFactory = new DefaultConfigurableOptionsFactory();
		optionsFactory.configure(configuration);
		String checkpointPath = tempFolder.newFolder().toURI().toString();
		RocksDBStateBackend rocksDbBackend = new RocksDBStateBackend(checkpointPath);
		rocksDbBackend.setOptions(optionsFactory);

		try (DBOptions dbOptions = rocksDbBackend.getDbOptions()) {
			assertEquals(-1, dbOptions.maxOpenFiles());
		}

		try (ColumnFamilyOptions columnOptions = rocksDbBackend.getColumnOptions()) {
			assertEquals(CompactionStyle.LEVEL, columnOptions.compactionStyle());
			assertTrue(columnOptions.levelCompactionDynamicLevelBytes());
			assertEquals(8 * SizeUnit.MB, columnOptions.targetFileSizeBase());
			assertEquals(128 * SizeUnit.MB, columnOptions.maxBytesForLevelBase());
			assertEquals(4, columnOptions.maxWriteBufferNumber());
			assertEquals(2, columnOptions.minWriteBufferNumberToMerge());
			assertEquals(64 * SizeUnit.MB, columnOptions.writeBufferSize());

			BlockBasedTableConfig tableConfig = (BlockBasedTableConfig) columnOptions.tableFormatConfig();
			assertEquals(4 * SizeUnit.KB, tableConfig.blockSize());
			assertEquals(512 * SizeUnit.MB, tableConfig.blockCacheSize());
		}
	}
}
 

@Test(timeout = 2000)
@RetryOnFailure(times = 3)
public void testRocksDbListStateAPIs() throws Exception {
	final File rocksDir = tmp.newFolder();

	// ensure the RocksDB library is loaded to a distinct location each retry
	NativeLibraryLoader.getInstance().loadLibrary(rocksDir.getAbsolutePath());

	final String key1 = "key1";
	final String key2 = "key2";
	final String value = "abcdefghijklmnopqrstuvwxyz0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ7890654321";

	final byte[] keyBytes1 = key1.getBytes(StandardCharsets.UTF_8);
	final byte[] keyBytes2 = key2.getBytes(StandardCharsets.UTF_8);
	final byte[] valueBytes = value.getBytes(StandardCharsets.UTF_8);

	// The number of values added to ListState. Can be changed for benchmarking
	final int num = 10;

	try (
		final Options options = new Options()
				.setCompactionStyle(CompactionStyle.LEVEL)
				.setLevelCompactionDynamicLevelBytes(true)
				.setIncreaseParallelism(4)
				.setUseFsync(false)
				.setMaxOpenFiles(-1)
				.setCreateIfMissing(true)
				.setMergeOperatorName(RocksDBKeyedStateBackend.MERGE_OPERATOR_NAME);

		final WriteOptions writeOptions = new WriteOptions()
				.setSync(false)
				.setDisableWAL(true);

		final RocksDB rocksDB = RocksDB.open(options, rocksDir.getAbsolutePath())) {

		// ----- add() API -----
		log.info("begin add");

		final long beginInsert1 = System.nanoTime();
		for (int i = 0; i < num; i++) {
			rocksDB.merge(writeOptions, keyBytes1, valueBytes);
		}
		final long endInsert1 = System.nanoTime();

		log.info("end add - duration: {} ns", (endInsert1 - beginInsert1));

		// ----- update() API -----

		List<byte[]> list = new ArrayList<>(num);
		for (int i = 0; i < num; i++) {
			list.add(valueBytes);
		}
		byte[] premerged = merge(list);

		log.info("begin update");

		final long beginInsert2 = System.nanoTime();
		rocksDB.merge(writeOptions, keyBytes2, premerged);
		final long endInsert2 = System.nanoTime();

		log.info("end update - duration: {} ns", (endInsert2 - beginInsert2));
	}
}
 

public DefaultConfigurableOptionsFactory setCompactionStyle(CompactionStyle compactionStyle) {
	setInternal(COMPACTION_STYLE.key(), compactionStyle.name());
	return this;
}
 

private CompactionStyle getCompactionStyle() {
	return CompactionStyle.valueOf(getInternal(COMPACTION_STYLE.key()).toUpperCase());
}
 

public static ColumnFamilyOptions getDefaultRocksDBColumnFamilyOptions() {
    final ColumnFamilyOptions opts = new ColumnFamilyOptions();

    // Flushing options:
    // write_buffer_size sets the size of a single mem_table. Once mem_table exceeds
    // this size, it is marked immutable and a new one is created.
    opts.setWriteBufferSize(64 * SizeUnit.MB);

    // Flushing options:
    // max_write_buffer_number sets the maximum number of mem_tables, both active
    // and immutable.  If the active mem_table fills up and the total number of
    // mem_tables is larger than max_write_buffer_number we stall further writes.
    // This may happen if the flush process is slower than the write rate.
    opts.setMaxWriteBufferNumber(3);

    // Flushing options:
    // min_write_buffer_number_to_merge is the minimum number of mem_tables to be
    // merged before flushing to storage. For example, if this option is set to 2,
    // immutable mem_tables are only flushed when there are two of them - a single
    // immutable mem_table will never be flushed.  If multiple mem_tables are merged
    // together, less data may be written to storage since two updates are merged to
    // a single key. However, every Get() must traverse all immutable mem_tables
    // linearly to check if the key is there. Setting this option too high may hurt
    // read performance.
    opts.setMinWriteBufferNumberToMerge(1);

    // Level Style Compaction:
    // level0_file_num_compaction_trigger -- Once level 0 reaches this number of
    // files, L0->L1 compaction is triggered. We can therefore estimate level 0
    // size in stable state as
    // write_buffer_size * min_write_buffer_number_to_merge * level0_file_num_compaction_trigger.
    opts.setLevel0FileNumCompactionTrigger(10);

    // Soft limit on number of level-0 files. We start slowing down writes at this
    // point. A value 0 means that no writing slow down will be triggered by number
    // of files in level-0.
    opts.setLevel0SlowdownWritesTrigger(20);

    // Maximum number of level-0 files.  We stop writes at this point.
    opts.setLevel0StopWritesTrigger(40);

    // Level Style Compaction:
    // max_bytes_for_level_base and max_bytes_for_level_multiplier
    //  -- max_bytes_for_level_base is total size of level 1. As mentioned, we
    // recommend that this be around the size of level 0. Each subsequent level
    // is max_bytes_for_level_multiplier larger than previous one. The default
    // is 10 and we do not recommend changing that.
    opts.setMaxBytesForLevelBase(512 * SizeUnit.MB);

    // Level Style Compaction:
    // target_file_size_base and target_file_size_multiplier
    //  -- Files in level 1 will have target_file_size_base bytes. Each next
    // level's file size will be target_file_size_multiplier bigger than previous
    // one. However, by default target_file_size_multiplier is 1, so files in all
    // L1..LMax levels are equal. Increasing target_file_size_base will reduce total
    // number of database files, which is generally a good thing. We recommend setting
    // target_file_size_base to be max_bytes_for_level_base / 10, so that there are
    // 10 files in level 1.
    opts.setTargetFileSizeBase(64 * SizeUnit.MB);

    // If prefix_extractor is set and memtable_prefix_bloom_size_ratio is not 0,
    // create prefix bloom for memtable with the size of
    // write_buffer_size * memtable_prefix_bloom_size_ratio.
    // If it is larger than 0.25, it is santinized to 0.25.
    opts.setMemtablePrefixBloomSizeRatio(0.125);

    // Seems like the rocksDB jni for Windows doesn't come linked with any of the
    // compression type
    if (!Platform.isWindows()) {
        opts.setCompressionType(CompressionType.LZ4_COMPRESSION) //
            .setCompactionStyle(CompactionStyle.LEVEL) //
            .optimizeLevelStyleCompaction();
    }

    // https://github.com/facebook/rocksdb/pull/5744
    opts.setForceConsistencyChecks(true);

    return opts;
}
 

@Override
public ColumnFamilyOptions createColumnOptions(ColumnFamilyOptions currentOptions) {
	return currentOptions.setCompactionStyle(CompactionStyle.UNIVERSAL);
}
 

@Test
public void testConfigurableOptionsFromConfig() throws IOException {
	Configuration configuration = new Configuration();
	DefaultConfigurableOptionsFactory defaultOptionsFactory = new DefaultConfigurableOptionsFactory();
	assertTrue(defaultOptionsFactory.configure(configuration).getConfiguredOptions().isEmpty());

	// verify illegal configuration
	{
		verifyIllegalArgument(RocksDBConfigurableOptions.MAX_BACKGROUND_THREADS, "-1");
		verifyIllegalArgument(RocksDBConfigurableOptions.MAX_WRITE_BUFFER_NUMBER, "-1");
		verifyIllegalArgument(RocksDBConfigurableOptions.MIN_WRITE_BUFFER_NUMBER_TO_MERGE, "-1");

		verifyIllegalArgument(RocksDBConfigurableOptions.TARGET_FILE_SIZE_BASE, "0KB");
		verifyIllegalArgument(RocksDBConfigurableOptions.MAX_SIZE_LEVEL_BASE, "1BB");
		verifyIllegalArgument(RocksDBConfigurableOptions.WRITE_BUFFER_SIZE, "-1KB");
		verifyIllegalArgument(RocksDBConfigurableOptions.BLOCK_SIZE, "0MB");
		verifyIllegalArgument(RocksDBConfigurableOptions.BLOCK_CACHE_SIZE, "0");

		verifyIllegalArgument(RocksDBConfigurableOptions.USE_DYNAMIC_LEVEL_SIZE, "1");

		verifyIllegalArgument(RocksDBConfigurableOptions.COMPACTION_STYLE, "LEV");
	}

	// verify legal configuration
	{
		configuration.setString(RocksDBConfigurableOptions.COMPACTION_STYLE, "level");
		configuration.setString(RocksDBConfigurableOptions.USE_DYNAMIC_LEVEL_SIZE, "TRUE");
		configuration.setString(RocksDBConfigurableOptions.TARGET_FILE_SIZE_BASE, "8 mb");
		configuration.setString(RocksDBConfigurableOptions.MAX_SIZE_LEVEL_BASE, "128MB");
		configuration.setString(RocksDBConfigurableOptions.MAX_BACKGROUND_THREADS, "4");
		configuration.setString(RocksDBConfigurableOptions.MAX_WRITE_BUFFER_NUMBER, "4");
		configuration.setString(RocksDBConfigurableOptions.MIN_WRITE_BUFFER_NUMBER_TO_MERGE, "2");
		configuration.setString(RocksDBConfigurableOptions.WRITE_BUFFER_SIZE, "64 MB");
		configuration.setString(RocksDBConfigurableOptions.BLOCK_SIZE, "4 kb");
		configuration.setString(RocksDBConfigurableOptions.BLOCK_CACHE_SIZE, "512 mb");

		DefaultConfigurableOptionsFactory optionsFactory = new DefaultConfigurableOptionsFactory();
		optionsFactory.configure(configuration);
		String checkpointPath = tempFolder.newFolder().toURI().toString();
		RocksDBStateBackend rocksDbBackend = new RocksDBStateBackend(checkpointPath);
		rocksDbBackend.setOptions(optionsFactory);

		try (DBOptions dbOptions = rocksDbBackend.getDbOptions()) {
			assertEquals(-1, dbOptions.maxOpenFiles());
		}

		try (ColumnFamilyOptions columnOptions = rocksDbBackend.getColumnOptions()) {
			assertEquals(CompactionStyle.LEVEL, columnOptions.compactionStyle());
			assertTrue(columnOptions.levelCompactionDynamicLevelBytes());
			assertEquals(8 * SizeUnit.MB, columnOptions.targetFileSizeBase());
			assertEquals(128 * SizeUnit.MB, columnOptions.maxBytesForLevelBase());
			assertEquals(4, columnOptions.maxWriteBufferNumber());
			assertEquals(2, columnOptions.minWriteBufferNumberToMerge());
			assertEquals(64 * SizeUnit.MB, columnOptions.writeBufferSize());

			BlockBasedTableConfig tableConfig = (BlockBasedTableConfig) columnOptions.tableFormatConfig();
			assertEquals(4 * SizeUnit.KB, tableConfig.blockSize());
			assertEquals(512 * SizeUnit.MB, tableConfig.blockCacheSize());
		}
	}
}
 

@Test(timeout = 2000)
@RetryOnFailure(times = 3)
public void testRocksDbListStateAPIs() throws Exception {
	final File rocksDir = tmp.newFolder();

	// ensure the RocksDB library is loaded to a distinct location each retry
	NativeLibraryLoader.getInstance().loadLibrary(rocksDir.getAbsolutePath());

	final String key1 = "key1";
	final String key2 = "key2";
	final String value = "abcdefghijklmnopqrstuvwxyz0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ7890654321";

	final byte[] keyBytes1 = key1.getBytes(StandardCharsets.UTF_8);
	final byte[] keyBytes2 = key2.getBytes(StandardCharsets.UTF_8);
	final byte[] valueBytes = value.getBytes(StandardCharsets.UTF_8);

	// The number of values added to ListState. Can be changed for benchmarking
	final int num = 10;

	try (
		final Options options = new Options()
				.setCompactionStyle(CompactionStyle.LEVEL)
				.setLevelCompactionDynamicLevelBytes(true)
				.setIncreaseParallelism(4)
				.setUseFsync(false)
				.setMaxOpenFiles(-1)
				.setCreateIfMissing(true)
				.setMergeOperatorName(RocksDBKeyedStateBackend.MERGE_OPERATOR_NAME);

		final WriteOptions writeOptions = new WriteOptions()
				.setSync(false)
				.setDisableWAL(true);

		final RocksDB rocksDB = RocksDB.open(options, rocksDir.getAbsolutePath())) {

		// ----- add() API -----
		log.info("begin add");

		final long beginInsert1 = System.nanoTime();
		for (int i = 0; i < num; i++) {
			rocksDB.merge(writeOptions, keyBytes1, valueBytes);
		}
		final long endInsert1 = System.nanoTime();

		log.info("end add - duration: {} ns", (endInsert1 - beginInsert1));

		// ----- update() API -----

		List<byte[]> list = new ArrayList<>(num);
		for (int i = 0; i < num; i++) {
			list.add(valueBytes);
		}
		byte[] premerged = merge(list);

		log.info("begin update");

		final long beginInsert2 = System.nanoTime();
		rocksDB.merge(writeOptions, keyBytes2, premerged);
		final long endInsert2 = System.nanoTime();

		log.info("end update - duration: {} ns", (endInsert2 - beginInsert2));
	}
}
 

public DefaultConfigurableOptionsFactory setCompactionStyle(CompactionStyle compactionStyle) {
	setInternal(COMPACTION_STYLE.key(), compactionStyle.name());
	return this;
}
 

private CompactionStyle getCompactionStyle() {
	return CompactionStyle.valueOf(getInternal(COMPACTION_STYLE.key()).toUpperCase());
}