org.rocksdb.NativeLibraryLoader Java Examples

@Before
public void init() throws IOException {
	rocksDir = tmp.newFolder();

	// ensure the RocksDB library is loaded to a distinct location each retry
	NativeLibraryLoader.getInstance().loadLibrary(rocksDir.getAbsolutePath());

	options = new Options()
			.setCompactionStyle(CompactionStyle.LEVEL)
			.setLevelCompactionDynamicLevelBytes(true)
			.setIncreaseParallelism(4)
			.setUseFsync(false)
			.setMaxOpenFiles(-1)
			.setCreateIfMissing(true)
			.setMergeOperatorName(RocksDBKeyedStateBackend.MERGE_OPERATOR_NAME);

	writeOptions = new WriteOptions()
			.setSync(false)
			.setDisableWAL(true);
}
 

@Before
public void init() throws IOException {
	rocksDir = tmp.newFolder();

	// ensure the RocksDB library is loaded to a distinct location each retry
	NativeLibraryLoader.getInstance().loadLibrary(rocksDir.getAbsolutePath());

	options = new Options()
			.setCompactionStyle(CompactionStyle.LEVEL)
			.setLevelCompactionDynamicLevelBytes(true)
			.setIncreaseParallelism(4)
			.setUseFsync(false)
			.setMaxOpenFiles(-1)
			.setCreateIfMissing(true)
			.setMergeOperatorName(RocksDBKeyedStateBackend.MERGE_OPERATOR_NAME);

	writeOptions = new WriteOptions()
			.setSync(false)
			.setDisableWAL(true);
}
 

@Before
public void init() throws IOException {
	rocksDir = tmp.newFolder();

	// ensure the RocksDB library is loaded to a distinct location each retry
	NativeLibraryLoader.getInstance().loadLibrary(rocksDir.getAbsolutePath());

	options = new Options()
			.setCompactionStyle(CompactionStyle.LEVEL)
			.setLevelCompactionDynamicLevelBytes(true)
			.setIncreaseParallelism(4)
			.setUseFsync(false)
			.setMaxOpenFiles(-1)
			.setCreateIfMissing(true)
			.setMergeOperatorName(RocksDBKeyedStateBackend.MERGE_OPERATOR_NAME);

	writeOptions = new WriteOptions()
			.setSync(false)
			.setDisableWAL(true);
}
 

private void ensureRocksDBIsLoaded(String tempDirectory) throws IOException {
	synchronized (RocksDBStateBackend.class) {
		if (!rocksDbInitialized) {

			final File tempDirParent = new File(tempDirectory).getAbsoluteFile();
			LOG.info("Attempting to load RocksDB native library and store it under '{}'", tempDirParent);

			Throwable lastException = null;
			for (int attempt = 1; attempt <= ROCKSDB_LIB_LOADING_ATTEMPTS; attempt++) {
				try {
					// when multiple instances of this class and RocksDB exist in different
					// class loaders, then we can see the following exception:
					// "java.lang.UnsatisfiedLinkError: Native Library /path/to/temp/dir/librocksdbjni-linux64.so
					// already loaded in another class loader"

					// to avoid that, we need to add a random element to the library file path
					// (I know, seems like an unnecessary hack, since the JVM obviously can handle multiple
					//  instances of the same JNI library being loaded in different class loaders, but
					//  apparently not when coming from the same file path, so there we go)

					final File rocksLibFolder = new File(tempDirParent, "rocksdb-lib-" + new AbstractID());

					// make sure the temp path exists
					LOG.debug("Attempting to create RocksDB native library folder {}", rocksLibFolder);
					// noinspection ResultOfMethodCallIgnored
					rocksLibFolder.mkdirs();

					// explicitly load the JNI dependency if it has not been loaded before
					NativeLibraryLoader.getInstance().loadLibrary(rocksLibFolder.getAbsolutePath());

					// this initialization here should validate that the loading succeeded
					RocksDB.loadLibrary();

					// seems to have worked
					LOG.info("Successfully loaded RocksDB native library");
					rocksDbInitialized = true;
					return;
				}
				catch (Throwable t) {
					lastException = t;
					LOG.debug("RocksDB JNI library loading attempt {} failed", attempt, t);

					// try to force RocksDB to attempt reloading the library
					try {
						resetRocksDBLoadedFlag();
					} catch (Throwable tt) {
						LOG.debug("Failed to reset 'initialized' flag in RocksDB native code loader", tt);
					}
				}
			}

			throw new IOException("Could not load the native RocksDB library", lastException);
		}
	}
}
 

@VisibleForTesting
static void resetRocksDBLoadedFlag() throws Exception {
	final Field initField = org.rocksdb.NativeLibraryLoader.class.getDeclaredField("initialized");
	initField.setAccessible(true);
	initField.setBoolean(null, false);
}
 

@Test(timeout = 2000)
@RetryOnFailure(times = 3)
public void testRocksDbListStateAPIs() throws Exception {
	final File rocksDir = tmp.newFolder();

	// ensure the RocksDB library is loaded to a distinct location each retry
	NativeLibraryLoader.getInstance().loadLibrary(rocksDir.getAbsolutePath());

	final String key1 = "key1";
	final String key2 = "key2";
	final String value = "abcdefghijklmnopqrstuvwxyz0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ7890654321";

	final byte[] keyBytes1 = key1.getBytes(StandardCharsets.UTF_8);
	final byte[] keyBytes2 = key2.getBytes(StandardCharsets.UTF_8);
	final byte[] valueBytes = value.getBytes(StandardCharsets.UTF_8);

	// The number of values added to ListState. Can be changed for benchmarking
	final int num = 10;

	try (
		final Options options = new Options()
				.setCompactionStyle(CompactionStyle.LEVEL)
				.setLevelCompactionDynamicLevelBytes(true)
				.setIncreaseParallelism(4)
				.setUseFsync(false)
				.setMaxOpenFiles(-1)
				.setCreateIfMissing(true)
				.setMergeOperatorName(RocksDBKeyedStateBackend.MERGE_OPERATOR_NAME);

		final WriteOptions writeOptions = new WriteOptions()
				.setSync(false)
				.setDisableWAL(true);

		final RocksDB rocksDB = RocksDB.open(options, rocksDir.getAbsolutePath())) {

		// ----- add() API -----
		log.info("begin add");

		final long beginInsert1 = System.nanoTime();
		for (int i = 0; i < num; i++) {
			rocksDB.merge(writeOptions, keyBytes1, valueBytes);
		}
		final long endInsert1 = System.nanoTime();

		log.info("end add - duration: {} ns", (endInsert1 - beginInsert1));

		// ----- update() API -----

		List<byte[]> list = new ArrayList<>(num);
		for (int i = 0; i < num; i++) {
			list.add(valueBytes);
		}
		byte[] premerged = merge(list);

		log.info("begin update");

		final long beginInsert2 = System.nanoTime();
		rocksDB.merge(writeOptions, keyBytes2, premerged);
		final long endInsert2 = System.nanoTime();

		log.info("end update - duration: {} ns", (endInsert2 - beginInsert2));
	}
}
 

private void ensureRocksDBIsLoaded(String tempDirectory) throws IOException {
	synchronized (RocksDBStateBackend.class) {
		if (!rocksDbInitialized) {

			final File tempDirParent = new File(tempDirectory).getAbsoluteFile();
			LOG.info("Attempting to load RocksDB native library and store it under '{}'", tempDirParent);

			Throwable lastException = null;
			for (int attempt = 1; attempt <= ROCKSDB_LIB_LOADING_ATTEMPTS; attempt++) {
				try {
					// when multiple instances of this class and RocksDB exist in different
					// class loaders, then we can see the following exception:
					// "java.lang.UnsatisfiedLinkError: Native Library /path/to/temp/dir/librocksdbjni-linux64.so
					// already loaded in another class loader"

					// to avoid that, we need to add a random element to the library file path
					// (I know, seems like an unnecessary hack, since the JVM obviously can handle multiple
					//  instances of the same JNI library being loaded in different class loaders, but
					//  apparently not when coming from the same file path, so there we go)

					final File rocksLibFolder = new File(tempDirParent, "rocksdb-lib-" + new AbstractID());

					// make sure the temp path exists
					LOG.debug("Attempting to create RocksDB native library folder {}", rocksLibFolder);
					// noinspection ResultOfMethodCallIgnored
					rocksLibFolder.mkdirs();

					// explicitly load the JNI dependency if it has not been loaded before
					NativeLibraryLoader.getInstance().loadLibrary(rocksLibFolder.getAbsolutePath());

					// this initialization here should validate that the loading succeeded
					RocksDB.loadLibrary();

					// seems to have worked
					LOG.info("Successfully loaded RocksDB native library");
					rocksDbInitialized = true;
					return;
				}
				catch (Throwable t) {
					lastException = t;
					LOG.debug("RocksDB JNI library loading attempt {} failed", attempt, t);

					// try to force RocksDB to attempt reloading the library
					try {
						resetRocksDBLoadedFlag();
					} catch (Throwable tt) {
						LOG.debug("Failed to reset 'initialized' flag in RocksDB native code loader", tt);
					}
				}
			}

			throw new IOException("Could not load the native RocksDB library", lastException);
		}
	}
}
 

@VisibleForTesting
static void resetRocksDBLoadedFlag() throws Exception {
	final Field initField = org.rocksdb.NativeLibraryLoader.class.getDeclaredField("initialized");
	initField.setAccessible(true);
	initField.setBoolean(null, false);
}
 

@Test(timeout = 2000)
@RetryOnFailure(times = 3)
public void testRocksDbListStateAPIs() throws Exception {
	final File rocksDir = tmp.newFolder();

	// ensure the RocksDB library is loaded to a distinct location each retry
	NativeLibraryLoader.getInstance().loadLibrary(rocksDir.getAbsolutePath());

	final String key1 = "key1";
	final String key2 = "key2";
	final String value = "abcdefghijklmnopqrstuvwxyz0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ7890654321";

	final byte[] keyBytes1 = key1.getBytes(StandardCharsets.UTF_8);
	final byte[] keyBytes2 = key2.getBytes(StandardCharsets.UTF_8);
	final byte[] valueBytes = value.getBytes(StandardCharsets.UTF_8);

	// The number of values added to ListState. Can be changed for benchmarking
	final int num = 10;

	try (
		final Options options = new Options()
				.setCompactionStyle(CompactionStyle.LEVEL)
				.setLevelCompactionDynamicLevelBytes(true)
				.setIncreaseParallelism(4)
				.setUseFsync(false)
				.setMaxOpenFiles(-1)
				.setCreateIfMissing(true)
				.setMergeOperatorName(RocksDBKeyedStateBackend.MERGE_OPERATOR_NAME);

		final WriteOptions writeOptions = new WriteOptions()
				.setSync(false)
				.setDisableWAL(true);

		final RocksDB rocksDB = RocksDB.open(options, rocksDir.getAbsolutePath())) {

		// ----- add() API -----
		log.info("begin add");

		final long beginInsert1 = System.nanoTime();
		for (int i = 0; i < num; i++) {
			rocksDB.merge(writeOptions, keyBytes1, valueBytes);
		}
		final long endInsert1 = System.nanoTime();

		log.info("end add - duration: {} ns", (endInsert1 - beginInsert1));

		// ----- update() API -----

		List<byte[]> list = new ArrayList<>(num);
		for (int i = 0; i < num; i++) {
			list.add(valueBytes);
		}
		byte[] premerged = merge(list);

		log.info("begin update");

		final long beginInsert2 = System.nanoTime();
		rocksDB.merge(writeOptions, keyBytes2, premerged);
		final long endInsert2 = System.nanoTime();

		log.info("end update - duration: {} ns", (endInsert2 - beginInsert2));
	}
}
 

@VisibleForTesting
static void resetRocksDBLoadedFlag() throws Exception {
	final Field initField = org.rocksdb.NativeLibraryLoader.class.getDeclaredField("initialized");
	initField.setAccessible(true);
	initField.setBoolean(null, false);
}
 

@BeforeClass
public static void loadRocksLibrary() throws Exception {
	NativeLibraryLoader.getInstance().loadLibrary(TMP_DIR.newFolder().getAbsolutePath());
}
 

@Test(timeout = 2000)
@RetryOnFailure(times = 3)
public void testRocksDbListStateAPIs() throws Exception {
	final File rocksDir = tmp.newFolder();

	// ensure the RocksDB library is loaded to a distinct location each retry
	NativeLibraryLoader.getInstance().loadLibrary(rocksDir.getAbsolutePath());

	final String key1 = "key1";
	final String key2 = "key2";
	final String value = "abcdefghijklmnopqrstuvwxyz0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ7890654321";

	final byte[] keyBytes1 = key1.getBytes(StandardCharsets.UTF_8);
	final byte[] keyBytes2 = key2.getBytes(StandardCharsets.UTF_8);
	final byte[] valueBytes = value.getBytes(StandardCharsets.UTF_8);

	// The number of values added to ListState. Can be changed for benchmarking
	final int num = 10;

	try (
		final Options options = new Options()
				.setCompactionStyle(CompactionStyle.LEVEL)
				.setLevelCompactionDynamicLevelBytes(true)
				.setIncreaseParallelism(4)
				.setUseFsync(false)
				.setMaxOpenFiles(-1)
				.setCreateIfMissing(true)
				.setMergeOperatorName(RocksDBKeyedStateBackend.MERGE_OPERATOR_NAME);

		final WriteOptions writeOptions = new WriteOptions()
				.setSync(false)
				.setDisableWAL(true);

		final RocksDB rocksDB = RocksDB.open(options, rocksDir.getAbsolutePath())) {

		// ----- add() API -----
		log.info("begin add");

		final long beginInsert1 = System.nanoTime();
		for (int i = 0; i < num; i++) {
			rocksDB.merge(writeOptions, keyBytes1, valueBytes);
		}
		final long endInsert1 = System.nanoTime();

		log.info("end add - duration: {} ns", (endInsert1 - beginInsert1));

		// ----- update() API -----

		List<byte[]> list = new ArrayList<>(num);
		for (int i = 0; i < num; i++) {
			list.add(valueBytes);
		}
		byte[] premerged = merge(list);

		log.info("begin update");

		final long beginInsert2 = System.nanoTime();
		rocksDB.merge(writeOptions, keyBytes2, premerged);
		final long endInsert2 = System.nanoTime();

		log.info("end update - duration: {} ns", (endInsert2 - beginInsert2));
	}
}
 

@Before
public void ensureRocksDbNativeLibraryLoaded() throws IOException {
	NativeLibraryLoader.getInstance().loadLibrary(temporaryFolder.newFolder().getAbsolutePath());
}
 

@BeforeClass
public static void ensureRocksDbNativeLibraryLoaded() throws IOException {
	NativeLibraryLoader.getInstance().loadLibrary(TMP_FOLDER.newFolder().getAbsolutePath());
}