org.rocksdb.DBOptions#setCreateIfMissing

Source File: TestTypedRDBTableStore.java From hadoop-ozone with Apache License 2.0

6 votes

@Before
public void setUp() throws Exception {
  options = new DBOptions();
  options.setCreateIfMissing(true);
  options.setCreateMissingColumnFamilies(true);

  Statistics statistics = new Statistics();
  statistics.setStatsLevel(StatsLevel.ALL);
  options = options.setStatistics(statistics);

  Set<TableConfig> configSet = new HashSet<>();
  for (String name : families) {
    TableConfig newConfig = new TableConfig(name, new ColumnFamilyOptions());
    configSet.add(newConfig);
  }
  rdbStore = new RDBStore(folder.newFolder(), options, configSet);

  codecRegistry = new CodecRegistry();

}

Source File: TestRDBStore.java From hadoop-ozone with Apache License 2.0

6 votes

@Before
public void setUp() throws Exception {
  options = new DBOptions();
  options.setCreateIfMissing(true);
  options.setCreateMissingColumnFamilies(true);

  Statistics statistics = new Statistics();
  statistics.setStatsLevel(StatsLevel.ALL);
  options = options.setStatistics(statistics);
  configSet = new HashSet<>();
  for(String name : families) {
    TableConfig newConfig = new TableConfig(name, new ColumnFamilyOptions());
    configSet.add(newConfig);
  }
  rdbStore = new RDBStore(folder.newFolder(), options, configSet);
}

Source File: TestRDBTableStore.java From hadoop-ozone with Apache License 2.0

6 votes

@Before
public void setUp() throws Exception {
  options = new DBOptions();
  options.setCreateIfMissing(true);
  options.setCreateMissingColumnFamilies(true);

  Statistics statistics = new Statistics();
  statistics.setStatsLevel(StatsLevel.ALL);
  options = options.setStatistics(statistics);

  Set<TableConfig> configSet = new HashSet<>();
  for(String name : families) {
    TableConfig newConfig = new TableConfig(name, new ColumnFamilyOptions());
    configSet.add(newConfig);
  }
  rdbStore = new RDBStore(folder.newFolder(), options, configSet);
}

Source File: SQLConnection.java From act with GNU General Public License v3.0

6 votes

public Pair<RocksDB, Map<String, ColumnFamilyHandle>> openSupportingIndex(File supportingIndex)
    throws RocksDBException {
  List<FromBrendaDB> instances = BrendaSupportingEntries.allFromBrendaDBInstances();
  List<ColumnFamilyDescriptor> columnFamilyDescriptors = new ArrayList<>(instances.size() + 1);
  columnFamilyDescriptors.add(new ColumnFamilyDescriptor("default".getBytes()));
  for (FromBrendaDB instance : instances) {
    columnFamilyDescriptors.add(new ColumnFamilyDescriptor(instance.getColumnFamilyName().getBytes()));
  }
  List<ColumnFamilyHandle> columnFamilyHandles = new ArrayList<>(columnFamilyDescriptors.size());

  DBOptions dbOptions = new DBOptions();
  dbOptions.setCreateIfMissing(false);
  RocksDB rocksDB = RocksDB.open(dbOptions, supportingIndex.getAbsolutePath(),
      columnFamilyDescriptors, columnFamilyHandles);
  Map<String, ColumnFamilyHandle> columnFamilyHandleMap = new HashMap<>(columnFamilyHandles.size());
  // TODO: can we zip these together more easily w/ Java 8?

  for (int i = 0; i < columnFamilyDescriptors.size(); i++) {
    ColumnFamilyDescriptor cfd = columnFamilyDescriptors.get(i);
    ColumnFamilyHandle cfh = columnFamilyHandles.get(i);
    columnFamilyHandleMap.put(new String(cfd.columnFamilyName(), BrendaSupportingEntries.UTF8), cfh);
  }

  return Pair.of(rocksDB, columnFamilyHandleMap);
}

Source File: RocksDBResourceContainer.java From flink with Apache License 2.0

6 votes

/**
 * Gets the RocksDB {@link DBOptions} to be used for RocksDB instances.
 */
public DBOptions getDbOptions() {
	// initial options from pre-defined profile
	DBOptions opt = predefinedOptions.createDBOptions(handlesToClose);
	handlesToClose.add(opt);

	// add user-defined options factory, if specified
	if (optionsFactory != null) {
		opt = optionsFactory.createDBOptions(opt, handlesToClose);
	}

	// add necessary default options
	opt = opt.setCreateIfMissing(true);

	// if sharedResources is non-null, use the write buffer manager from it.
	if (sharedResources != null) {
		opt.setWriteBufferManager(sharedResources.getResourceHandle().getWriteBufferManager());
	}

	return opt;
}

Source File: RocksDBStateBackend.java From Flink-CEPplus with Apache License 2.0

5 votes

/**
 * Gets the RocksDB {@link DBOptions} to be used for all RocksDB instances.
 */
public DBOptions getDbOptions() {
	// initial options from pre-defined profile
	DBOptions opt = getPredefinedOptions().createDBOptions();

	// add user-defined options factory, if specified
	if (optionsFactory != null) {
		opt = optionsFactory.createDBOptions(opt);
	}

	// add necessary default options
	opt = opt.setCreateIfMissing(true);

	return opt;
}

Source File: KVStateRequestSerializerRocksDBTest.java From Flink-CEPplus with Apache License 2.0

5 votes

/**
 * Tests list serialization and deserialization match.
 *
 * @see KvStateRequestSerializerTest#testListSerialization()
 * KvStateRequestSerializerTest#testListSerialization() using the heap state back-end
 * test
 */
@Test
public void testListSerialization() throws Exception {
	final long key = 0L;

	// objects for RocksDB state list serialisation
	DBOptions dbOptions = PredefinedOptions.DEFAULT.createDBOptions();
	dbOptions.setCreateIfMissing(true);
	ExecutionConfig executionConfig = new ExecutionConfig();
	final RocksDBKeyedStateBackend<Long> longHeapKeyedStateBackend =
		new RocksDBKeyedStateBackendBuilder<>(
			"no-op",
			ClassLoader.getSystemClassLoader(),
			temporaryFolder.getRoot(),
			dbOptions,
			stateName -> PredefinedOptions.DEFAULT.createColumnOptions(),
			mock(TaskKvStateRegistry.class),
			LongSerializer.INSTANCE,
			1,
			new KeyGroupRange(0, 0),
			executionConfig,
			TestLocalRecoveryConfig.disabled(),
			RocksDBStateBackend.PriorityQueueStateType.HEAP,
			TtlTimeProvider.DEFAULT,
			new UnregisteredMetricsGroup(),
			Collections.emptyList(),
			AbstractStateBackend.getCompressionDecorator(executionConfig),
			new CloseableRegistry()
		).build();
	longHeapKeyedStateBackend.setCurrentKey(key);

	final InternalListState<Long, VoidNamespace, Long> listState = longHeapKeyedStateBackend.createInternalState(VoidNamespaceSerializer.INSTANCE,
			new ListStateDescriptor<>("test", LongSerializer.INSTANCE));

	KvStateRequestSerializerTest.testListSerialization(key, listState);
	longHeapKeyedStateBackend.dispose();
}

Source File: StorageOptionsFactory.java From sofa-jraft with Apache License 2.0

5 votes

public static DBOptions getDefaultRocksDBOptions() {
    // Turn based on https://github.com/facebook/rocksdb/wiki/RocksDB-Tuning-Guide
    final DBOptions opts = new DBOptions();

    // If this value is set to true, then the database will be created if it is
    // missing during {@code RocksDB.open()}.
    opts.setCreateIfMissing(true);

    // If true, missing column families will be automatically created.
    opts.setCreateMissingColumnFamilies(true);

    // Number of open files that can be used by the DB.  You may need to increase
    // this if your database has a large working set. Value -1 means files opened
    // are always kept open.
    opts.setMaxOpenFiles(-1);

    // The maximum number of concurrent background compactions. The default is 1,
    // but to fully utilize your CPU and storage you might want to increase this
    // to approximately number of cores in the system.
    opts.setMaxBackgroundCompactions(Math.min(Utils.cpus(), 4));

    // The maximum number of concurrent flush operations. It is usually good enough
    // to set this to 1.
    opts.setMaxBackgroundFlushes(1);

    return opts;
}

Source File: RocksDBStateBackend.java From flink with Apache License 2.0

5 votes

/**
 * Gets the RocksDB {@link DBOptions} to be used for all RocksDB instances.
 */
public DBOptions getDbOptions() {
	// initial options from pre-defined profile
	DBOptions opt = getPredefinedOptions().createDBOptions();

	// add user-defined options factory, if specified
	if (optionsFactory != null) {
		opt = optionsFactory.createDBOptions(opt);
	}

	// add necessary default options
	opt = opt.setCreateIfMissing(true);

	return opt;
}

Source File: KVStateRequestSerializerRocksDBTest.java From flink with Apache License 2.0

5 votes

/**
 * Tests list serialization and deserialization match.
 *
 * @see KvStateRequestSerializerTest#testListSerialization()
 * KvStateRequestSerializerTest#testListSerialization() using the heap state back-end
 * test
 */
@Test
public void testListSerialization() throws Exception {
	final long key = 0L;

	// objects for RocksDB state list serialisation
	DBOptions dbOptions = PredefinedOptions.DEFAULT.createDBOptions();
	dbOptions.setCreateIfMissing(true);
	ExecutionConfig executionConfig = new ExecutionConfig();
	final RocksDBKeyedStateBackend<Long> longHeapKeyedStateBackend =
		new RocksDBKeyedStateBackendBuilder<>(
			"no-op",
			ClassLoader.getSystemClassLoader(),
			temporaryFolder.getRoot(),
			dbOptions,
			stateName -> PredefinedOptions.DEFAULT.createColumnOptions(),
			mock(TaskKvStateRegistry.class),
			LongSerializer.INSTANCE,
			1,
			new KeyGroupRange(0, 0),
			executionConfig,
			TestLocalRecoveryConfig.disabled(),
			RocksDBStateBackend.PriorityQueueStateType.HEAP,
			TtlTimeProvider.DEFAULT,
			new UnregisteredMetricsGroup(),
			Collections.emptyList(),
			AbstractStateBackend.getCompressionDecorator(executionConfig),
			new CloseableRegistry()
		).build();
	longHeapKeyedStateBackend.setCurrentKey(key);

	final InternalListState<Long, VoidNamespace, Long> listState = longHeapKeyedStateBackend.createInternalState(VoidNamespaceSerializer.INSTANCE,
			new ListStateDescriptor<>("test", LongSerializer.INSTANCE));

	KvStateRequestSerializerTest.testListSerialization(key, listState);
	longHeapKeyedStateBackend.dispose();
}

Source File: PubchemTTLMerger.java From act with GNU General Public License v3.0

5 votes

/**
 * Open an existing RocksDB index.  Use this after successful index generation to access the map of Pubchem compound
 * ids to synonyms/MeSH ids using the column family CID_TO_SYNONYMS.
 * @param pathToIndex A path to the RocksDB index directory to use.
 * @return
 * @throws RocksDBException
 */
public static Pair<RocksDB, Map<COLUMN_FAMILIES, ColumnFamilyHandle>> openExistingRocksDB(File pathToIndex)
    throws RocksDBException {
  List<ColumnFamilyDescriptor> columnFamilyDescriptors = new ArrayList<>(COLUMN_FAMILIES.values().length + 1);
  // Must also open the "default" family or RocksDB will probably choke.
  columnFamilyDescriptors.add(new ColumnFamilyDescriptor(DEFAULT_ROCKSDB_COLUMN_FAMILY.getBytes()));
  for (COLUMN_FAMILIES family : COLUMN_FAMILIES.values()) {
    columnFamilyDescriptors.add(new ColumnFamilyDescriptor(family.getName().getBytes()));
  }
  List<ColumnFamilyHandle> columnFamilyHandles = new ArrayList<>(columnFamilyDescriptors.size());

  DBOptions dbOptions = ROCKS_DB_OPEN_OPTIONS;
  dbOptions.setCreateIfMissing(false);
  RocksDB rocksDB = RocksDB.open(dbOptions, pathToIndex.getAbsolutePath(),
      columnFamilyDescriptors, columnFamilyHandles);
  Map<COLUMN_FAMILIES, ColumnFamilyHandle> columnFamilyHandleMap = new HashMap<>(COLUMN_FAMILIES.values().length);
  // TODO: can we zip these together more easily w/ Java 8?

  for (int i = 0; i < columnFamilyDescriptors.size(); i++) {
    ColumnFamilyDescriptor cfd = columnFamilyDescriptors.get(i);
    ColumnFamilyHandle cfh = columnFamilyHandles.get(i);
    String familyName = new String(cfd.columnFamilyName(), UTF8);
    COLUMN_FAMILIES descriptorFamily = COLUMN_FAMILIES.getFamilyByName(familyName);
    if (descriptorFamily == null) {
      if (!DEFAULT_ROCKSDB_COLUMN_FAMILY.equals(familyName)) {
        String msg = String.format("Found unexpected family name '%s' when trying to open RocksDB at %s",
            familyName, pathToIndex.getAbsolutePath());
        LOGGER.error(msg);
        // Crash if we don't recognize the contents of this DB.
        throw new RuntimeException(msg);
      }
      // Just skip this column family if it doesn't map to something we know but is expected.
      continue;
    }

    columnFamilyHandleMap.put(descriptorFamily, cfh);
  }

  return Pair.of(rocksDB, columnFamilyHandleMap);
}

Source File: KVStateRequestSerializerRocksDBTest.java From Flink-CEPplus with Apache License 2.0

4 votes

/**
 * Tests map serialization and deserialization match.
 *
 * @see KvStateRequestSerializerTest#testMapSerialization()
 * KvStateRequestSerializerTest#testMapSerialization() using the heap state back-end
 * test
 */
@Test
public void testMapSerialization() throws Exception {
	final long key = 0L;

	// objects for RocksDB state list serialisation
	DBOptions dbOptions = PredefinedOptions.DEFAULT.createDBOptions();
	dbOptions.setCreateIfMissing(true);
	ExecutionConfig executionConfig = new ExecutionConfig();
	final RocksDBKeyedStateBackend<Long> longHeapKeyedStateBackend =
		new RocksDBKeyedStateBackendBuilder<>(
			"no-op",
			ClassLoader.getSystemClassLoader(),
			temporaryFolder.getRoot(),
			dbOptions,
			stateName -> PredefinedOptions.DEFAULT.createColumnOptions(),
			mock(TaskKvStateRegistry.class),
			LongSerializer.INSTANCE,
			1,
			new KeyGroupRange(0, 0),
			executionConfig,
			TestLocalRecoveryConfig.disabled(),
			RocksDBStateBackend.PriorityQueueStateType.HEAP,
			TtlTimeProvider.DEFAULT,
			new UnregisteredMetricsGroup(),
			Collections.emptyList(),
			AbstractStateBackend.getCompressionDecorator(executionConfig),
			new CloseableRegistry()
		).build();
	longHeapKeyedStateBackend.setCurrentKey(key);

	final InternalMapState<Long, VoidNamespace, Long, String> mapState =
			(InternalMapState<Long, VoidNamespace, Long, String>)
					longHeapKeyedStateBackend.getPartitionedState(
							VoidNamespace.INSTANCE,
							VoidNamespaceSerializer.INSTANCE,
							new MapStateDescriptor<>("test", LongSerializer.INSTANCE, StringSerializer.INSTANCE));

	KvStateRequestSerializerTest.testMapSerialization(key, mapState);
	longHeapKeyedStateBackend.dispose();
}

Source File: KVStateRequestSerializerRocksDBTest.java From flink with Apache License 2.0

4 votes

/**
 * Tests map serialization and deserialization match.
 *
 * @see KvStateRequestSerializerTest#testMapSerialization()
 * KvStateRequestSerializerTest#testMapSerialization() using the heap state back-end
 * test
 */
@Test
public void testMapSerialization() throws Exception {
	final long key = 0L;

	// objects for RocksDB state list serialisation
	DBOptions dbOptions = PredefinedOptions.DEFAULT.createDBOptions();
	dbOptions.setCreateIfMissing(true);
	ExecutionConfig executionConfig = new ExecutionConfig();
	final RocksDBKeyedStateBackend<Long> longHeapKeyedStateBackend =
		new RocksDBKeyedStateBackendBuilder<>(
			"no-op",
			ClassLoader.getSystemClassLoader(),
			temporaryFolder.getRoot(),
			dbOptions,
			stateName -> PredefinedOptions.DEFAULT.createColumnOptions(),
			mock(TaskKvStateRegistry.class),
			LongSerializer.INSTANCE,
			1,
			new KeyGroupRange(0, 0),
			executionConfig,
			TestLocalRecoveryConfig.disabled(),
			RocksDBStateBackend.PriorityQueueStateType.HEAP,
			TtlTimeProvider.DEFAULT,
			new UnregisteredMetricsGroup(),
			Collections.emptyList(),
			AbstractStateBackend.getCompressionDecorator(executionConfig),
			new CloseableRegistry()
		).build();
	longHeapKeyedStateBackend.setCurrentKey(key);

	final InternalMapState<Long, VoidNamespace, Long, String> mapState =
			(InternalMapState<Long, VoidNamespace, Long, String>)
					longHeapKeyedStateBackend.getPartitionedState(
							VoidNamespace.INSTANCE,
							VoidNamespaceSerializer.INSTANCE,
							new MapStateDescriptor<>("test", LongSerializer.INSTANCE, StringSerializer.INSTANCE));

	KvStateRequestSerializerTest.testMapSerialization(key, mapState);
	longHeapKeyedStateBackend.dispose();
}

Source File: DBUtil.java From act with GNU General Public License v3.0

4 votes

/**
 * Open an existing RocksDB index.
 * @param pathToIndex A path to the RocksDB index directory to use.
 * @param columnFamilies A list of column familities to open.  Must be exhaustive, non-empty, and non-null.
 * @return A DB and map of column family labels (as T) to enums.
 * @throws RocksDBException
 */
public static <T extends ColumnFamilyEnumeration<T>> RocksDBAndHandles<T> openExistingRocksDB(
    File pathToIndex, T[] columnFamilies) throws RocksDBException {
  if (columnFamilies == null || columnFamilies.length == 0) {
    throw new RuntimeException("Cannot open a RocksDB with an empty list of column families.");
  }

  List<ColumnFamilyDescriptor> columnFamilyDescriptors = new ArrayList<>(columnFamilies.length + 1);
  // Must also open the "default" family or RocksDB will probably choke.
  columnFamilyDescriptors.add(new ColumnFamilyDescriptor(DEFAULT_ROCKSDB_COLUMN_FAMILY.getBytes()));
  for (T family : columnFamilies) {
    columnFamilyDescriptors.add(new ColumnFamilyDescriptor(family.getName().getBytes()));
  }
  List<ColumnFamilyHandle> columnFamilyHandles = new ArrayList<>(columnFamilyDescriptors.size());

  DBOptions dbOptions = ROCKS_DB_OPEN_OPTIONS;
  dbOptions.setCreateIfMissing(false);
  RocksDB db = RocksDB.open(dbOptions, pathToIndex.getAbsolutePath(),
      columnFamilyDescriptors, columnFamilyHandles);
  Map<T, ColumnFamilyHandle> columnFamilyHandleMap = new HashMap<>(columnFamilies.length);
  // TODO: can we zip these together more easily w/ Java 8?

  for (int i = 0; i < columnFamilyDescriptors.size(); i++) {
    ColumnFamilyDescriptor cfd = columnFamilyDescriptors.get(i);
    ColumnFamilyHandle cfh = columnFamilyHandles.get(i);
    String familyName = new String(cfd.columnFamilyName(), UTF8);
    T descriptorFamily = columnFamilies[0].getFamilyByName(familyName); // Use any instance to get the next family.
    if (descriptorFamily == null) {
      if (!DEFAULT_ROCKSDB_COLUMN_FAMILY.equals(familyName)) {
        String msg = String.format("Found unexpected family name '%s' when trying to open RocksDB at %s",
            familyName, pathToIndex.getAbsolutePath());
        LOGGER.error(msg);
        // Crash if we don't recognize the contents of this DB.
        throw new RuntimeException(msg);
      }
      // Just skip this column family if it doesn't map to something we know but is expected.
      continue;
    }

    columnFamilyHandleMap.put(descriptorFamily, cfh);
  }

  return new RocksDBAndHandles<T>(db, columnFamilyHandleMap);
}

Source File: RocksDbUnitTest.java From jstorm with Apache License 2.0

4 votes

public static void main(String[] args) {
    Map conf = JStormHelper.LoadConf(args[0]);
    putNum = JStormUtils.parseInt(conf.get("put.number"), 100);
    isFlush = JStormUtils.parseBoolean(conf.get("is.flush"), true);
    isCheckpoint = JStormUtils.parseBoolean(conf.get("is.checkpoint"), true);
    sleepTime = JStormUtils.parseInt(conf.get("sleep.time"), 5000);
    compactionInterval = JStormUtils.parseInt(conf.get("compaction.interval"), 30000);
    flushInterval = JStormUtils.parseInt(conf.get("flush.interval"), 3000);
    isCompaction = JStormUtils.parseBoolean(conf.get("is.compaction"), true);
    fileSizeBase = JStormUtils.parseLong(conf.get("file.size.base"), 10 * SizeUnit.KB);
    levelNum = JStormUtils.parseInt(conf.get("db.level.num"), 1);
    compactionTriggerNum = JStormUtils.parseInt(conf.get("db.compaction.trigger.num"), 4);
    LOG.info("Conf={}", conf);
    
    RocksDB db;
    File file = new File(cpPath);
    file.mkdirs();

    List<ColumnFamilyHandle> columnFamilyHandles = new ArrayList<>();
    try {
        Options options = new Options();
        options.setCreateMissingColumnFamilies(true);
        options.setCreateIfMissing(true);
        options.setTargetFileSizeBase(fileSizeBase);
        options.setMaxBackgroundFlushes(2);
        options.setMaxBackgroundCompactions(2);
        options.setCompactionStyle(CompactionStyle.LEVEL);
        options.setNumLevels(levelNum);
        options.setLevelZeroFileNumCompactionTrigger(compactionTriggerNum);

        DBOptions dbOptions = new DBOptions();
        dbOptions.setCreateMissingColumnFamilies(true);
        dbOptions.setCreateIfMissing(true);
        dbOptions.setMaxBackgroundFlushes(2);
        dbOptions.setMaxBackgroundCompactions(2);
        ColumnFamilyOptions familyOptions = new ColumnFamilyOptions();
        familyOptions.setTargetFileSizeBase(fileSizeBase);
        familyOptions.setCompactionStyle(CompactionStyle.LEVEL);
        familyOptions.setNumLevels(levelNum);
        familyOptions.setLevelZeroFileNumCompactionTrigger(compactionTriggerNum);
        List<byte[]> families = RocksDB.listColumnFamilies(options, dbPath);
        List<ColumnFamilyDescriptor> columnFamilyDescriptors = new ArrayList<>();
        if (families != null) {
            for (byte[] bytes : families) {
                columnFamilyDescriptors.add(new ColumnFamilyDescriptor(bytes, familyOptions));
                LOG.info("Load colum family of {}", new String(bytes));
            }
        }
        
        if (columnFamilyDescriptors.size() > 0) {
            db = RocksDB.open(dbOptions, dbPath, columnFamilyDescriptors, columnFamilyHandles);
        } else {
            db = RocksDB.open(options, dbPath);
        }
    } catch (RocksDBException e) {
        LOG.error("Failed to open db", e);
        return;
    }

    rocksDbTest(db, columnFamilyHandles);
    
    db.close();
}

Java Code Examples for org.rocksdb.DBOptions#setCreateIfMissing()