Java Code Examples for org.rocksdb.RocksDB#createColumnFamily()

protected static Pair<RocksDB, Map<COLUMN_FAMILIES, ColumnFamilyHandle>> createNewRocksDB(File pathToIndex)
    throws RocksDBException {
  RocksDB db = null; // Not auto-closable.
  Map<COLUMN_FAMILIES, ColumnFamilyHandle> columnFamilyHandles = new HashMap<>();

  Options options = ROCKS_DB_CREATE_OPTIONS;
  System.out.println("Opening index at " + pathToIndex.getAbsolutePath());
  db = RocksDB.open(options, pathToIndex.getAbsolutePath());

  for (COLUMN_FAMILIES cf : COLUMN_FAMILIES.values()) {
    LOGGER.info("Creating column family %s", cf.getName());
    ColumnFamilyHandle cfh =
        db.createColumnFamily(new ColumnFamilyDescriptor(cf.getName().getBytes(UTF8)));
    columnFamilyHandles.put(cf, cfh);
  }

  return Pair.of(db, columnFamilyHandles);
}
 

/**
 * Create a new rocks DB at a particular location on disk.
 * @param pathToIndex A path to the directory where the index will be created.
 * @param columnFamilies Column families to create in the DB.
 * @param <T> A type (probably an enum) that represents a set of column families.
 * @return A DB and map of column family labels (as T) to enums.
 * @throws RocksDBException
 */
public static <T extends ColumnFamilyEnumeration<T>> RocksDBAndHandles<T> createNewRocksDB(
    File pathToIndex, T[] columnFamilies) throws RocksDBException {
  RocksDB db = null; // Not auto-closable.
  Map<T, ColumnFamilyHandle> columnFamilyHandles = new HashMap<>();

  db = RocksDB.open(ROCKS_DB_CREATE_OPTIONS, pathToIndex.getAbsolutePath());

  for (T cf : columnFamilies) {
    LOGGER.info("Creating column family %s", cf.getName());
    ColumnFamilyHandle cfh =
        db.createColumnFamily(new ColumnFamilyDescriptor(cf.getName().getBytes(UTF8)));
    columnFamilyHandles.put(cf, cfh);
  }

  return new RocksDBAndHandles<T>(db, columnFamilyHandles);
}
 

private static ColumnFamilyHandle createColumnFamily(ColumnFamilyDescriptor columnDescriptor, RocksDB db) {
	try {
		return db.createColumnFamily(columnDescriptor);
	} catch (RocksDBException e) {
		IOUtils.closeQuietly(columnDescriptor.getOptions());
		throw new FlinkRuntimeException("Error creating ColumnFamilyHandle.", e);
	}
}
 

private static ColumnFamilyHandle createColumnFamily(ColumnFamilyDescriptor columnDescriptor, RocksDB db) {
	try {
		return db.createColumnFamily(columnDescriptor);
	} catch (RocksDBException e) {
		IOUtils.closeQuietly(columnDescriptor.getOptions());
		throw new FlinkRuntimeException("Error creating ColumnFamilyHandle.", e);
	}
}
 

/**
 * Create an on-disk index of reaction-supporting data from BRENDA using RocksDB.  This DB will contain a number
 * of `column families` (i.e. per-table namespaces).  Each FromBrendaDB instance
 *
 * All index rows are keyed on EC number, literature reference (individually, lists are split during construction),
 * and organism names.   Values are serialized (via Serializable) lists of FromBrendaDB objects; each column family
 * contains one type of object.
 *
 * Creating this data on an in-office MBP with a BRENDA MySQL instance running in EC2 takes just a couple of minutes.
 * Looking up supporting data locally vs. running MySQL queries for every data type * every reaction results in a
 * ~30x speedup of reaction processing.
 *
 * @param pathToIndex The local path where the index should be built.  This will become a directory containing
 *                    RocksDB files.
 * @param conn A connection to the BRENDA MySQL DB (`brenda` database) from which data will be read.
 * @throws IOException
 * @throws ClassNotFoundException
 * @throws RocksDBException
 * @throws SQLException
 */
public void constructOnDiskBRENDAIndex(File pathToIndex, Connection conn)
    throws IOException, ClassNotFoundException, RocksDBException, SQLException {
  if (pathToIndex.exists()) {
    System.out.println("Index already exists, not recreating.");
    return;
  }

  RocksDB db = null; // Not auto-closable.
  List<? extends FromBrendaDB> instances = allFromBrendaDBInstances();
  try {
    Options options = new Options().setCreateIfMissing(true);
    System.out.println("Opening index at " + pathToIndex.getAbsolutePath());
    db = RocksDB.open(options, pathToIndex.getAbsolutePath());

    for (FromBrendaDB instance : instances) {
      System.out.println("Writing index for " + instance.getColumnFamilyName());
      ColumnFamilyHandle cfh =
          db.createColumnFamily(new ColumnFamilyDescriptor(instance.getColumnFamilyName().getBytes(UTF8)));
      IndexWriter writer = new IndexWriter(cfh, db, instance);
      writer.run(conn);
      db.flush(new FlushOptions());
    }
  } finally {
    if (db != null) {
      db.close();
    }
  }
}
 

private static ColumnFamilyHandle createColumnFamily(ColumnFamilyDescriptor columnDescriptor, RocksDB db) {
	try {
		return db.createColumnFamily(columnDescriptor);
	} catch (RocksDBException e) {
		IOUtils.closeQuietly(columnDescriptor.getOptions());
		throw new FlinkRuntimeException("Error creating ColumnFamilyHandle.", e);
	}
}
 

private static void rocksDbTest(RocksDB db, List<ColumnFamilyHandle> handlers) {
    try {
        ColumnFamilyHandle handler1 = null;
        ColumnFamilyHandle handler2 = null;
        if (handlers.size() > 0) {
            // skip default column family
            handler1 = handlers.get(1);
            handler2 = handlers.get(2);
        } else {
            handler1 = db.createColumnFamily(new ColumnFamilyDescriptor("test1".getBytes()));
            handler2 = db.createColumnFamily(new ColumnFamilyDescriptor("test2".getBytes()));
        }
        int startValue1 = getStartValue(db, handler1);
        int startValue2 = getStartValue(db, handler2);;

        Checkpoint cp = Checkpoint.create(db);
   
        if (isCompaction) {
            db.compactRange();
            LOG.info("Compaction!");
        }

        long flushWaitTime = System.currentTimeMillis() + flushInterval;
        for (int i = 0; i < putNum || putNum == -1; i++) {
            db.put(handler1, String.valueOf(i % 1000).getBytes(), String.valueOf(startValue1 + i).getBytes());
            db.put(handler2, String.valueOf(i % 1000).getBytes(), String.valueOf(startValue2 + i).getBytes());
            if (isFlush && flushWaitTime <= System.currentTimeMillis()) {
                db.flush(new FlushOptions());
                if (isCheckpoint) {
                    cp.createCheckpoint(cpPath + "/" + i);
                }
                flushWaitTime = System.currentTimeMillis() + flushInterval;
            }
        }
    } catch (RocksDBException e) {
        LOG.error("Failed to put or flush", e);
    }
}