org.rocksdb.RocksDBException Java Examples

@Override
public void writeBatch(BatchOperation operation)
    throws IOException {
  List<BatchOperation.SingleOperation> operations =
      operation.getOperations();
  if (!operations.isEmpty()) {
    try (WriteBatch writeBatch = new WriteBatch()) {
      for (BatchOperation.SingleOperation opt : operations) {
        switch (opt.getOpt()) {
        case DELETE:
          writeBatch.delete(opt.getKey());
          break;
        case PUT:
          writeBatch.put(opt.getKey(), opt.getValue());
          break;
        default:
          throw new IllegalArgumentException("Invalid operation "
              + opt.getOpt());
        }
      }
      db.write(writeOptions, writeBatch);
    } catch (RocksDBException e) {
      throw toIOException("Batch write operation failed", e);
    }
  }
}
 

@Override
public void removeWord(int index) {
  try (WriteBatch writeBatch = new WriteBatch()) {
    try (RocksIterator rocksIterator = rocksDB.newIterator()) {
      rocksIterator.seekToFirst();
      while (rocksIterator.isValid()) {
        SparseVector sparseVector = new SparseVector(rocksIterator.value());
        sparseVector.remove(index);
        writeBatch.put(rocksIterator.key(), sparseVector.toBytes());
      }
    }
    rocksDB.write(new WriteOptions(), writeBatch);
  } catch (RocksDBException e) {
    throw new RuntimeException(e);
  }
}
 

void ingestSstFiles(final EnumMap<SstColumnFamily, File> sstFileTable) {
    final Timer.Context timeCtx = getTimeContext("INGEST_SST_FILE");
    final Lock readLock = this.readWriteLock.readLock();
    readLock.lock();
    try {
        for (final Map.Entry<SstColumnFamily, File> entry : sstFileTable.entrySet()) {
            final SstColumnFamily sstColumnFamily = entry.getKey();
            final File sstFile = entry.getValue();
            final ColumnFamilyHandle columnFamilyHandle = findColumnFamilyHandle(sstColumnFamily);
            try (final IngestExternalFileOptions ingestOptions = new IngestExternalFileOptions()) {
                if (FileUtils.sizeOf(sstFile) == 0L) {
                    return;
                }
                final String filePath = sstFile.getAbsolutePath();
                LOG.info("Start ingest sst file {}.", filePath);
                this.db.ingestExternalFile(columnFamilyHandle, Collections.singletonList(filePath), ingestOptions);
            } catch (final RocksDBException e) {
                throw new StorageException("Fail to ingest sst file at path: " + sstFile, e);
            }
        }
    } finally {
        readLock.unlock();
        timeCtx.stop();
    }
}
 

private String stats() {
  try {
    StringBuilder sb = new StringBuilder();
    append(sb, "rocksdb.estimate-num-keys", "Estimated Number of Keys");
    append(sb, "rocksdb.estimate-live-data-size", "Estimated Live Data Size");
    append(sb, "rocksdb.total-sst-files-size", "Total SST files size");
    append(sb, "rocksdb.estimate-pending-compaction-bytes", "Pending Compaction Bytes");

    final BlobStats blobStats = metaManager.getStats();
    if (blobStats != null) {
      blobStats.append(sb);
    }
    return sb.toString();
  } catch(RocksDBException e) {
    throw new RuntimeException(e);
  }
}
 

@Test
public void testGet1KeyWithMultiValues() throws RocksDBException {

    put("person:1gname", "James");
    put("person:1gage", "19");
    put("person:1gcity", "Beijing");

    put("person:2gname", "Lisa");
    put("person:2gage", "20");
    put("person:2gcity", "Beijing");

    put("person:2all", "name=Lisa,age=20,city=Beijing");

    Session session = this.rocks.session();
    for (int i = 0; i < TIMES; i++) {
        s(session.get(TABLE, b("person:2all")));
    }
}
 

/**
 * Flush the data in memtable of RocksDB into disk, and then create checkpoint
 * 
 * @param batchId
 */
@Override
public void checkpoint(long batchId) {
    long startTime = System.currentTimeMillis();
    try {
        rocksDb.flush(new FlushOptions());
        Checkpoint cp = Checkpoint.create(rocksDb);
        cp.createCheckpoint(getLocalCheckpointPath(batchId));
    } catch (RocksDBException e) {
        LOG.error("Failed to create checkpoint for batch-" + batchId, e);
        throw new RuntimeException(e.getMessage());
    }

    if (JStormMetrics.enabled)
        rocksDbFlushAndCpLatency.update(System.currentTimeMillis() - startTime);
}
 

public void build(ILookupTable srcLookupTable) {
    File dbFolder = new File(dbPath);
    if (dbFolder.exists()) {
        logger.info("remove rocksdb folder:{} to rebuild table cache:{}", dbPath, tableDesc.getIdentity());
        FileUtils.deleteQuietly(dbFolder);
    } else {
        logger.info("create new rocksdb folder:{} for table cache:{}", dbPath, tableDesc.getIdentity());
        dbFolder.mkdirs();
    }
    logger.info("start to build lookup table:{} to rocks db:{}", tableDesc.getIdentity(), dbPath);
    try (RocksDB rocksDB = RocksDB.open(options, dbPath)) {
        // todo use batch may improve write performance
        for (String[] row : srcLookupTable) {
            KV kv = encoder.encode(row);
            rocksDB.put(kv.getKey(), kv.getValue());
        }
    } catch (RocksDBException e) {
        logger.error("error when put data to rocksDB", e);
        throw new RuntimeException("error when write data to rocks db", e);
    }

    logger.info("source table:{} has been written to rocks db:{}", tableDesc.getIdentity(), dbPath);
}
 

@Override
public void commit() {
    check();

    if (batch != null) {
        try {
            db.write(writeOptions, batch);
        } catch (RocksDBException e) {
            LOG.error(
                    "Unable to execute batch put/update/delete operation on "
                            + this.toString()
                            + ".",
                    e);
        }
        batch.close();
        batch = null;
    }
}
 

@Override
public synchronized void createTable(String table) throws RocksDBException {
    if (this.cfs.containsKey(table)) {
        return;
    }

    this.checkValid();

    // Should we use options.setCreateMissingColumnFamilies() to create CF
    ColumnFamilyDescriptor cfd = new ColumnFamilyDescriptor(encode(table));
    ColumnFamilyOptions options = cfd.getOptions();
    initOptions(this.config(), null, null, options, options);
    this.cfs.put(table, new CFHandle(this.rocksdb.createColumnFamily(cfd)));

    ingestExternalFile();
}
 

@SuppressWarnings("resource")
public static KnowledgeBase getInstance(final String kbDir, final String kbMetadataPathname, final boolean readOnly) throws RocksDBException, ClassNotFoundException, IOException {
	final boolean metadataExists = new File(kbMetadataPathname).exists();
	final boolean kbDirExists = new File(kbDir).exists();
	if (metadataExists != kbDirExists) throw new IllegalArgumentException("Either both or none of the knowledge-base directory and metadata must exist");

	RocksDB.loadLibrary();
	final ColumnFamilyOptions cfOptions = new ColumnFamilyOptions().setCompressionType(CompressionType.LZ4_COMPRESSION);
	final DBOptions dbOptions = new DBOptions().setCreateIfMissing(true).setCreateMissingColumnFamilies(true);
	final List<ColumnFamilyDescriptor> cfDescriptors = Arrays.asList(new ColumnFamilyDescriptor(RocksDB.DEFAULT_COLUMN_FAMILY, cfOptions), new ColumnFamilyDescriptor(GID2URI, cfOptions), new ColumnFamilyDescriptor(URI2GID, cfOptions));

	final List<ColumnFamilyHandle> columnFamilyHandles = new ArrayList<>();
	final RocksDB db = readOnly ? RocksDB.openReadOnly(dbOptions, kbDir, cfDescriptors, columnFamilyHandles) : RocksDB.open(dbOptions, kbDir, cfDescriptors, columnFamilyHandles);

	final KnowledgeBase kb;
	if (metadataExists) {
		kb = (KnowledgeBase) BinIO.loadObject(kbMetadataPathname);
		kb.readOnly = readOnly;
		kb.callGraphDB = db;
		kb.defaultHandle = columnFamilyHandles.get(0);
		kb.gid2uriFamilyHandle = columnFamilyHandles.get(1);
		kb.uri2gidFamilyHandle = columnFamilyHandles.get(2);
	} else kb = new KnowledgeBase(db, columnFamilyHandles.get(0), columnFamilyHandles.get(1), columnFamilyHandles.get(2), kbMetadataPathname, readOnly);
	return kb;
}
 

/**
 * Delete the record falls into [beginKeyBytes, endKeyBytes) of the db.
 *
 * @param db the target need to be clipped.
 * @param columnFamilyHandles the column family need to be clipped.
 * @param beginKeyBytes the begin key bytes
 * @param endKeyBytes the end key bytes
 */
private static void deleteRange(
	RocksDB db,
	List<ColumnFamilyHandle> columnFamilyHandles,
	byte[] beginKeyBytes,
	byte[] endKeyBytes) throws RocksDBException {

	for (ColumnFamilyHandle columnFamilyHandle : columnFamilyHandles) {
		try (RocksIteratorWrapper iteratorWrapper = RocksDBOperationUtils.getRocksIterator(db, columnFamilyHandle);
			RocksDBWriteBatchWrapper writeBatchWrapper = new RocksDBWriteBatchWrapper(db)) {

			iteratorWrapper.seek(beginKeyBytes);

			while (iteratorWrapper.isValid()) {
				final byte[] currentKey = iteratorWrapper.key();
				if (beforeThePrefixBytes(currentKey, endKeyBytes)) {
					writeBatchWrapper.remove(columnFamilyHandle, currentKey);
				} else {
					break;
				}
				iteratorWrapper.next();
			}
		}
	}
}
 

/**
 * Helper to load managed column family descriptors.
 */
private List<ColumnFamilyDescriptor> loadManagedColumnFamilies(DBOptions dbOptions) throws RocksDBException {
  final List<ColumnFamilyDescriptor> managedColumnFamilies = new ArrayList<>();
  final Options options = new Options(dbOptions, new ColumnFamilyOptions());
  List<byte[]> existing = RocksDB.listColumnFamilies(options, rocksDBBasePath);

  if (existing.isEmpty()) {
    LOG.info("No column family found. Loading default");
    managedColumnFamilies.add(getColumnFamilyDescriptor(RocksDB.DEFAULT_COLUMN_FAMILY));
  } else {
    LOG.info("Loading column families :" + existing.stream().map(String::new).collect(Collectors.toList()));
    managedColumnFamilies
        .addAll(existing.stream().map(RocksDBDAO::getColumnFamilyDescriptor).collect(Collectors.toList()));
  }
  return managedColumnFamilies;
}
 

@Override
@SuppressWarnings("PMD.CloseResource")
public void setup(final BenchmarkParams b) throws IOException {
  super.setup(b);
  wkb = new UnsafeBuffer(new byte[keySize]);
  wvb = new UnsafeBuffer(new byte[valSize]);
  loadLibrary();
  final Options options = new Options();
  options.setCreateIfMissing(true);
  options.setCompressionType(NO_COMPRESSION);
  try {
    db = open(options, tmp.getAbsolutePath());
  } catch (final RocksDBException ex) {
    throw new IOException(ex);
  }
}
 

@Test
public void databaseTest5() throws IOException, RocksDBException {
    var json = new JSONObject("{" +
            "\"index\": 2," +
            "\"product\": \"test\"," +
            "\"version\": \"0.0.1\"," +
            "\"nodes\": [9223372036854775804, 9223372036854775805, 9223372036854775806, 9223372036854775807]," +
            "\"numInternalNodes\": 3," +
            "\"edges\": [[9223372036854775804, 9223372036854775805], [9223372036854775804, 9223372036854775807], [9223372036854775805, 9223372036854775806], [9223372036854775806, 9223372036854775807]]" +
            "}");
    var graph = GidGraph.getGraph(json);
    rocksDao.saveToRocksDb(graph.getIndex(), graph.getNodes(), graph.getNumInternalNodes(), graph.getEdges());
    var graphData = rocksDao.getGraphData(graph.getIndex());
    assertEquals(graph.getNumInternalNodes(), graphData.nodes().size() - graphData.externalNodes().size());
    assertEquals(graph.getNodes().size(), graphData.nodes().size());
    assertEquals(new LongOpenHashSet(graph.getNodes()), graphData.nodes());
    assertEquals(new LongArrayList(List.of(9223372036854775805L, 9223372036854775807L)), graphData.successors(9223372036854775804L));
    assertEquals(new LongArrayList(List.of(9223372036854775806L)), graphData.successors(9223372036854775805L));
    assertEquals(new LongArrayList(List.of(9223372036854775807L)), graphData.successors(9223372036854775806L));
    assertEquals(new LongArrayList(), graphData.predecessors(9223372036854775804L));
    assertEquals(new LongArrayList(List.of(9223372036854775804L)), graphData.predecessors(9223372036854775805L));
    assertEquals(new LongArrayList(List.of(9223372036854775805L)), graphData.predecessors(9223372036854775806L));
    assertEquals(new LongArrayList(List.of(9223372036854775804L, 9223372036854775806L)), graphData.predecessors(9223372036854775807L));
    assertEquals(graph.getEdges().size(), graphData.numArcs());
    assertEquals(new LongOpenHashSet(List.of(9223372036854775807L)), graphData.externalNodes());
}
 

@Override
public SuffixDataStore createSuffixDataStore(int id) {
  RocksDB.loadLibrary();
  try (Options options = new Options().setCreateIfMissing(true).prepareForBulkLoad()) {
    Files.createDirectories(dbPath);
    RocksDB rocksDB = RocksDB.open(options, dbPath.resolve(getSuffixesName(id)).toString());
    rocksDBS.add(rocksDB);

    return new RocksDbSuffixDataStore(rocksDB);
  } catch (RocksDBException | IOException e) {
    throw new RuntimeException(e);
  }
}
 

@Override
public void put(final ColumnFamilyHandle segment, final byte[] key, final byte[] value) {
  try (final OperationTimer.TimingContext ignored = metrics.getWriteLatency().startTimer()) {
    innerTx.put(segment, key, value);
  } catch (final RocksDBException e) {
    throw new StorageException(e);
  }
}
 

public void add(String key, Sha256Hash hash)
{
  String s = name + "/" + key + "/" + hash.toString();
  byte b[]=new byte[0];
  try
  {
    db.put(jdb.getWriteOption(), s.getBytes(), b);
  }
  catch(RocksDBException e)
  {
    throw new RuntimeException(e);
  }

}
 

private long uri2GID(final FastenURI uri) {
	byte[] result;
	try {
		result = callGraphDB.get(uri2gidFamilyHandle, uri.toString().getBytes(StandardCharsets.UTF_8));
	} catch (final RocksDBException e) {
		throw new RuntimeException(e);
	}
	if (result == null) return -1;
	return Longs.fromByteArray(result);
}
 

private ByteStore newStore(String name) throws RocksDBException {
  if (inMemory) {
    return new MapStore(name);
  } else {
    final ColumnFamilyDescriptor columnFamilyDescriptor = new ColumnFamilyDescriptor(name.getBytes(UTF_8));
    ColumnFamilyHandle handle = db.createColumnFamily(columnFamilyDescriptor);
    handleIdToNameMap.put(handle.getID(), name);
    metadataManager.createEntry(name, false);

    return newRocksDBStore(name, columnFamilyDescriptor, handle);
  }
}
 

/**
 * 保存最新一个区块的Hash值
 *
 * @param tipBlockHash
 */
public void putLastBlockHash(String tipBlockHash) {
    try {
        blocksBucket.put(LAST_BLOCK_KEY, SerializeUtils.serialize(tipBlockHash));
        db.put(SerializeUtils.serialize(BLOCKS_BUCKET_KEY), SerializeUtils.serialize(blocksBucket));
    } catch (RocksDBException e) {
        log.error("Fail to put last block hash ! tipBlockHash=" + tipBlockHash, e);
        throw new RuntimeException("Fail to put last block hash ! tipBlockHash=" + tipBlockHash, e);
    }
}
 

/**
 * Helper to add delete operation in batch.
 *
 * @param batch Batch Handle
 * @param columnFamilyName Column Family
 * @param key Key
 */
public void deleteInBatch(WriteBatch batch, String columnFamilyName, String key) {
  try {
    batch.delete(managedHandlesMap.get(columnFamilyName), key.getBytes());
  } catch (RocksDBException e) {
    throw new HoodieException(e);
  }
}
 

public void put(String key, ByteString value)
{
  try
  {
    String key_str = name + "/" + key;

    db.put(jdb.getWriteOption(), key_str.getBytes(), value.toByteArray());

  }
  catch(RocksDBException e)
  {
    throw new RuntimeException(e);
  }
}
 

protected byte[] getBytes(byte[] keyBytes) {
    Preconditions.checkNotNull(this.db, "Make sure the database is open.");
    byte[] valueBytes;
    try {
        valueBytes = db.get(keyBytes);
    } catch (RocksDBException e) {
        String hexKey = BaseEncoding.base16().encode(keyBytes);
        throw new RuntimeException("Failed to get value from database for key: " + hexKey, e);
    }
    return valueBytes;
}
 

public void put(ByteString key, ByteString value)
{
  try
  {
    ByteString key_str = prefix.concat(key);
    db.put(jdb.getWriteOption(), key_str.toByteArray(), value.toByteArray());
  }
  catch(RocksDBException e)
  {
    throw new RuntimeException(e);
  }
}
 

@Nullable
@Override
public List<ConceptRow> forNorms(StringsBag norms) {
  if (norms.uniqueTerms() == 0) {
    return null;
  }
  try {
    byte[] bytes = normsDB.get(norms.getBytes());
    return bytes == null ? null : toList(bytes);
  } catch (RocksDBException e) {
    throw new RuntimeException(e);
  }
}
 

@Override
public List<PartOfSpeech> getCandidates(String word) {
  try {
    return getPartsOfSpeechFromBytes(
        candidatesDB.get(word.getBytes(StandardCharsets.UTF_8)));
  } catch (RocksDBException e) {
    throw new RuntimeException(e);
  }
}
 

@Override
public void put(UK userKey, UV userValue) throws IOException, RocksDBException {

	byte[] rawKeyBytes = serializeCurrentKeyWithGroupAndNamespacePlusUserKey(userKey, userKeySerializer);
	byte[] rawValueBytes = serializeValueNullSensitive(userValue, userValueSerializer);

	backend.db.put(columnFamily, writeOptions, rawKeyBytes, rawValueBytes);
}
 

public boolean put(final ColumnFamilyHandle cfh,final WriteOptions writeOptions,final byte[] key,final byte[] value){
    try {
        this.DB.put(cfh, writeOptions, key, value);
        log.debug("[RocksDB] -> success put value");
    } catch (RocksDBException e) {
        log.error("[RocksDB] -> error while put, columnFamilyHandle:{}, key:{}, err:{}",
                cfh.isOwningHandle(), new String(key), e.getMessage(), e);
        return false;
}
    return true;
}
 

private Response getAccessRule(Web.Request req) throws IOException, Web.ResponseException, RocksDBException {
    Index index = getIndex(req);
    Long ruleId = Long.parseLong(req.param("ruleId"));
    AccessRule rule = index.accessControl.rule(ruleId);
    if (rule == null) {
        return notFound();
    }
    return jsonResponse(rule);
}
 

@Override
public void markBeginPrepare() throws RocksDBException {
  /**
   * There are no use cases yet for this method in Recon. These will be
   * implemented as and when need arises.
   */
}