Java Code Examples for org.rocksdb.RocksDB#put()

/**
 * Fill database with some test values.
 *
 * @param db {@link RocksDB} instance.
 * @throws RocksDBException thrown if an error occurs within the native
 *                          part of the library.
 */
private void prepareDatabase(final RocksDB db)
        throws RocksDBException {
    db.put("key1".getBytes(), "valueV1".getBytes());
    db.put("key2".getBytes(), "valueV1".getBytes());
    db.put("key3".getBytes(), "valueV1".getBytes());
    db.put("key4".getBytes(), "valueV1".getBytes());
}
 

/**
 * Fill database with some test values.
 *
 * @param db {@link RocksDB} instance.
 * @throws RocksDBException thrown if an error occurs within the native
 *                          part of the library.
 */
private void prepareDatabase(final RocksDB db)
        throws RocksDBException {
    db.put("key1".getBytes(), "valueV1".getBytes());
    db.put("key2".getBytes(), "valueV1".getBytes());
    db.put("key3".getBytes(), "valueV1".getBytes());
    db.put("key4".getBytes(), "valueV1".getBytes());
}
 

@Override
    public void merge() throws Exception {
        LOG.info("start merge");
        RocksDB crawldbDatabase = RocksDBUtils.openCrawldbDatabase(crawlPath);


        /*合并fetch库*/
        LOG.info("merge fetch database");
        RocksDB fetchDatabase = RocksDBUtils.openFetchDatabase(crawlPath);
        RocksIterator fetchIterator = fetchDatabase.newIterator();
        for(fetchIterator.seekToFirst(); fetchIterator.isValid(); fetchIterator.next()){
            crawldbDatabase.put(fetchIterator.key(), fetchIterator.value());
        }
        fetchDatabase.close();

        /*合并link库*/
        LOG.info("merge link database");
        RocksDB linkDatabase = RocksDBUtils.openLinkDatabase(crawlPath);
        RocksIterator linkIterator = linkDatabase.newIterator();
        for(linkIterator.seekToFirst(); linkIterator.isValid(); linkIterator.next()){
            if(crawldbDatabase.get(linkIterator.key()) == null){
                crawldbDatabase.put(linkIterator.key(), linkIterator.value());
            }
        }
        linkDatabase.close();

        LOG.info("end merge");
        crawldbDatabase.close();



//        env.removeDatabase(null, "fetch");
        RocksDBUtils.destroyFetchDatabase(crawlPath);
        LOG.debug("remove fetch database");
//        env.removeDatabase(null, "link");
        RocksDBUtils.destroyLinkDatabase(crawlPath);
        LOG.debug("remove link database");

    }
 

private static void rocksDbTest(RocksDB db, List<ColumnFamilyHandle> handlers) {
    try {
        ColumnFamilyHandle handler1 = null;
        ColumnFamilyHandle handler2 = null;
        if (handlers.size() > 0) {
            // skip default column family
            handler1 = handlers.get(1);
            handler2 = handlers.get(2);
        } else {
            handler1 = db.createColumnFamily(new ColumnFamilyDescriptor("test1".getBytes()));
            handler2 = db.createColumnFamily(new ColumnFamilyDescriptor("test2".getBytes()));
        }
        int startValue1 = getStartValue(db, handler1);
        int startValue2 = getStartValue(db, handler2);;

        Checkpoint cp = Checkpoint.create(db);
   
        if (isCompaction) {
            db.compactRange();
            LOG.info("Compaction!");
        }

        long flushWaitTime = System.currentTimeMillis() + flushInterval;
        for (int i = 0; i < putNum || putNum == -1; i++) {
            db.put(handler1, String.valueOf(i % 1000).getBytes(), String.valueOf(startValue1 + i).getBytes());
            db.put(handler2, String.valueOf(i % 1000).getBytes(), String.valueOf(startValue2 + i).getBytes());
            if (isFlush && flushWaitTime <= System.currentTimeMillis()) {
                db.flush(new FlushOptions());
                if (isCheckpoint) {
                    cp.createCheckpoint(cpPath + "/" + i);
                }
                flushWaitTime = System.currentTimeMillis() + flushInterval;
            }
        }
    } catch (RocksDBException e) {
        LOG.error("Failed to put or flush", e);
    }
}
 

@Test
public void restoreLatestBackup() throws RocksDBException {
    try (final Options opt = new Options().setCreateIfMissing(true)) {
        // Open empty database.
        RocksDB db = null;
        try {
            db = RocksDB.open(opt,
                    dbFolder.getRoot().getAbsolutePath());
            // Fill database with some test values
            prepareDatabase(db);

            try (final BackupableDBOptions bopt = new BackupableDBOptions(
                    backupFolder.getRoot().getAbsolutePath());
                 final BackupEngine be = BackupEngine.open(opt.getEnv(), bopt)) {
                be.createNewBackup(db, true);
                verifyNumberOfValidBackups(be, 1);
                db.put("key1".getBytes(), "valueV2".getBytes());
                db.put("key2".getBytes(), "valueV2".getBytes());
                be.createNewBackup(db, true);
                verifyNumberOfValidBackups(be, 2);
                db.put("key1".getBytes(), "valueV3".getBytes());
                db.put("key2".getBytes(), "valueV3".getBytes());
                assertThat(new String(db.get("key1".getBytes()))).endsWith("V3");
                assertThat(new String(db.get("key2".getBytes()))).endsWith("V3");

                db.close();
                db = null;

                verifyNumberOfValidBackups(be, 2);
                // restore db from latest backup
                try (final RestoreOptions ropts = new RestoreOptions(false)) {
                    be.restoreDbFromLatestBackup(dbFolder.getRoot().getAbsolutePath(),
                            dbFolder.getRoot().getAbsolutePath(), ropts);
                }

                // Open database again.
                db = RocksDB.open(opt, dbFolder.getRoot().getAbsolutePath());

                // Values must have suffix V2 because of restoring latest backup.
                assertThat(new String(db.get("key1".getBytes()))).endsWith("V2");
                assertThat(new String(db.get("key2".getBytes()))).endsWith("V2");
            }
        } finally {
            if (db != null) {
                db.close();
            }
        }
    }
}
 

@Test
public void restoreFromBackup()
        throws RocksDBException {
    try (final Options opt = new Options().setCreateIfMissing(true)) {
        RocksDB db = null;
        try {
            // Open empty database.
            db = RocksDB.open(opt,
                    dbFolder.getRoot().getAbsolutePath());
            // Fill database with some test values
            prepareDatabase(db);
            try (final BackupableDBOptions bopt = new BackupableDBOptions(
                    backupFolder.getRoot().getAbsolutePath());
                 final BackupEngine be = BackupEngine.open(opt.getEnv(), bopt)) {
                be.createNewBackup(db, true);
                verifyNumberOfValidBackups(be, 1);
                db.put("key1".getBytes(), "valueV2".getBytes());
                db.put("key2".getBytes(), "valueV2".getBytes());
                be.createNewBackup(db, true);
                verifyNumberOfValidBackups(be, 2);
                db.put("key1".getBytes(), "valueV3".getBytes());
                db.put("key2".getBytes(), "valueV3".getBytes());
                assertThat(new String(db.get("key1".getBytes()))).endsWith("V3");
                assertThat(new String(db.get("key2".getBytes()))).endsWith("V3");

                //close the database
                db.close();
                db = null;

                //restore the backup
                final List<BackupInfo> backupInfo = verifyNumberOfValidBackups(be, 2);
                // restore db from first backup
                be.restoreDbFromBackup(backupInfo.get(0).backupId(),
                        dbFolder.getRoot().getAbsolutePath(),
                        dbFolder.getRoot().getAbsolutePath(),
                        new RestoreOptions(false));
                // Open database again.
                db = RocksDB.open(opt,
                        dbFolder.getRoot().getAbsolutePath());
                // Values must have suffix V2 because of restoring latest backup.
                assertThat(new String(db.get("key1".getBytes()))).endsWith("V1");
                assertThat(new String(db.get("key2".getBytes()))).endsWith("V1");
            }
        } finally {
            if (db != null) {
                db.close();
            }
        }
    }
}
 

@Test
public void restoreLatestBackup() throws RocksDBException {
    try (final Options opt = new Options().setCreateIfMissing(true)) {
        // Open empty database.
        RocksDB db = null;
        try {
            db = RocksDB.open(opt,
                    dbFolder.getRoot().getAbsolutePath());
            // Fill database with some test values
            prepareDatabase(db);

            try (final BackupableDBOptions bopt = new BackupableDBOptions(
                    backupFolder.getRoot().getAbsolutePath());
                 final BackupEngine be = BackupEngine.open(opt.getEnv(), bopt)) {
                be.createNewBackup(db, true);
                verifyNumberOfValidBackups(be, 1);
                db.put("key1".getBytes(), "valueV2".getBytes());
                db.put("key2".getBytes(), "valueV2".getBytes());
                be.createNewBackup(db, true);
                verifyNumberOfValidBackups(be, 2);
                db.put("key1".getBytes(), "valueV3".getBytes());
                db.put("key2".getBytes(), "valueV3".getBytes());
                assertThat(new String(db.get("key1".getBytes()))).endsWith("V3");
                assertThat(new String(db.get("key2".getBytes()))).endsWith("V3");

                db.close();
                db = null;

                verifyNumberOfValidBackups(be, 2);
                // restore db from latest backup
                try (final RestoreOptions ropts = new RestoreOptions(false)) {
                    be.restoreDbFromLatestBackup(dbFolder.getRoot().getAbsolutePath(),
                            dbFolder.getRoot().getAbsolutePath(), ropts);
                }

                // Open database again.
                db = RocksDB.open(opt, dbFolder.getRoot().getAbsolutePath());

                // Values must have suffix V2 because of restoring latest backup.
                assertThat(new String(db.get("key1".getBytes()))).endsWith("V2");
                assertThat(new String(db.get("key2".getBytes()))).endsWith("V2");
            }
        } finally {
            if (db != null) {
                db.close();
            }
        }
    }
}
 

@Test
public void restoreFromBackup()
        throws RocksDBException {
    try (final Options opt = new Options().setCreateIfMissing(true)) {
        RocksDB db = null;
        try {
            // Open empty database.
            db = RocksDB.open(opt,
                    dbFolder.getRoot().getAbsolutePath());
            // Fill database with some test values
            prepareDatabase(db);
            try (final BackupableDBOptions bopt = new BackupableDBOptions(
                    backupFolder.getRoot().getAbsolutePath());
                 final BackupEngine be = BackupEngine.open(opt.getEnv(), bopt)) {
                be.createNewBackup(db, true);
                verifyNumberOfValidBackups(be, 1);
                db.put("key1".getBytes(), "valueV2".getBytes());
                db.put("key2".getBytes(), "valueV2".getBytes());
                be.createNewBackup(db, true);
                verifyNumberOfValidBackups(be, 2);
                db.put("key1".getBytes(), "valueV3".getBytes());
                db.put("key2".getBytes(), "valueV3".getBytes());
                assertThat(new String(db.get("key1".getBytes()))).endsWith("V3");
                assertThat(new String(db.get("key2".getBytes()))).endsWith("V3");

                //close the database
                db.close();
                db = null;

                //restore the backup
                final List<BackupInfo> backupInfo = verifyNumberOfValidBackups(be, 2);
                // restore db from first backup
                be.restoreDbFromBackup(backupInfo.get(0).backupId(),
                        dbFolder.getRoot().getAbsolutePath(),
                        dbFolder.getRoot().getAbsolutePath(),
                        new RestoreOptions(false));
                // Open database again.
                db = RocksDB.open(opt,
                        dbFolder.getRoot().getAbsolutePath());
                // Values must have suffix V2 because of restoring latest backup.
                assertThat(new String(db.get("key1".getBytes()))).endsWith("V1");
                assertThat(new String(db.get("key2".getBytes()))).endsWith("V1");
            }
        } finally {
            if (db != null) {
                db.close();
            }
        }
    }
}
 

protected void merge(Pair<RocksDB, Map<COLUMN_FAMILIES, ColumnFamilyHandle>> dbAndHandles)
    throws RocksDBException, IOException, ClassNotFoundException {
  LOGGER.info("Beginning merge on Pubchem CID");
  RocksDB db = dbAndHandles.getLeft();
  ColumnFamilyHandle pubchemIdCFH = dbAndHandles.getRight().get(COLUMN_FAMILIES.CID_TO_HASHES);
  ColumnFamilyHandle meshCFH = dbAndHandles.getRight().get(COLUMN_FAMILIES.HASH_TO_MESH);
  ColumnFamilyHandle synonymCFH = dbAndHandles.getRight().get(COLUMN_FAMILIES.HASH_TO_SYNONYMS);
  ColumnFamilyHandle synonymTypeCFH = dbAndHandles.getRight().get(COLUMN_FAMILIES.HASH_TO_SYNONYM_TYPE);
  ColumnFamilyHandle mergeResultsCFH = dbAndHandles.getRight().get(COLUMN_FAMILIES.CID_TO_SYNONYMS);

  RocksIterator cidIterator = db.newIterator(pubchemIdCFH);
  // With help from https://github.com/facebook/rocksdb/wiki/Basic-Operations
  int processed = 0;
  for (cidIterator.seekToFirst(); cidIterator.isValid(); cidIterator.next()) {
    byte[] key = cidIterator.key();
    byte[] val = cidIterator.value();
    String pubchemId = new String(key, UTF8);
    List<String> hashes;
    try (ObjectInputStream ois = new ObjectInputStream(new ByteArrayInputStream(val))) {
      // We know all our values so far have been lists of strings, so this should be completely safe.
      hashes = (List<String>) ois.readObject();
    }

    PubchemSynonyms pubchemSynonyms = new PubchemSynonyms(pubchemId);

    /* The hash keys are based on synonym value, which we can manually compute with:
     *   $ echo -n  'dimethyltin(iv)' | md5
     * This means that MeSH ids are linked to synonyms rather than pubchem ids.  We need to look up each cid-linked
     * hash in both the MeSH and synonym collections, as the key may legitimately exist in both (and serve to link
     * cid to synonym and cid to MeSH). */
    for (String hash : hashes) {
      /* Note: these ids are not proper MeSH topic ids, but are internal MeSH ids found in the RDF and TTL
       * representations of the MeSH corpus.  You can find them in the MeSH .nt or .xml files, but they won't turn up
       * anything on the MeSH website. */
      List<String> meshIds = getValueAsObject(db, meshCFH, hash);
      if (meshIds != null) {
        pubchemSynonyms.addMeSHIds(meshIds);
      }

      List<String> synonyms = getValueAsObject(db, synonymCFH, hash);
      // There are, surprisingly, some dangling hashes in the DB!  Handle them gracefully.
      if (synonyms == null) {
        LOGGER.warn("Dangling synonym hash reference, adding empty list in place of value: cid = %s, hash = %s",
            pubchemId, hash);
        synonyms = Collections.emptyList();
      }

      List<String> synonymTypeStrings = getValueAsObject(db, synonymTypeCFH, hash);
      Set<PC_SYNONYM_TYPES> synonymTypes = DEFAULT_SYNONYM_DATA_TYPES;
      if (synonymTypeStrings != null) {
        synonymTypes = synonymTypeStrings.stream().map(PC_SYNONYM_TYPES::valueOf).collect(Collectors.toSet());
      }

      if (synonymTypes.size() == 0) {
        LOGGER.warn("Found zero synonym types for synonym, defaulting to %s: %s %s, synonyms = %s",
            PC_SYNONYM_TYPES.UNKNOWN.name(), pubchemId, hash, StringUtils.join(synonyms, ", "));
      }
      /* It turns out that *lots* of synonyms are duplicated as depositor supplied names, so don't complain about it
       * here.  For performance sake we might want to consider changing the data model of PubchemSynonyms to reduce
       * synonym string duplication, as the current model is pretty inefficient. */

      for (PC_SYNONYM_TYPES synonymType : synonymTypes) {
        for (String synonym : synonyms) {
          // Let the PubchemSynonyms object do the de-duplication for us rather than reducing `synonyms` to a Set.
          pubchemSynonyms.addSynonym(synonymType, synonym);
        }
      }
    }

    try (ByteArrayOutputStream bos = new ByteArrayOutputStream();
         ObjectOutputStream oo = new ObjectOutputStream(bos)) {
      oo.writeObject(pubchemSynonyms);
      oo.flush();

      db.put(mergeResultsCFH, key, bos.toByteArray());
    }

    processed++;
    if (processed % 100000 == 0) {
      LOGGER.info("Merged %d entries on Pubchem compound id", processed);
    }
  }
  LOGGER.info("Merge complete, %d entries processed", processed);
}
 

public static void put(RocksDB rocksDB, String key,String value) throws UnsupportedEncodingException, RocksDBException {
    rocksDB.put(strToKeyOrValue(key), strToKeyOrValue(value));
}