Java Code Examples for org.rocksdb.RocksIterator#isValid()
The following examples show how to use
org.rocksdb.RocksIterator#isValid() .
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: TestRocksDBMetronome.java From nifi with Apache License 2.0 | 6 votes |
@Test public void testIterator() throws Exception { try (RocksDBMetronome db = new RocksDBMetronome.Builder() .setStoragePath(temporaryFolder.newFolder().toPath()) .build()) { db.initialize(); db.put(KEY, VALUE); db.put(KEY_2, VALUE_2); RocksIterator iterator = db.getIterator(); iterator.seekToFirst(); Map<String, byte[]> recovered = new HashMap<>(); while (iterator.isValid()) { recovered.put(new String(iterator.key(), StandardCharsets.UTF_8), iterator.value()); iterator.next(); } assertEquals(2, recovered.size()); assertArrayEquals(VALUE, recovered.get(new String(KEY, StandardCharsets.UTF_8))); assertArrayEquals(VALUE_2, recovered.get(new String(KEY_2, StandardCharsets.UTF_8))); } }
Example 2
Source File: RocksDBLogStorage.java From sofa-jraft with Apache License 2.0 | 6 votes |
@Override public long getFirstLogIndex() { this.readLock.lock(); RocksIterator it = null; try { if (this.hasLoadFirstLogIndex) { return this.firstLogIndex; } checkState(); it = this.db.newIterator(this.defaultHandle, this.totalOrderReadOptions); it.seekToFirst(); if (it.isValid()) { final long ret = Bits.getLong(it.key(), 0); saveFirstLogIndex(ret); setFirstLogIndex(ret); return ret; } return 1L; } finally { if (it != null) { it.close(); } this.readLock.unlock(); } }
Example 3
Source File: RDB.java From iot-mqtt with Apache License 2.0 | 6 votes |
public List<byte[]> getByPrefix(final ColumnFamilyHandle cfh,final byte[] prefixKey){ List<byte[]> values = new ArrayList<>(); try{ RocksIterator iterator = this.newIterator(cfh); for(iterator.seek(prefixKey);iterator.isValid();iterator.next()){ if(new String(iterator.key()).startsWith(new String(prefixKey))) { values.add(iterator.value()); } } log.debug("[RocksDB] -> succ while get by prefix,columnFamilyHandle:{}, prefixKey:{}",cfh.toString(),new String(prefixKey)); }catch(Exception e){ log.error("[RocksDB] -> error while get by prefix, columnFamilyHandle:{}, prefixKey:{}, err:{}", cfh.toString(), new String(prefixKey), e); } return values; }
Example 4
Source File: RDB.java From iot-mqtt with Apache License 2.0 | 6 votes |
public int getCountByPrefix(final ColumnFamilyHandle cfh,final byte[] prefixKey){ int count = 0; try{ RocksIterator iterator = this.newIterator(cfh); for(iterator.seek(prefixKey);iterator.isValid();iterator.next()){ if(new String(iterator.key()).startsWith(new String(prefixKey))) { count++; } } log.debug("[RocksDB] -> succ while get count by prefix,columnFamilyHandle:{}, prefixKey:{}",cfh.toString(),new String(prefixKey)); }catch(Exception e){ log.error("[RocksDB] -> error while get count by prefix, columnFamilyHandle:{}, prefixKey:{}, err:{}", cfh.toString(), new String(prefixKey), e); } return count; }
Example 5
Source File: RDB.java From iot-mqtt with Apache License 2.0 | 6 votes |
public boolean deleteByPrefix(final ColumnFamilyHandle cfh,final byte[] prefixKey,boolean sync){ try{ RocksIterator iterator = this.newIterator(cfh); int item = 0; WriteBatch writeBatch = new WriteBatch(); for(iterator.seek(prefixKey);iterator.isValid();iterator.next()){ if(new String(iterator.key()).startsWith(new String(prefixKey))) { writeBatch.delete(cfh,iterator.key()); item++; } } if(item > 0){ this.DB.write(sync?WRITE_OPTIONS_SYNC:WRITE_OPTIONS_ASYNC,writeBatch); } log.debug("[RocksDB] -> succ while delete by prefix,columnFamilyHandle:{}, prefixKey:{}, nums:{}",cfh.toString(),new String(prefixKey),item); }catch(RocksDBException e){ log.error("[RocksDB] -> error while delete by prefix, columnFamilyHandle:{}, prefixKey:{}, err:{}", cfh.toString(), new String(prefixKey), e); return false; } return true; }
Example 6
Source File: Index.java From outbackcdx with Apache License 2.0 | 6 votes |
public Records(RocksDB db, ColumnFamilyHandle columnFamilyHandle, byte[] startKey, RecordConstructor<T> constructor, Predicate<T> scope, boolean reverse, long cap) { final RocksIterator it = db.newIterator(columnFamilyHandle); it.seek(startKey); if (reverse) { if (it.isValid()) { it.prev(); } else { it.seekToLast(); } } this.constructor = constructor; this.scope = scope; this.it = it; this.reverse = reverse; this.cap = cap; }
Example 7
Source File: RocksDBLookupTable.java From kylin with Apache License 2.0 | 5 votes |
@Override public Iterator<String[]> iterator() { final RocksIterator rocksIterator = getRocksIterator(); rocksIterator.seekToFirst(); return new Iterator<String[]>() { int counter; @Override public boolean hasNext() { boolean valid = rocksIterator.isValid(); if (!valid) { rocksIterator.close(); } return valid; } @Override public String[] next() { counter++; if (counter % 100000 == 0) { logger.info("scanned {} rows from rocksDB", counter); } String[] result = rowEncoder.decode(new KV(rocksIterator.key(), rocksIterator.value())); rocksIterator.next(); return result; } @Override public void remove() { throw new UnsupportedOperationException("not support operation"); } }; }
Example 8
Source File: RocksDBDAO.java From hudi with Apache License 2.0 | 5 votes |
/** * Perform a prefix delete and return stream of key-value pairs retrieved. * * @param columnFamilyName Column Family Name * @param prefix Prefix Key * @param <T> Type of value stored */ public <T extends Serializable> void prefixDelete(String columnFamilyName, String prefix) { ValidationUtils.checkArgument(!closed); LOG.info("Prefix DELETE (query=" + prefix + ") on " + columnFamilyName); final RocksIterator it = getRocksDB().newIterator(managedHandlesMap.get(columnFamilyName)); it.seek(prefix.getBytes()); // Find first and last keys to be deleted String firstEntry = null; String lastEntry = null; while (it.isValid() && new String(it.key()).startsWith(prefix)) { String result = new String(it.key()); it.next(); if (firstEntry == null) { firstEntry = result; } lastEntry = result; } it.close(); if (null != firstEntry) { try { // This will not delete the last entry getRocksDB().deleteRange(managedHandlesMap.get(columnFamilyName), firstEntry.getBytes(), lastEntry.getBytes()); // Delete the last entry getRocksDB().delete(lastEntry.getBytes()); } catch (RocksDBException e) { LOG.error("Got exception performing range delete"); throw new HoodieException(e); } } }
Example 9
Source File: RocksDb.java From benchmarks with Apache License 2.0 | 5 votes |
@Benchmark @SuppressWarnings("PMD.CloseResource") public void readXxh64(final Reader r, final Blackhole bh) { long result = 0; final RocksIterator iterator = r.db.newIterator(); iterator.seekToFirst(); while (iterator.isValid()) { result += xx_r39().hashBytes(iterator.key()); result += xx_r39().hashBytes(iterator.value()); iterator.next(); } bh.consume(result); }
Example 10
Source File: RocksDbStorage.java From deeplearning4j with Apache License 2.0 | 5 votes |
/** * Clear the array storage */ @Override public void clear() { RocksIterator iterator = db.newIterator(); while (iterator.isValid()) try { db.remove(iterator.key()); } catch (RocksDBException e) { throw new RuntimeException(e); } iterator.close(); size = 0; }
Example 11
Source File: RocksDbIdentifiers.java From biomedicus with Apache License 2.0 | 5 votes |
@Override public MappingIterator mappingIterator() { RocksIterator rocksIterator = indices.newIterator(); return new MappingIterator() { @Override public void close() { rocksIterator.close(); } @Override public boolean isValid() { return rocksIterator.isValid(); } @Override public int identifier() { byte[] value = rocksIterator.value(); return ByteBuffer.wrap(value).getInt(); } @Override public String string() { byte[] key = rocksIterator.key(); return new String(key, StandardCharsets.UTF_8); } @Override public void next() { rocksIterator.next(); } }; }
Example 12
Source File: RocksDBManager.java From WebCollector with GNU General Public License v3.0 | 5 votes |
public void list() throws Exception { String crawldbPath = FilenameUtils.concat(crawlPath, "crawldb"); RocksDB crawldbDatabase = RocksDBUtils.open(crawldbPath); RocksIterator crawldbIterator = crawldbDatabase.newIterator(); for(crawldbIterator.seekToFirst(); crawldbIterator.isValid(); crawldbIterator.next()){ CrawlDatum datum = RocksDBUtils.createCrawlDatum(crawldbIterator.key(), crawldbIterator.value()); System.out.println(CrawlDatumFormater.datumToString(datum)); } crawldbDatabase.close(); }
Example 13
Source File: RocksDbStorage.java From nd4j with Apache License 2.0 | 5 votes |
/** * Clear the array storage */ @Override public void clear() { RocksIterator iterator = db.newIterator(); while (iterator.isValid()) try { db.remove(iterator.key()); } catch (RocksDBException e) { throw new RuntimeException(e); } iterator.close(); size = 0; }
Example 14
Source File: RocksdbMap.java From Lealone-Plugins with Apache License 2.0 | 5 votes |
@Override public StorageMapCursor<K, V> cursor(K from) { RocksIterator iterator = db.newIterator(); if (from == null) { iterator.seekToFirst(); return new RocksdbMapCursor<>(this, iterator); } for (iterator.seekToFirst(); iterator.isValid(); iterator.next()) { K key = k(iterator.key()); if (keyType.compare(key, from) >= 0) { break; } } return new RocksdbMapCursor<>(this, iterator); }
Example 15
Source File: RocksDBLookupTable.java From kylin-on-parquet-v2 with Apache License 2.0 | 5 votes |
@Override public Iterator<String[]> iterator() { final RocksIterator rocksIterator = getRocksIterator(); rocksIterator.seekToFirst(); return new Iterator<String[]>() { int counter; @Override public boolean hasNext() { boolean valid = rocksIterator.isValid(); if (!valid) { rocksIterator.close(); } return valid; } @Override public String[] next() { counter++; if (counter % 100000 == 0) { logger.info("scanned {} rows from rocksDB", counter); } String[] result = rowEncoder.decode(new KV(rocksIterator.key(), rocksIterator.value())); rocksIterator.next(); return result; } @Override public void remove() { throw new UnsupportedOperationException("not support operation"); } }; }
Example 16
Source File: RocksDBStore.java From hadoop-ozone with Apache License 2.0 | 5 votes |
@Override public ImmutablePair<byte[], byte[]> peekAround(int offset, byte[] from) throws IOException, IllegalArgumentException { RocksIterator it = null; try { it = db.newIterator(); if (from == null) { it.seekToFirst(); } else { it.seek(from); } if (!it.isValid()) { return null; } switch (offset) { case 0: break; case 1: it.next(); break; case -1: it.prev(); break; default: throw new IllegalArgumentException( "Position can only be -1, 0 " + "or 1, but found " + offset); } return it.isValid() ? new ImmutablePair<>(it.key(), it.value()) : null; } finally { if (it != null) { it.close(); } } }
Example 17
Source File: PubchemTTLMerger.java From act with GNU General Public License v3.0 | 4 votes |
protected void merge(Pair<RocksDB, Map<COLUMN_FAMILIES, ColumnFamilyHandle>> dbAndHandles) throws RocksDBException, IOException, ClassNotFoundException { LOGGER.info("Beginning merge on Pubchem CID"); RocksDB db = dbAndHandles.getLeft(); ColumnFamilyHandle pubchemIdCFH = dbAndHandles.getRight().get(COLUMN_FAMILIES.CID_TO_HASHES); ColumnFamilyHandle meshCFH = dbAndHandles.getRight().get(COLUMN_FAMILIES.HASH_TO_MESH); ColumnFamilyHandle synonymCFH = dbAndHandles.getRight().get(COLUMN_FAMILIES.HASH_TO_SYNONYMS); ColumnFamilyHandle synonymTypeCFH = dbAndHandles.getRight().get(COLUMN_FAMILIES.HASH_TO_SYNONYM_TYPE); ColumnFamilyHandle mergeResultsCFH = dbAndHandles.getRight().get(COLUMN_FAMILIES.CID_TO_SYNONYMS); RocksIterator cidIterator = db.newIterator(pubchemIdCFH); // With help from https://github.com/facebook/rocksdb/wiki/Basic-Operations int processed = 0; for (cidIterator.seekToFirst(); cidIterator.isValid(); cidIterator.next()) { byte[] key = cidIterator.key(); byte[] val = cidIterator.value(); String pubchemId = new String(key, UTF8); List<String> hashes; try (ObjectInputStream ois = new ObjectInputStream(new ByteArrayInputStream(val))) { // We know all our values so far have been lists of strings, so this should be completely safe. hashes = (List<String>) ois.readObject(); } PubchemSynonyms pubchemSynonyms = new PubchemSynonyms(pubchemId); /* The hash keys are based on synonym value, which we can manually compute with: * $ echo -n 'dimethyltin(iv)' | md5 * This means that MeSH ids are linked to synonyms rather than pubchem ids. We need to look up each cid-linked * hash in both the MeSH and synonym collections, as the key may legitimately exist in both (and serve to link * cid to synonym and cid to MeSH). */ for (String hash : hashes) { /* Note: these ids are not proper MeSH topic ids, but are internal MeSH ids found in the RDF and TTL * representations of the MeSH corpus. You can find them in the MeSH .nt or .xml files, but they won't turn up * anything on the MeSH website. */ List<String> meshIds = getValueAsObject(db, meshCFH, hash); if (meshIds != null) { pubchemSynonyms.addMeSHIds(meshIds); } List<String> synonyms = getValueAsObject(db, synonymCFH, hash); // There are, surprisingly, some dangling hashes in the DB! Handle them gracefully. if (synonyms == null) { LOGGER.warn("Dangling synonym hash reference, adding empty list in place of value: cid = %s, hash = %s", pubchemId, hash); synonyms = Collections.emptyList(); } List<String> synonymTypeStrings = getValueAsObject(db, synonymTypeCFH, hash); Set<PC_SYNONYM_TYPES> synonymTypes = DEFAULT_SYNONYM_DATA_TYPES; if (synonymTypeStrings != null) { synonymTypes = synonymTypeStrings.stream().map(PC_SYNONYM_TYPES::valueOf).collect(Collectors.toSet()); } if (synonymTypes.size() == 0) { LOGGER.warn("Found zero synonym types for synonym, defaulting to %s: %s %s, synonyms = %s", PC_SYNONYM_TYPES.UNKNOWN.name(), pubchemId, hash, StringUtils.join(synonyms, ", ")); } /* It turns out that *lots* of synonyms are duplicated as depositor supplied names, so don't complain about it * here. For performance sake we might want to consider changing the data model of PubchemSynonyms to reduce * synonym string duplication, as the current model is pretty inefficient. */ for (PC_SYNONYM_TYPES synonymType : synonymTypes) { for (String synonym : synonyms) { // Let the PubchemSynonyms object do the de-duplication for us rather than reducing `synonyms` to a Set. pubchemSynonyms.addSynonym(synonymType, synonym); } } } try (ByteArrayOutputStream bos = new ByteArrayOutputStream(); ObjectOutputStream oo = new ObjectOutputStream(bos)) { oo.writeObject(pubchemSynonyms); oo.flush(); db.put(mergeResultsCFH, key, bos.toByteArray()); } processed++; if (processed % 100000 == 0) { LOGGER.info("Merged %d entries on Pubchem compound id", processed); } } LOGGER.info("Merge complete, %d entries processed", processed); }
Example 18
Source File: TestDB.java From DDMQ with Apache License 2.0 | 4 votes |
@Test public void testDeleteRange() { init(); WriteBatch wb = new WriteBatch(); ColumnFamilyHandle cfHandle = CFManager.CFH_DEFAULT; long st = System.currentTimeMillis(); for(int i=100000; i<200000; i++) { wb.put(cfHandle, ("1324356527-" + i + "-5-5-345-356-234-232").getBytes(), "tasdfasdgasdfestfordb".getBytes()); if(i % 30 == 0) { RDB.writeAsync(wb); wb.clear(); } } for(int i=100000; i<200000; i++) { wb.put(cfHandle, ("1324356525-" + i + "-5-5-345-356-234-232").getBytes(), "tasdfasdgasdfestfordb".getBytes()); if(i % 30 == 0) { RDB.writeAsync(wb); wb.clear(); } } for(int i=100000; i<200000; i++) { wb.put(cfHandle, ("1324356529-" + i + "-5-5-345-356-234-232").getBytes(), "tasdfasdgasdfestfordb".getBytes()); if(i % 30 == 0) { RDB.writeAsync(wb); wb.clear(); } } RDB.writeAsync(wb); long ed = System.currentTimeMillis(); System.out.println("write cost :" + (ed - st)); try { TimeUnit.SECONDS.sleep(2); } catch (InterruptedException e) { e.printStackTrace(); } long start = System.currentTimeMillis(); RocksIterator it = RDB.newIterator(cfHandle); byte[] now = "1324356527".getBytes(); long count = 0; for(it.seek(now); it.isValid(); it.next()) { // System.out.println(new String(it.key()) + " " + new String(it.value())); count++; if(count == 100000) break; } it.close(); long end = System.currentTimeMillis(); System.out.println("cost : " + (end - start) + " count:" +count); RDB.deleteFilesInRange(CFManager.CFH_DEFAULT, "132435653".getBytes(), "1324356529".getBytes()); count = 0; it = RDB.newIterator(cfHandle); now = "1324356525".getBytes(); for(it.seek(now); it.isValid(); it.next()) { // System.out.println(new String(it.key()) + " " + new String(it.value())); count++; if(count == 100000) break; } it.close(); end = System.currentTimeMillis(); System.out.println("cost : " + (end - start) + " count:" +count); destructor(); }
Example 19
Source File: RocksDBStore.java From hadoop-ozone with Apache License 2.0 | 4 votes |
private List<Map.Entry<byte[], byte[]>> getRangeKVs(byte[] startKey, int count, boolean sequential, MetadataKeyFilters.MetadataKeyFilter... filters) throws IOException, IllegalArgumentException { List<Map.Entry<byte[], byte[]>> result = new ArrayList<>(); long start = System.currentTimeMillis(); if (count < 0) { throw new IllegalArgumentException( "Invalid count given " + count + ", count must be greater than 0"); } RocksIterator it = null; try { it = db.newIterator(); if (startKey == null) { it.seekToFirst(); } else { if(get(startKey) == null) { // Key not found, return empty list return result; } it.seek(startKey); } while(it.isValid() && result.size() < count) { byte[] currentKey = it.key(); byte[] currentValue = it.value(); it.prev(); final byte[] prevKey = it.isValid() ? it.key() : null; it.seek(currentKey); it.next(); final byte[] nextKey = it.isValid() ? it.key() : null; if (filters == null) { result.add(new AbstractMap.SimpleImmutableEntry<>(currentKey, currentValue)); } else { if (Arrays.asList(filters).stream() .allMatch(entry -> entry.filterKey(prevKey, currentKey, nextKey))) { result.add(new AbstractMap.SimpleImmutableEntry<>(currentKey, currentValue)); } else { if (result.size() > 0 && sequential) { // if the caller asks for a sequential range of results, // and we met a dis-match, abort iteration from here. // if result is empty, we continue to look for the first match. break; } } } } } finally { if (it != null) { it.close(); } long end = System.currentTimeMillis(); long timeConsumed = end - start; if (LOG.isDebugEnabled()) { if (filters != null) { for (MetadataKeyFilters.MetadataKeyFilter filter : filters) { int scanned = filter.getKeysScannedNum(); int hinted = filter.getKeysHintedNum(); if (scanned > 0 || hinted > 0) { LOG.debug( "getRangeKVs ({}) numOfKeysScanned={}, numOfKeysHinted={}", filter.getClass().getSimpleName(), filter.getKeysScannedNum(), filter.getKeysHintedNum()); } } } LOG.debug("Time consumed for getRangeKVs() is {}ms," + " result length is {}.", timeConsumed, result.size()); } } return result; }
Example 20
Source File: RocksDBMapSet.java From jelectrum with MIT License | 4 votes |
public Set<Sha256Hash> getSet(String key, int max_reply) { String s = name + "/" + key + "/"; HashSet<Sha256Hash> set = new HashSet<Sha256Hash>(); int count = 0; RocksIterator it = db.newIterator(); try { it.seek(s.getBytes()); while(it.isValid()) { String curr_key = new String(it.key()); if (!curr_key.startsWith(s)) break; String hash_string = curr_key.substring(s.length()); set.add(Sha256Hash.wrap(hash_string)); count++; if (count > max_reply) throw new DBTooManyResultsException(); it.next(); } } finally { it.dispose(); } /*catch(RocksDBException e) { throw new RuntimeException(e); }*/ return set; }