org.rocksdb.ReadOptions Java Examples

private void writeSnapshotToOutputStream(
	@Nonnull CheckpointStreamWithResultProvider checkpointStreamWithResultProvider,
	@Nonnull KeyGroupRangeOffsets keyGroupRangeOffsets) throws IOException, InterruptedException {

	final List<Tuple2<RocksIteratorWrapper, Integer>> kvStateIterators =
		new ArrayList<>(metaData.size());
	final DataOutputView outputView =
		new DataOutputViewStreamWrapper(checkpointStreamWithResultProvider.getCheckpointOutputStream());
	final ReadOptions readOptions = new ReadOptions();
	try {
		readOptions.setSnapshot(snapshot);
		writeKVStateMetaData(kvStateIterators, readOptions, outputView);
		writeKVStateData(kvStateIterators, checkpointStreamWithResultProvider, keyGroupRangeOffsets);
	} finally {

		for (Tuple2<RocksIteratorWrapper, Integer> kvStateIterator : kvStateIterators) {
			IOUtils.closeQuietly(kvStateIterator.f0);
		}

		IOUtils.closeQuietly(readOptions);
	}
}
 

public CloseableIterator<GeoWaveRow> dataIndexIterator(
    final byte[] startDataId,
    final byte[] endDataId) {
  final RocksDB readDb = getReadDb();
  if (readDb == null) {
    return new CloseableIterator.Empty<>();
  }
  final ReadOptions options;
  final RocksIterator it;
  if (endDataId == null) {
    options = null;
    it = readDb.newIterator();
  } else {
    options =
        new ReadOptions().setIterateUpperBound(
            new Slice(ByteArrayUtils.getNextPrefix(endDataId)));
    it = readDb.newIterator(options);
  }
  if (startDataId == null) {
    it.seekToFirst();
  } else {
    it.seek(startDataId);
  }
  return new DataIndexRowIterator(options, it, adapterId, visibilityEnabled);
}
 

public CloseableIterator<GeoWaveRow> iterator() {
  final RocksDB readDb = getReadDb();
  if (readDb == null) {
    return new CloseableIterator.Empty<>();
  }
  final ReadOptions options = new ReadOptions().setFillCache(false);
  final RocksIterator it = readDb.newIterator(options);
  it.seekToFirst();
  return new RocksDBRowIterator(
      options,
      it,
      adapterId,
      partition,
      requiresTimestamp,
      visibilityEnabled);
}
 

private void writeKVStateMetaData(
	final List<Tuple2<RocksIteratorWrapper, Integer>> kvStateIterators,
	final ReadOptions readOptions,
	final DataOutputView outputView) throws IOException {

	int kvStateId = 0;

	for (MetaData metaDataEntry : metaData) {
		RocksIteratorWrapper rocksIteratorWrapper = getRocksIterator(
			db, metaDataEntry.rocksDbKvStateInfo.columnFamilyHandle, metaDataEntry.stateSnapshotTransformer, readOptions);
		kvStateIterators.add(Tuple2.of(rocksIteratorWrapper, kvStateId));
		++kvStateId;
	}

	KeyedBackendSerializationProxy<K> serializationProxy =
		new KeyedBackendSerializationProxy<>(
			// TODO: this code assumes that writing a serializer is threadsafe, we should support to
			// get a serialized form already at state registration time in the future
			keySerializer,
			stateMetaInfoSnapshots,
			!Objects.equals(
				UncompressedStreamCompressionDecorator.INSTANCE,
				keyGroupCompressionDecorator));

	serializationProxy.write(outputView);
}
 

private void writeSnapshotToOutputStream(
	@Nonnull CheckpointStreamWithResultProvider checkpointStreamWithResultProvider,
	@Nonnull KeyGroupRangeOffsets keyGroupRangeOffsets) throws IOException, InterruptedException {

	final List<Tuple2<RocksIteratorWrapper, Integer>> kvStateIterators =
		new ArrayList<>(metaData.size());
	final DataOutputView outputView =
		new DataOutputViewStreamWrapper(checkpointStreamWithResultProvider.getCheckpointOutputStream());
	final ReadOptions readOptions = new ReadOptions();
	try {
		readOptions.setSnapshot(snapshot);
		writeKVStateMetaData(kvStateIterators, readOptions, outputView);
		writeKVStateData(kvStateIterators, checkpointStreamWithResultProvider, keyGroupRangeOffsets);
	} finally {

		for (Tuple2<RocksIteratorWrapper, Integer> kvStateIterator : kvStateIterators) {
			IOUtils.closeQuietly(kvStateIterator.f0);
		}

		IOUtils.closeQuietly(readOptions);
	}
}
 

RocksDBCachingPriorityQueueSet(
	@Nonnegative int keyGroupId,
	@Nonnegative int keyGroupPrefixBytes,
	@Nonnull RocksDB db,
	@Nonnull ReadOptions readOptions,
	@Nonnull ColumnFamilyHandle columnFamilyHandle,
	@Nonnull TypeSerializer<E> byteOrderProducingSerializer,
	@Nonnull DataOutputSerializer outputStream,
	@Nonnull DataInputDeserializer inputStream,
	@Nonnull RocksDBWriteBatchWrapper batchWrapper,
	@Nonnull OrderedByteArraySetCache orderedByteArraySetCache) {
	this.db = db;
	this.readOptions = readOptions;
	this.columnFamilyHandle = columnFamilyHandle;
	this.byteOrderProducingSerializer = byteOrderProducingSerializer;
	this.batchWrapper = batchWrapper;
	this.outputView = outputStream;
	this.inputView = inputStream;
	this.orderedCache = orderedByteArraySetCache;
	this.allElementsInCache = false;
	this.groupPrefixBytes = createKeyGroupBytes(keyGroupId, keyGroupPrefixBytes);
	this.seekHint = groupPrefixBytes;
	this.internalIndex = HeapPriorityQueueElement.NOT_CONTAINED;
}
 

RocksDBPriorityQueueSetFactory(
	KeyGroupRange keyGroupRange,
	int keyGroupPrefixBytes,
	int numberOfKeyGroups,
	Map<String, RocksDBKeyedStateBackend.RocksDbKvStateInfo> kvStateInformation,
	RocksDB db,
	ReadOptions readOptions,
	RocksDBWriteBatchWrapper writeBatchWrapper,
	RocksDBNativeMetricMonitor nativeMetricMonitor,
	Function<String, ColumnFamilyOptions> columnFamilyOptionsFactory) {
	this.keyGroupRange = keyGroupRange;
	this.keyGroupPrefixBytes = keyGroupPrefixBytes;
	this.numberOfKeyGroups = numberOfKeyGroups;
	this.kvStateInformation = kvStateInformation;
	this.db = db;
	this.readOptions = readOptions;
	this.writeBatchWrapper = writeBatchWrapper;
	this.nativeMetricMonitor = nativeMetricMonitor;
	this.columnFamilyOptionsFactory = columnFamilyOptionsFactory;
	this.sharedElementOutView = new DataOutputSerializer(128);
	this.sharedElementInView = new DataInputDeserializer();
}
 

@Override
protected void before() throws Throwable {
	this.temporaryFolder = new TemporaryFolder();
	this.temporaryFolder.create();
	final File rocksFolder = temporaryFolder.newFolder();
	this.dbOptions = optionsFactory.createDBOptions(PredefinedOptions.DEFAULT.createDBOptions()).
		setCreateIfMissing(true);
	this.columnFamilyOptions = optionsFactory.createColumnOptions(PredefinedOptions.DEFAULT.createColumnOptions());
	this.writeOptions = new WriteOptions();
	this.writeOptions.disableWAL();
	this.readOptions = new ReadOptions();
	this.columnFamilyHandles = new ArrayList<>(1);
	this.rocksDB = RocksDB.open(
		dbOptions,
		rocksFolder.getAbsolutePath(),
		Collections.singletonList(new ColumnFamilyDescriptor("default".getBytes(), columnFamilyOptions)),
		columnFamilyHandles);
	this.batchWrapper = new RocksDBWriteBatchWrapper(rocksDB, writeOptions);
}
 

private void writeKVStateMetaData(
	final List<Tuple2<RocksIteratorWrapper, Integer>> kvStateIterators,
	final ReadOptions readOptions,
	final DataOutputView outputView) throws IOException {

	int kvStateId = 0;

	for (MetaData metaDataEntry : metaData) {
		RocksIteratorWrapper rocksIteratorWrapper = getRocksIterator(
			db, metaDataEntry.rocksDbKvStateInfo.columnFamilyHandle, metaDataEntry.stateSnapshotTransformer, readOptions);
		kvStateIterators.add(Tuple2.of(rocksIteratorWrapper, kvStateId));
		++kvStateId;
	}

	KeyedBackendSerializationProxy<K> serializationProxy =
		new KeyedBackendSerializationProxy<>(
			// TODO: this code assumes that writing a serializer is threadsafe, we should support to
			// get a serialized form already at state registration time in the future
			keySerializer,
			stateMetaInfoSnapshots,
			!Objects.equals(
				UncompressedStreamCompressionDecorator.INSTANCE,
				keyGroupCompressionDecorator));

	serializationProxy.write(outputView);
}
 

private void writeSnapshotToOutputStream(
	@Nonnull CheckpointStreamWithResultProvider checkpointStreamWithResultProvider,
	@Nonnull KeyGroupRangeOffsets keyGroupRangeOffsets) throws IOException, InterruptedException {

	final List<Tuple2<RocksIteratorWrapper, Integer>> kvStateIterators =
		new ArrayList<>(metaData.size());
	final DataOutputView outputView =
		new DataOutputViewStreamWrapper(checkpointStreamWithResultProvider.getCheckpointOutputStream());
	final ReadOptions readOptions = new ReadOptions();
	try {
		readOptions.setSnapshot(snapshot);
		writeKVStateMetaData(kvStateIterators, readOptions, outputView);
		writeKVStateData(kvStateIterators, checkpointStreamWithResultProvider, keyGroupRangeOffsets);
	} finally {

		for (Tuple2<RocksIteratorWrapper, Integer> kvStateIterator : kvStateIterators) {
			IOUtils.closeQuietly(kvStateIterator.f0);
		}

		IOUtils.closeQuietly(readOptions);
	}
}
 

private void writeKVStateMetaData(
	final List<Tuple2<RocksIteratorWrapper, Integer>> kvStateIterators,
	final ReadOptions readOptions,
	final DataOutputView outputView) throws IOException {

	int kvStateId = 0;

	for (MetaData metaDataEntry : metaData) {
		RocksIteratorWrapper rocksIteratorWrapper = getRocksIterator(
			db, metaDataEntry.rocksDbKvStateInfo.columnFamilyHandle, metaDataEntry.stateSnapshotTransformer, readOptions);
		kvStateIterators.add(Tuple2.of(rocksIteratorWrapper, kvStateId));
		++kvStateId;
	}

	KeyedBackendSerializationProxy<K> serializationProxy =
		new KeyedBackendSerializationProxy<>(
			// TODO: this code assumes that writing a serializer is threadsafe, we should support to
			// get a serialized form already at state registration time in the future
			keySerializer,
			stateMetaInfoSnapshots,
			!Objects.equals(
				UncompressedStreamCompressionDecorator.INSTANCE,
				keyGroupCompressionDecorator));

	serializationProxy.write(outputView);
}
 

@Override
protected void before() throws Throwable {
	this.temporaryFolder = new TemporaryFolder();
	this.temporaryFolder.create();
	final File rocksFolder = temporaryFolder.newFolder();
	this.dbOptions = optionsFactory.createDBOptions(PredefinedOptions.DEFAULT.createDBOptions()).
		setCreateIfMissing(true);
	this.columnFamilyOptions = optionsFactory.createColumnOptions(PredefinedOptions.DEFAULT.createColumnOptions());
	this.writeOptions = new WriteOptions();
	this.writeOptions.disableWAL();
	this.readOptions = new ReadOptions();
	this.columnFamilyHandles = new ArrayList<>(1);
	this.rocksDB = RocksDB.open(
		dbOptions,
		rocksFolder.getAbsolutePath(),
		Collections.singletonList(new ColumnFamilyDescriptor("default".getBytes(), columnFamilyOptions)),
		columnFamilyHandles);
	this.batchWrapper = new RocksDBWriteBatchWrapper(rocksDB, writeOptions);
}
 

@Override
public long getApproximateKeysInRange(final byte[] startKey, final byte[] endKey) {
    // TODO This is a sad code, the performance is too damn bad
    final Timer.Context timeCtx = getTimeContext("APPROXIMATE_KEYS");
    final Lock readLock = this.readWriteLock.readLock();
    readLock.lock();
    final Snapshot snapshot = this.db.getSnapshot();
    try (final ReadOptions readOptions = new ReadOptions()) {
        readOptions.setSnapshot(snapshot);
        try (final RocksIterator it = this.db.newIterator(readOptions)) {
            if (startKey == null) {
                it.seekToFirst();
            } else {
                it.seek(startKey);
            }
            long approximateKeys = 0;
            for (;;) {
                // The accuracy is 100, don't ask more
                for (int i = 0; i < 100; i++) {
                    if (!it.isValid()) {
                        return approximateKeys;
                    }
                    it.next();
                    ++approximateKeys;
                }
                if (endKey != null && BytesUtil.compare(it.key(), endKey) >= 0) {
                    return approximateKeys;
                }
            }
        }
    } finally {
        // Nothing to release, rocksDB never own the pointer for a snapshot.
        snapshot.close();
        // The pointer to the snapshot is released by the database instance.
        this.db.releaseSnapshot(snapshot);
        readLock.unlock();
        timeCtx.stop();
    }
}
 

@SuppressWarnings("unchecked")
private static RocksIteratorWrapper getRocksIterator(
	RocksDB db,
	ColumnFamilyHandle columnFamilyHandle,
	StateSnapshotTransformer<byte[]> stateSnapshotTransformer,
	ReadOptions readOptions) {
	RocksIterator rocksIterator = db.newIterator(columnFamilyHandle, readOptions);
	return stateSnapshotTransformer == null ?
		new RocksIteratorWrapper(rocksIterator) :
		new RocksTransformingIteratorWrapper(rocksIterator, stateSnapshotTransformer);
}
 

@SuppressWarnings("unchecked")
private static RocksIteratorWrapper getRocksIterator(
	RocksDB db,
	ColumnFamilyHandle columnFamilyHandle,
	StateSnapshotTransformer<byte[]> stateSnapshotTransformer,
	ReadOptions readOptions) {
	RocksIterator rocksIterator = db.newIterator(columnFamilyHandle, readOptions);
	return stateSnapshotTransformer == null ?
		new RocksIteratorWrapper(rocksIterator) :
		new RocksTransformingIteratorWrapper(rocksIterator, stateSnapshotTransformer);
}
 

@Test
public void testFreeWriteReadOptionsAfterClose() throws Exception {
	RocksDBResourceContainer container = new RocksDBResourceContainer();
	WriteOptions writeOptions = container.getWriteOptions();
	ReadOptions readOptions = container.getReadOptions();
	assertThat(writeOptions.isOwningHandle(), is(true));
	assertThat(readOptions.isOwningHandle(), is(true));
	container.close();
	assertThat(writeOptions.isOwningHandle(), is(false));
	assertThat(readOptions.isOwningHandle(), is(false));
}
 

/**
 * Gets the RocksDB {@link ReadOptions} to be used for read operations.
 */
public ReadOptions getReadOptions() {
	// We ensure total order seek by default to prevent user misuse, see FLINK-17800 for more details
	ReadOptions opt = RocksDBOperationUtils.createTotalOrderSeekReadOptions();
	handlesToClose.add(opt);

	// add user-defined options factory, if specified
	if (optionsFactory != null) {
		opt = optionsFactory.createReadOptions(opt, handlesToClose);
	}

	return opt;
}
 

public CloseableIterator<GeoWaveRow> iterator(final ByteArrayRange range) {
  final RocksDB readDb = getReadDb();
  if (readDb == null) {
    return new CloseableIterator.Empty<>();
  }
  final ReadOptions options;
  final RocksIterator it;
  if (range.getEnd() == null) {
    options = null;
    it = readDb.newIterator();
  } else {
    options = new ReadOptions().setIterateUpperBound(new Slice(range.getEndAsNextPrefix()));
    it = readDb.newIterator(options);
  }
  if (range.getStart() == null) {
    it.seekToFirst();
  } else {
    it.seek(range.getStart());
  }

  return new RocksDBRowIterator(
      options,
      it,
      adapterId,
      partition,
      requiresTimestamp,
      visibilityEnabled);
}
 

public DataIndexRowIterator(
    final ReadOptions options,
    final RocksIterator it,
    final short adapterId,
    final boolean visiblityEnabled) {
  super(options, it);
  this.adapterId = adapterId;
  visibilityEnabled = visiblityEnabled;
}
 

public RocksDBRowIterator(
    final ReadOptions options,
    final RocksIterator it,
    final short adapterId,
    final byte[] partition,
    final boolean containsTimestamp,
    final boolean visiblityEnabled) {
  super(options, it);
  this.adapterId = adapterId;
  this.partition = partition;
  this.containsTimestamp = containsTimestamp;
  visibilityEnabled = visiblityEnabled;
}
 

/**
 * Delete the record falls into [beginKeyBytes, endKeyBytes) of the db.
 *
 * @param db the target need to be clipped.
 * @param columnFamilyHandles the column family need to be clipped.
 * @param beginKeyBytes the begin key bytes
 * @param endKeyBytes the end key bytes
 */
private static void deleteRange(
	RocksDB db,
	List<ColumnFamilyHandle> columnFamilyHandles,
	byte[] beginKeyBytes,
	byte[] endKeyBytes,
	@Nonnegative long writeBatchSize) throws RocksDBException {

	for (ColumnFamilyHandle columnFamilyHandle : columnFamilyHandles) {
		try (ReadOptions readOptions = RocksDBOperationUtils.createTotalOrderSeekReadOptions();
			RocksIteratorWrapper iteratorWrapper = RocksDBOperationUtils.getRocksIterator(db, columnFamilyHandle, readOptions);
			RocksDBWriteBatchWrapper writeBatchWrapper = new RocksDBWriteBatchWrapper(db, writeBatchSize)) {

			iteratorWrapper.seek(beginKeyBytes);

			while (iteratorWrapper.isValid()) {
				final byte[] currentKey = iteratorWrapper.key();
				if (beforeThePrefixBytes(currentKey, endKeyBytes)) {
					writeBatchWrapper.remove(columnFamilyHandle, currentKey);
				} else {
					break;
				}
				iteratorWrapper.next();
			}
		}
	}
}
 

/**
 * @implNote Building two wrappers for the same {@link RocksIterator} will lead to
 *     inconsistent behavior.
 */
RocksDBIteratorWrapper(final ReadOptions readOptions, final RocksIterator iterator) {
    this.readOptions = readOptions;
    this.iterator = iterator;
    iterator.seekToFirst();
    closed = false;
}
 

@Override
public Iterator<byte[]> keys() {
    check();

    try {
        ReadOptions readOptions = new ReadOptions();
        readOptions.setSnapshot(db.getSnapshot());
        return new RocksDBIteratorWrapper(readOptions, db.newIterator(readOptions));
    } catch (Exception e) {
        LOG.error("Unable to extract keys from database " + this.toString() + ".", e);
    }

    // empty when retrieval failed
    return Collections.emptyIterator();
}
 

public RocksDBMetadataIterator(
    final ReadOptions options,
    final RocksIterator it,
    final boolean containsTimestamp,
    final boolean visibilityEnabled) {
  super(options, it);
  this.options = options;
  this.it = it;
  this.containsTimestamp = containsTimestamp;
  this.visibilityEnabled = visibilityEnabled;
}
 

@SuppressWarnings("unchecked")
private static RocksIteratorWrapper getRocksIterator(
	RocksDB db,
	ColumnFamilyHandle columnFamilyHandle,
	StateSnapshotTransformer<byte[]> stateSnapshotTransformer,
	ReadOptions readOptions) {
	RocksIterator rocksIterator = db.newIterator(columnFamilyHandle, readOptions);
	return stateSnapshotTransformer == null ?
		new RocksIteratorWrapper(rocksIterator) :
		new RocksTransformingIteratorWrapper(rocksIterator, stateSnapshotTransformer);
}
 

@Override
public boolean init(final LogStorageOptions opts) {
    Requires.requireNonNull(opts.getConfigurationManager(), "Null conf manager");
    Requires.requireNonNull(opts.getLogEntryCodecFactory(), "Null log entry codec factory");
    this.writeLock.lock();
    try {
        if (this.db != null) {
            LOG.warn("RocksDBLogStorage init() already.");
            return true;
        }
        this.logEntryDecoder = opts.getLogEntryCodecFactory().decoder();
        this.logEntryEncoder = opts.getLogEntryCodecFactory().encoder();
        Requires.requireNonNull(this.logEntryDecoder, "Null log entry decoder");
        Requires.requireNonNull(this.logEntryEncoder, "Null log entry encoder");
        this.dbOptions = createDBOptions();
        if (this.openStatistics) {
            this.statistics = new DebugStatistics();
            this.dbOptions.setStatistics(this.statistics);
        }

        this.writeOptions = new WriteOptions();
        this.writeOptions.setSync(this.sync);
        this.totalOrderReadOptions = new ReadOptions();
        this.totalOrderReadOptions.setTotalOrderSeek(true);

        return initAndLoad(opts.getConfigurationManager());
    } catch (final RocksDBException e) {
        LOG.error("Fail to init RocksDBLogStorage, path={}.", this.path, e);
        return false;
    } finally {
        this.writeLock.unlock();
    }

}
 

private CloseableIterator<GeoWaveMetadata> prefixIterator(final byte[] prefix) {
  final ReadOptions options = new ReadOptions().setPrefixSameAsStart(true);
  final RocksIterator it = db.newIterator(options);
  it.seek(prefix);
  return new RocksDBMetadataIterator(options, it, requiresTimestamp, visibilityEnabled);
}
 

public AbstractRocksDBIterator(final ReadOptions options, final RocksIterator it) {
  super();
  this.options = options;
  this.it = it;
}
 

@Override
public TableIterator<byte[], ByteArrayKeyValue> iterator() {
  ReadOptions readOptions = new ReadOptions();
  readOptions.setFillCache(false);
  return new RDBStoreIterator(db.newIterator(handle, readOptions));
}
 

public ReadOptions getReadOptions() {
	return readOptions;
}