com.indeed.util.core.reference.SharedReference Java Examples

@Override
public Option<SharedReference<BlockCompressedRecordFile<E>>> load(final Integer segmentNum) {
    try {
        final File segmentFile = getSegmentPath(dir, segmentNum, false);
        if (!segmentFile.exists()) {
            return Option.none();
        }
        final long start = System.nanoTime();
        final BlockCompressedRecordFile<E> recordFile =
                new BlockCompressedRecordFile.Builder(segmentFile, serializer, codec)
                        .setDecompressorPool(decompressorPool)
                        .setBlockSize(blockSize)
                        .setRecordIndexBits(recordIndexBits)
                        .setPadBits(padBits)
                        .setMlockFiles(mlockFiles)
                        .build();
        log.debug("segment open time: "+(System.nanoTime()-start)/1000d+" us");
        return Option.some(SharedReference.create(recordFile));
    } catch (IOException e) {
        log.error("error opening file with segment number "+segmentNum, e);
        return Option.none();
    }
}
 

public Shard(final ReloadableSharedReference<CachedFlamdexReader, IOException> ref,
              final SharedReference<ReadLock> readLock,
              final long shardVersion,
              final String indexDir,
              final String dataset,
              final String shardId) throws IOException {
    this.ref = ref;
    this.readLock = readLock;
    this.shardId = new ShardId(dataset, shardId, shardVersion, indexDir);
    final SharedReference<CachedFlamdexReader> copy = ref.copy();
    numDocs = copy.get().getNumDocs();
    intFields = copy.get().getIntFields();
    stringFields = copy.get().getStringFields();
    availableMetrics = copy.get().getAvailableMetrics();
    copy.close();
}
 

protected Session(
        ImhotepSession imhotepSession,
        E sessionState,
        String username,
        String ipAddress,
        int clientVersion,
        String dataset
) {
    this.imhotepSession = SharedReference.create(imhotepSession);
    this.sessionState = sessionState;
    this.username = username;
    this.ipAddress = ipAddress;
    this.clientVersion = clientVersion;
    this.dataset = dataset;

    lastActionTime = System.currentTimeMillis();
}
 

public VolatileGeneration(File logPath, Serializer<K> keySerializer, Serializer<V> valueSerializer, Comparator<K> comparator, boolean loadExistingReadOnly) throws IOException {
    this.ordering = Ordering.from(comparator);
    map = new ConcurrentSkipListMap(comparator);
    this.logPath = logPath;
    this.keySerializer = keySerializer;
    this.valueSerializer = valueSerializer;
    deleted = new Object();
    if (loadExistingReadOnly) {
        if (!logPath.exists()) throw new IllegalArgumentException(logPath.getAbsolutePath()+" does not exist");
        transactionLog = null;
        replayTransactionLog(logPath, true);
    } else {
        if (logPath.exists()) throw new IllegalArgumentException("to load existing logs set loadExistingReadOnly to true or create a new log and use replayTransactionLog");
        transactionLog = new TransactionLog.Writer(logPath, keySerializer, valueSerializer, false);
    }
    stuffToClose = SharedReference.create((Closeable)new Closeable() {
        public void close() throws IOException {
            closeWriter();
        }
    });
}
 

public FilteredGeneration(Generation<K, V> generation, SharedReference<Closeable> closeable, @Nullable K minKey, boolean minInclusive, @Nullable K maxKey, boolean maxInclusive) {
    this.generation = generation;
    this.closeable = closeable;
    this.minKey = minKey;
    this.minInclusive = minInclusive;
    this.maxKey = maxKey;
    this.maxInclusive = maxInclusive;
    this.ordering = Ordering.from(generation.getComparator());
    checkMax = new F<Entry<K, V>, Boolean>() {
        @Override
        public Boolean f(Entry<K, V> kvEntry) {
            final int cmp = ordering.compare(kvEntry.getKey(), FilteredGeneration.this.maxKey);
            return (cmp <= 0) & ((cmp != 0) | FilteredGeneration.this.maxInclusive);
        }
    };
    checkMin = new F<Entry<K, V>, Boolean>() {
        @Override
        public Boolean f(Entry<K, V> kvEntry) {
            final int cmp = ordering.compare(kvEntry.getKey(), FilteredGeneration.this.minKey);
            return (cmp >= 0) & ((cmp != 0) | FilteredGeneration.this.minInclusive);
        }
    };
}
 

public BlockCompressedStableGeneration(
        BloomFilter.MemoryManager memoryManager, File file, Comparator<K> comparator, Serializer<K> keySerializer, Serializer<V> valueSerializer, final CompressionCodec codec, final boolean mlockBTree
) throws IOException {
    this.file = file;
    reader = new ImmutableBTreeIndex.Reader(file, comparator, keySerializer, new LongSerializer(), mlockBTree);
    final File valuesFile = new File(file, "values.bin");
    recordFile =
            new BlockCompressedRecordFile.Builder(valuesFile, valueSerializer, codec).setMlockFiles(mlockBTree).build();
    final File bloomFilterFile = new File(file, "bloomfilter.bin");
    if (bloomFilterFile.exists()) {
        bloomFilter = new BloomFilter.Reader(memoryManager, bloomFilterFile, keySerializer);
    } else {
        bloomFilter = null;
    }
    sizeInBytes = reader.sizeInBytes()+valuesFile.length()+(bloomFilter == null ? 0 : bloomFilter.sizeInBytes());
    stuffToClose = SharedReference.create((Closeable)new Closeable() {
        public void close() throws IOException {
            Closeables2.closeQuietly(reader, log);
            if (bloomFilter != null) Closeables2.closeQuietly(bloomFilter, log);
            Closeables2.closeQuietly(recordFile, log);
        }
    });
}
 

public void checkpoint(File checkpointDir) throws IOException {
    final SharedReference<GenerationState<K, V>> localState = generationState.getCopy();
    try {
        if (localState == null) {
            throw new IOException("store is closed");
        }
        checkpointDir.mkdirs();
        localState.get().volatileGeneration.checkpoint(checkpointDir);
        for (Generation<K, V> generation : localState.get().stableGenerations) {
            generation.checkpoint(checkpointDir);
        }
        PosixFileOperations.cplr(new File(localState.get().path, "state"), checkpointDir);
    } finally {
        Closeables2.closeQuietly(localState, log);
    }
}
 

/**
 * Flushes volatile generation to disk.
 *
 * @throws IOException  if an I/O error occurs
 */
public void sync() throws IOException {
    final SharedReference<GenerationState<K, V>> localState = generationState.getCopy();
    try {
        if (localState == null) {
            throw new IOException("store is closed");
        }
        try {
            localState.get().volatileGeneration.sync();
        } catch (IOException e) {
            compactor.compact();
            throw e;
        }
    } finally {
        Closeables2.closeQuietly(localState, log);
    }
}
 

public void compact() throws IOException {
    lock.lock();
    try {
        if (!closed) {
            final SharedReference<GenerationState<K, V>> localStateReference = generationState.getCopy();
            try {
                if (localStateReference == null) return;
                final GenerationState<K, V> localState = localStateReference.get();
                //this is double checked locking but in this case it doesn't really matter since it's just a heuristic
                if (localState.volatileGeneration.sizeInBytes() > maxVolatileGenerationSize) {
                    final GenerationState<K, V> nextState = startNewLog(localState);
                    startCompaction(nextState);
                }
            } finally {
                Closeables2.closeQuietly(localStateReference, log);
            }
        }
    } finally {
        lock.unlock();
    }
}
 

private GenerationState<K, V> startNewLog(final GenerationState<K, V> localState) throws IOException {
    //create new volatile generation and checkpoint
    final File newLog = getNextLogFile();
    final VolatileGeneration<K,V> nextVolatileGeneration = new VolatileGeneration<K, V>(newLog, keySerializer, valueSerializer, comparator);
    final List<SharedReference<? extends Generation<K,V>>> nextStableGenerations = Lists.newArrayList();
    nextStableGenerations.add(localState.volatileGenerationReference.copy());
    for (SharedReference<? extends Generation<K, V>> reference : localState.stableGenerationReferences) {
        nextStableGenerations.add(reference.copy());
    }
    final File checkpointDir = getNextCheckpointDir();
    checkpointDir.mkdirs();
    final GenerationState<K,V> nextState = new GenerationState<K, V>(nextStableGenerations, SharedReference.create(nextVolatileGeneration), checkpointDir);
    checkpointGenerationState(nextState, checkpointDir);
    //there will be a brief period of time where there is no writable generation, put and delete will block during this time
    localState.volatileGeneration.closeWriter();
    PosixFileOperations.atomicLink(checkpointDir, new File(root, "latest"));
    final SharedReference<GenerationState<K, V>> oldState = Preconditions.checkNotNull(generationState.getAndSet(nextState));
    oldState.get().delete();
    Closeables2.closeQuietly(oldState, log);
    return nextState;
}
 

public BlockCompressedRecordFile(final Supplier<? extends Either<IOException, ? extends RandomAccessDataInput>> inputSupplier, final Closeable closeable, String file, Serializer<E> serializer, CompressionCodec codec, BlockingQueue<Decompressor> decompressorPool, int blockSize, int recordIndexBits, int padBits, int maxChunkSize) throws IOException {
    this.inputSupplier = inputSupplier;
    this.file = file;
    this.serializer = serializer;
    this.codec = codec;
    this.blockSize = blockSize;
    this.padBits = padBits;
    this.maxChunkSize = maxChunkSize;
    pad = 1<<padBits;
    padMask = ~(long)(pad-1);
    shift = Math.max(recordIndexBits - padBits, 0);
    mask = (1L<<recordIndexBits)-1;
    closeableRef = SharedReference.create(closeable);
    try {
        blockCache = new BlockCache(decompressorPool);
    } catch (Throwable t) {
        Closeables2.closeQuietly(closeableRef, log);
        Throwables.propagateIfInstanceOf(t, IOException.class);
        throw Throwables.propagate(t);
    }
}
 

public Reader(SharedReference<Closeable> ref, long seekAddress) throws IOException {
    this.ref = ref;
    initialized = true;
    final long newBlockAddress = (seekAddress>>>shift)&padMask;
    currentBlock = blockCache.get(newBlockAddress).get();
    blockAddress = newBlockAddress;
    if (currentBlock.isNone()) {
        done = true;
        throw new IOException("address "+seekAddress+" is invalid because block does not exist in file "+file);
    }
    final BlockCacheEntry block = currentBlock.some();
    currentRecord = (int)(seekAddress&mask);
    if (currentRecord >= block.size()) {
        done = true;
        throw new IOException("there are only "+block.size()+" in block at address "+newBlockAddress+", seek request is for record number "+currentRecord);
    }
}
 

@Override
public E get(long address) throws IOException {
    final int segmentNum = (int)(address>>> segmentShift);
    final Option<SharedReference<BlockCompressedRecordFile<E>>> option = fileCache.get(segmentNum);
    if (option.isNone()) {
        throw new IOException("address is invalid: "+address);
    }
    final SharedReference<BlockCompressedRecordFile<E>> reference = option.some();
    final E ret;
    try {
        ret = reference.get().get(address & segmentMask);
    } finally {
        Closeables2.closeQuietly(reference, log);
    }
    return ret;
}
 

public FileCache(final boolean mlockFiles) {
    this.mlockFiles = mlockFiles;
    decompressorPool = new LinkedBlockingQueue<Decompressor>();
    readerCache = CacheBuilder.newBuilder().maximumSize(maxCachedFiles)
            .removalListener(new RemovalListener<Integer, Option<SharedReference<BlockCompressedRecordFile<E>>>>() {
                @Override
                public void onRemoval(RemovalNotification<Integer, Option<SharedReference<BlockCompressedRecordFile<E>>>> notification) {
                    final Integer segmentNum = notification.getKey();
                    final Option<SharedReference<BlockCompressedRecordFile<E>>> referenceOption = notification.getValue();
                    for (SharedReference<BlockCompressedRecordFile<E>> reference : referenceOption) {
                        try {
                            reference.close();
                        } catch (IOException e) {
                            log.error("error on block cleanup", e);
                        }
                    }
                }
            })
            .build(open);
}
 

public Reader(long address) throws IOException {
    currentSegmentNum = (int)(address>>> segmentShift);
    final Option<SharedReference<BlockCompressedRecordFile<E>>> recordFile = fileCache.get(currentSegmentNum);
    if (recordFile.isNone()) {
        throw new IOException("address is invalid: "+address);
    }
    final SharedReference<BlockCompressedRecordFile<E>> reference = recordFile.some();
    try {
        currentReader = reference.get().reader(address & segmentMask);
    } finally {
        Closeables2.closeQuietly(reference, log);
    }
}
 

public long getSizeInBytes() {
    try (final SharedReference<TableReader<K, V>> reader = getCopy()) {
        return reader.get().getSizeInBytes();
    } catch (final IOException e) {
        return 0L;
    }
}
 

private List<ShardInfo> buildShardList() throws IOException {
    final Map<String, Map<String, AtomicSharedReference<Shard>>> localShards = shards;
    final List<ShardInfo> ret = new ArrayList<ShardInfo>();
    for (final Map<String, AtomicSharedReference<Shard>> map : localShards.values()) {
        for (final String shardName : map.keySet()) {
            final SharedReference<Shard> ref = map.get(shardName).getCopy();
            try {
                if (ref != null) {
                    final Shard shard = ref.get();
                    ret.add(new ShardInfo(shard.getDataset(), shardName,
                                          shard.getLoadedMetrics(), shard.getNumDocs(),
                                          shard.getShardVersion()));
                }
            } finally {
                Closeables2.closeQuietly(ref, log);
            }
        }
    }
    Collections.sort(ret, new Comparator<ShardInfo>() {
        @Override
        public int compare(ShardInfo o1, ShardInfo o2) {
            final int c = o1.dataset.compareTo(o2.dataset);
            if (c != 0) {
                return c;
            }
            return o1.shardId.compareTo(o2.shardId);
        }
    });
    return ret;
}
 

private List<DatasetInfo> buildDatasetList() throws IOException {
    final Map<String, Map<String, AtomicSharedReference<Shard>>> localShards = shards;
    final List<DatasetInfo> ret = Lists.newArrayList();
    for (final Map.Entry<String, Map<String, AtomicSharedReference<Shard>>> e : localShards.entrySet()) {
        final String dataset = e.getKey();
        final Map<String, AtomicSharedReference<Shard>> map = e.getValue();
        final List<ShardInfo> shardList = Lists.newArrayList();
        final Set<String> intFields = Sets.newHashSet();
        final Set<String> stringFields = Sets.newHashSet();
        final Set<String> metrics = Sets.newHashSet();
        for (final String shardName : map.keySet()) {
            final SharedReference<Shard> ref = map.get(shardName).getCopy();
            try {
                if (ref != null) {
                    final Shard shard = ref.get();
                    shardList.add(new ShardInfo(shard.getDataset(), shardName,
                                                shard.getLoadedMetrics(), shard.getNumDocs(),
                                                shard.getShardVersion()));
                    intFields.addAll(shard.getIntFields());
                    stringFields.addAll(shard.getStringFields());
                    metrics.addAll(shard.getAvailableMetrics());
                }
            } finally {
                Closeables2.closeQuietly(ref, log);
            }
        }
        ret.add(new DatasetInfo(dataset, shardList, intFields, stringFields, metrics));
    }
    return ret;
}
 

public Option<SharedReference<BlockCompressedRecordFile<E>>> get(final Integer key) {
    final Option<SharedReference<BlockCompressedRecordFile<E>>> option = readerCache.getUnchecked(key);
    if (option.isNone()) {
        readerCache.invalidate(key);
        return option;
    } else {
        final SharedReference<BlockCompressedRecordFile<E>> copy = option.some().tryCopy();
        if (copy == null) {
            return get(key);
        }
        return Option.some(copy);
    }
}
 

public Set<String> getLoadedMetrics() {
    final SharedReference<CachedFlamdexReader> copy = ref.copyIfLoaded();
    if (copy != null) {
        try {
            return copy.get().getLoadedMetrics();
        } finally {
            Closeables2.closeQuietly(copy, log);
        }
    }
    return Collections.emptySet();
}
 

public List<ImhotepStatusDump.MetricDump> getMetricDump() {
    final SharedReference<CachedFlamdexReader> copy = ref.copyIfLoaded();
    if (copy != null) {
        try {
            return copy.get().getMetricDump();
        } finally {
            Closeables2.closeQuietly(copy, log);
        }
    }
    return Collections.emptyList();
}
 

public Reader(Path path, Comparator<K> comparator, Serializer<K> keySerializer, Serializer<V> valueSerializer, final boolean mlockFiles) throws IOException {
    this.comparator = comparator;
    indexFile = path.resolve("index.bin");
    sizeInBytes = Files.size(indexFile);
    buffer = new MMapBuffer(indexFile, FileChannel.MapMode.READ_ONLY, ByteOrder.LITTLE_ENDIAN);
    try {
        stuffToClose = SharedReference.create((Closeable)buffer);
        final MemoryDataInput in = new MemoryDataInput(buffer.memory());
        if (sizeInBytes < Header.length()) {
            throw new IOException("file is less than header length bytes");
        }
        final byte[] headerBytes = new byte[Header.length()];
        in.seek(sizeInBytes - Header.length());
        in.readFully(headerBytes);
        final LittleEndianDataInputStream headerStream = new LittleEndianDataInputStream(new ByteArrayInputStream(headerBytes));
        final Header header = new HeaderSerializer().read(headerStream);
        hasDeletions = header.hasDeletions;
        size = header.size;
        if (header.fileLength != sizeInBytes) {
            log.error(header.fileLength);
            throw new IOException("file length written to last 8 bytes of file does not match file length, file is inconsistent");
        }
        rootLevel = Level.build(buffer.memory(), keySerializer, valueSerializer, comparator, header.hasDeletions, header.indexLevels);
        rootLevelStartAddress = header.rootLevelStartAddress;
        if (mlockFiles) buffer.mlock(0, buffer.memory().length());
    } catch (Throwable t) {
        Closeables2.closeQuietly(buffer, log);
        Throwables.propagateIfInstanceOf(t, IOException.class);
        throw Throwables.propagate(t);
    }
}
 

private boolean getSegmentReader(int segmentNum) throws IOException {
    Closeables2.closeQuietly(currentReader, log);
    final Option<SharedReference<BlockCompressedRecordFile<E>>> option = fileCache.get(segmentNum);
    for (SharedReference<BlockCompressedRecordFile<E>> reference : option) {
        try {
            currentReader = reference.get().reader();
        } finally {
            Closeables2.closeQuietly(reference, log);
        }
        return true;
    }
    return false;
}
 

public GenerationState(
        final List<SharedReference<? extends Generation<K, V>>> stableGenerationReferences,
        final SharedReference<VolatileGeneration<K, V>> volatileGenerationReference,
        File path
) {
    this.path = path;
    this.stableGenerationReferences = ImmutableList.copyOf(stableGenerationReferences);
    this.volatileGenerationReference = volatileGenerationReference;
    this.volatileGeneration = volatileGenerationReference.get();
    final ImmutableList.Builder<Generation<K,V>> builder = ImmutableList.builder();
    for (SharedReference<? extends Generation<K, V>> generation : stableGenerationReferences) {
        builder.add(generation.get());
    }
    stableGenerations = builder.build();
}
 

public Option<RecordFile.Reader<E>> getFileReader(final long segmentNum) throws IOException {
    final Option<SharedReference<BlockCompressedRecordFile<E>>> option = fileCache.get((int) segmentNum);
    if (option.isNone()) return Option.none();
    final SharedReference<BlockCompressedRecordFile<E>> reference = option.some();
    final RecordFile.Reader<E> segmentReader;
    try {
        segmentReader = reference.get().reader();
    } finally {
        Closeables2.closeQuietly(reference, log);
    }
    return Option.<RecordFile.Reader<E>>some(new RecordFile.Reader<E>() {

        final long segmentShift = 64-fileIndexBits;
        final long maxSegmentPosition = (1L << segmentShift)-1;

        @Override
        public boolean next() throws IOException {
            return segmentReader.next();
        }

        @Override
        public long getPosition() {
            final long segmentPosition = segmentReader.getPosition();
            if (segmentPosition > maxSegmentPosition) {
                throw new IllegalStateException("position in segment file"+segmentNum+" is too high to be addressable in record log directory with "+fileIndexBits+" file index bits");
            }
            return (segmentNum<<segmentShift)+segmentPosition;
        }

        @Override
        public E get() {
            return segmentReader.get();
        }

        @Override
        public void close() throws IOException {
            Closeables2.closeQuietly(segmentReader, log);
        }
    });
}
 

private <A, B> A doWithState(F2<GenerationState<K,V>, B, A> function, @Nullable B b) throws IOException {
    final SharedReference<GenerationState<K, V>> localState = generationState.getCopy();
    try {
        if (localState == null) {
            throw new IOException("store is closed");
        }
        return function.f(localState.get(), b);
    } catch (RuntimeIOException e) {
        Throwables.propagateIfInstanceOf(e.getCause(), IOException.class);
        log.error("RuntimeIOException inner exception is not IOException", e);
        throw Throwables.propagate(e.getCause());
    } finally {
        Closeables2.closeQuietly(localState, log);
    }
}
 

@Override
public void close() throws IOException {
    Closeables2.closeQuietly(volatileGenerationReference, log);
    for (SharedReference<? extends Generation<K, V>> reference : stableGenerationReferences) {
        Closeables2.closeQuietly(reference, log);
    }
}
 

private void startCompaction(final GenerationState<K, V> localState) throws IOException {
    //find generations eligible for compaction and start compaction in background
    final List<SharedReference<? extends Generation<K,V>>> toCompact = Lists.newArrayList();
    long sum = 0;
    boolean hasDeletions = false;
    for (SharedReference<? extends Generation<K, V>> reference : localState.stableGenerationReferences) {
        final Generation<K, V> generation = reference.get();
        final String name = generation.getPath().getName();
        if (!currentlyCompacting.contains(name)) {
            if ((generation instanceof VolatileGeneration || (sum*2 > generation.sizeInBytes()))) {
                sum+=generation.sizeInBytes();
                toCompact.add(reference.copy());
                currentlyCompacting.add(generation.getPath().getName());
            } else {
                hasDeletions = true;
                break;
            }
        } else {
            hasDeletions = true;
            break;
        }
    }
    if (toCompact.size() > 0) {
        runningCompactions++;
        threadPool.execute(new Compaction(toCompact, hasDeletions));
    }
}
 

private void finishCompaction(final Set<String> compactedGenerations, final List<Generation<K, V>> toCompactGenerations, final Generation<K, V> stableGeneration) throws IOException {
    final List<SharedReference<? extends Generation<K,V>>> nextStableGenerations = Lists.newArrayList();
    final SharedReference<GenerationState<K, V>> stateReference = Preconditions.checkNotNull(generationState.getCopy());
    final GenerationState<K,V> state = stateReference.get();
    try {
        boolean compactionAdded = false;
        for (SharedReference<? extends Generation<K, V>> reference : state.stableGenerationReferences) {
            final String name = reference.get().getPath().getName();
            if (!compactedGenerations.contains(name)) {
                nextStableGenerations.add(reference.copy());
            } else {
                if (!compactionAdded) {
                    nextStableGenerations.add(SharedReference.create(stableGeneration));
                    compactionAdded = true;
                }
                currentlyCompacting.remove(name);
            }
        }
        final File checkpointDir = getNextCheckpointDir();
        checkpointDir.mkdirs();
        final GenerationState<K,V> nextState = new GenerationState<K, V>(nextStableGenerations, state.volatileGenerationReference.copy(), checkpointDir);
        checkpointGenerationState(nextState, checkpointDir);
        PosixFileOperations.atomicLink(checkpointDir, new File(root, "latest"));
        final SharedReference<GenerationState<K, V>> oldState = Preconditions.checkNotNull(generationState.getAndSet(nextState));
        oldState.get().delete();
        Closeables2.closeQuietly(oldState, log);
        for (Generation<K, V> generation : toCompactGenerations) {
            final long sizeInBytes = generation.sizeInBytes();
            generation.delete();
            totalGenerationSpace.addAndGet(-sizeInBytes);
        }
    } finally {
        Closeables2.closeQuietly(stateReference, log);
    }
}
 

private void finishClose() throws IOException {
    try {
        final SharedReference<GenerationState<K, V>> state = generationState.getAndUnset();
        try {
            if (state != null) {
                final VolatileGeneration<K, V> volatileGeneration = state.get().volatileGeneration;
                if (volatileGeneration != null) volatileGeneration.closeWriter();
            }
        } finally {
            Closeables2.closeQuietly(state, log);
        }
    } finally {
        threadPool.shutdown();
    }
}