com.indeed.util.core.io.Closeables2 Java Examples

public BlockCompressedStableGeneration(
        BloomFilter.MemoryManager memoryManager, File file, Comparator<K> comparator, Serializer<K> keySerializer, Serializer<V> valueSerializer, final CompressionCodec codec, final boolean mlockBTree
) throws IOException {
    this.file = file;
    reader = new ImmutableBTreeIndex.Reader(file, comparator, keySerializer, new LongSerializer(), mlockBTree);
    final File valuesFile = new File(file, "values.bin");
    recordFile =
            new BlockCompressedRecordFile.Builder(valuesFile, valueSerializer, codec).setMlockFiles(mlockBTree).build();
    final File bloomFilterFile = new File(file, "bloomfilter.bin");
    if (bloomFilterFile.exists()) {
        bloomFilter = new BloomFilter.Reader(memoryManager, bloomFilterFile, keySerializer);
    } else {
        bloomFilter = null;
    }
    sizeInBytes = reader.sizeInBytes()+valuesFile.length()+(bloomFilter == null ? 0 : bloomFilter.sizeInBytes());
    stuffToClose = SharedReference.create((Closeable)new Closeable() {
        public void close() throws IOException {
            Closeables2.closeQuietly(reader, log);
            if (bloomFilter != null) Closeables2.closeQuietly(bloomFilter, log);
            Closeables2.closeQuietly(recordFile, log);
        }
    });
}
 

public void checkpoint(File checkpointDir) throws IOException {
    final SharedReference<GenerationState<K, V>> localState = generationState.getCopy();
    try {
        if (localState == null) {
            throw new IOException("store is closed");
        }
        checkpointDir.mkdirs();
        localState.get().volatileGeneration.checkpoint(checkpointDir);
        for (Generation<K, V> generation : localState.get().stableGenerations) {
            generation.checkpoint(checkpointDir);
        }
        PosixFileOperations.cplr(new File(localState.get().path, "state"), checkpointDir);
    } finally {
        Closeables2.closeQuietly(localState, log);
    }
}
 

@Override
public final void close() {
    synchronized (session) {
        if (intTermDocIterator != null) {
            Closeables2.closeQuietly(intTermDocIterator, ImhotepLocalSession.log);
            intTermDocIterator = null;
        }
        if (stringTermDocIterator != null) {
            Closeables2.closeQuietly(stringTermDocIterator, ImhotepLocalSession.log);
            stringTermDocIterator = null;
        }
        if (flamdexReader != null) {
            Closeables2.closeQuietly(flamdexReader, ImhotepLocalSession.log);
            flamdexReader = null;
        }
    }
}
 

@Override
public final void close() {
    synchronized (session) {
        if (docIdStream != null) {
            Closeables2.closeQuietly(docIdStream, ImhotepLocalSession.log);
        }
        if (intTermIterator != null) {
            Closeables2.closeQuietly(intTermIterator, ImhotepLocalSession.log);
            intTermIterator = null;
        }
        if (stringTermIterator != null) {
            Closeables2.closeQuietly(stringTermIterator, ImhotepLocalSession.log);
            stringTermIterator = null;
        }
        if (flamdexReader != null) {
            Closeables2.closeQuietly(flamdexReader, ImhotepLocalSession.log);
            flamdexReader = null;
        }
    }
}
 

/**
 * Flushes volatile generation to disk.
 *
 * @throws IOException  if an I/O error occurs
 */
public void sync() throws IOException {
    final SharedReference<GenerationState<K, V>> localState = generationState.getCopy();
    try {
        if (localState == null) {
            throw new IOException("store is closed");
        }
        try {
            localState.get().volatileGeneration.sync();
        } catch (IOException e) {
            compactor.compact();
            throw e;
        }
    } finally {
        Closeables2.closeQuietly(localState, log);
    }
}
 

public RawFTGSIterator mergeFTGSSplit(final String[] intFields, final String[] stringFields, final String sessionId, final InetSocketAddress[] nodes, final int splitIndex) {
    final RawFTGSIterator[] splits = new RawFTGSIterator[nodes.length];
    try {
        execute(splits, nodes, new ThrowingFunction<InetSocketAddress, RawFTGSIterator>() {
            public RawFTGSIterator apply(final InetSocketAddress node) throws Exception {
                final ImhotepRemoteSession remoteSession = new ImhotepRemoteSession(node.getHostName(), node.getPort(), sessionId, tempFileSizeBytesLeft);
                remoteSession.setNumStats(numStats);
                return remoteSession.getFTGSIteratorSplit(intFields, stringFields, splitIndex, nodes.length);
            }
        });
    } catch (Throwable t) {
        Closeables2.closeAll(log, splits);
        throw Throwables.propagate(t);
    }
    return mergeFTGSSplits(splits);
}
 

@Override
public E get(long address) throws IOException {
    final int segmentNum = (int)(address>>> segmentShift);
    final Option<SharedReference<BlockCompressedRecordFile<E>>> option = fileCache.get(segmentNum);
    if (option.isNone()) {
        throw new IOException("address is invalid: "+address);
    }
    final SharedReference<BlockCompressedRecordFile<E>> reference = option.some();
    final E ret;
    try {
        ret = reference.get().get(address & segmentMask);
    } finally {
        Closeables2.closeQuietly(reference, log);
    }
    return ret;
}
 

public BlockCompressedRecordFile(final Supplier<? extends Either<IOException, ? extends RandomAccessDataInput>> inputSupplier, final Closeable closeable, String file, Serializer<E> serializer, CompressionCodec codec, BlockingQueue<Decompressor> decompressorPool, int blockSize, int recordIndexBits, int padBits, int maxChunkSize) throws IOException {
    this.inputSupplier = inputSupplier;
    this.file = file;
    this.serializer = serializer;
    this.codec = codec;
    this.blockSize = blockSize;
    this.padBits = padBits;
    this.maxChunkSize = maxChunkSize;
    pad = 1<<padBits;
    padMask = ~(long)(pad-1);
    shift = Math.max(recordIndexBits - padBits, 0);
    mask = (1L<<recordIndexBits)-1;
    closeableRef = SharedReference.create(closeable);
    try {
        blockCache = new BlockCache(decompressorPool);
    } catch (Throwable t) {
        Closeables2.closeQuietly(closeableRef, log);
        Throwables.propagateIfInstanceOf(t, IOException.class);
        throw Throwables.propagate(t);
    }
}
 

@Override
public RawFTGSIterator mergeSubsetFTGSSplit(final Map<String, long[]> intFields, final Map<String, String[]> stringFields, final String sessionId, final InetSocketAddress[] nodes, final int splitIndex) {
    final RawFTGSIterator[] splits = new RawFTGSIterator[nodes.length];
    try {
        execute(splits, nodes, new ThrowingFunction<InetSocketAddress, RawFTGSIterator>() {
            public RawFTGSIterator apply(final InetSocketAddress node) throws Exception {
                final ImhotepRemoteSession remoteSession = new ImhotepRemoteSession(node.getHostName(), node.getPort(), sessionId, tempFileSizeBytesLeft);
                remoteSession.setNumStats(numStats);
                return remoteSession.getSubsetFTGSIteratorSplit(intFields, stringFields, splitIndex, nodes.length);
            }
        });
    } catch (Throwable t) {
        Closeables2.closeAll(log, splits);
        throw Throwables.propagate(t);
    }
    return mergeFTGSSplits(splits);
}
 

public void compact() throws IOException {
    lock.lock();
    try {
        if (!closed) {
            final SharedReference<GenerationState<K, V>> localStateReference = generationState.getCopy();
            try {
                if (localStateReference == null) return;
                final GenerationState<K, V> localState = localStateReference.get();
                //this is double checked locking but in this case it doesn't really matter since it's just a heuristic
                if (localState.volatileGeneration.sizeInBytes() > maxVolatileGenerationSize) {
                    final GenerationState<K, V> nextState = startNewLog(localState);
                    startCompaction(nextState);
                }
            } finally {
                Closeables2.closeQuietly(localStateReference, log);
            }
        }
    } finally {
        lock.unlock();
    }
}
 

@Override
public void close() {
    try {
        if (readLockRef == null) {
            Closeables2.closeAll(log, metricCache, wrapped);
        } else {
            Closeables2.closeAll(log, metricCache, wrapped, readLockRef);                
        }
    } finally {
        if (memory == null) {
            return;
        }
        if (memory.usedMemory() > 0) {
            log.error("CachedFlamdexReader is leaking! memory reserved after all memory has been freed: "+memory.usedMemory());
        }
        Closeables2.closeQuietly(memory, log);
    }
}
 

private GenerationState<K, V> startNewLog(final GenerationState<K, V> localState) throws IOException {
    //create new volatile generation and checkpoint
    final File newLog = getNextLogFile();
    final VolatileGeneration<K,V> nextVolatileGeneration = new VolatileGeneration<K, V>(newLog, keySerializer, valueSerializer, comparator);
    final List<SharedReference<? extends Generation<K,V>>> nextStableGenerations = Lists.newArrayList();
    nextStableGenerations.add(localState.volatileGenerationReference.copy());
    for (SharedReference<? extends Generation<K, V>> reference : localState.stableGenerationReferences) {
        nextStableGenerations.add(reference.copy());
    }
    final File checkpointDir = getNextCheckpointDir();
    checkpointDir.mkdirs();
    final GenerationState<K,V> nextState = new GenerationState<K, V>(nextStableGenerations, SharedReference.create(nextVolatileGeneration), checkpointDir);
    checkpointGenerationState(nextState, checkpointDir);
    //there will be a brief period of time where there is no writable generation, put and delete will block during this time
    localState.volatileGeneration.closeWriter();
    PosixFileOperations.atomicLink(checkpointDir, new File(root, "latest"));
    final SharedReference<GenerationState<K, V>> oldState = Preconditions.checkNotNull(generationState.getAndSet(nextState));
    oldState.get().delete();
    Closeables2.closeQuietly(oldState, log);
    return nextState;
}
 

@Override
public void close() {
    synchronized (loadedMetrics) {
        if (!closed) {
            closed = true;
            try {
                Closeables2.closeAll(log, Collections2.transform(loadedMetrics.entrySet(), new Function<Map.Entry<String, IntValueLookup>, Closeable>() {
                    public Closeable apply(final Map.Entry<String, IntValueLookup> metric) {
                        return closeMetric.asCloseable(metric);
                    }
                }));
            } finally {
                loadedMetrics.clear();
            }
        }
    }
}
 

private boolean getSegmentReader(int segmentNum) throws IOException {
    Closeables2.closeQuietly(currentReader, log);
    final Option<SharedReference<BlockCompressedRecordFile<E>>> option = fileCache.get(segmentNum);
    for (SharedReference<BlockCompressedRecordFile<E>> reference : option) {
        try {
            currentReader = reference.get().reader();
        } finally {
            Closeables2.closeQuietly(reference, log);
        }
        return true;
    }
    return false;
}
 

/**
 * writing to or reading from referenced pages after this call will result in undefined behaviour
 * @throws IOException in exceptional circumstances
 */
@Override
public void close() throws IOException {
    try {
        sync();
    } finally {
        synchronized (addressSpaces) {
            addressSpaces.remove(this);
        }
        for (PageTableEntry pte : pageTable) {
            final Memory freePage;
            synchronized (pte) {
                if (pte.memory != null) {
                    final long address = (long)pte.index << PAGE_BITS;
                    if (log.isDebugEnabled()) log.debug("evicting page in file "+file.getPath()+" at address "+ address);
                    if (pte.dirty) {
                        log.error("page in file "+file.getPath()+" at address "+address+" is dirty");
                    }
                    if (pte.refCount > 0) {
                        log.error("page in file "+file.getPath()+" at address "+address+" is still in use, refcount: "+pte.refCount);
                    }
                }
                freePage = pte.memory;
                pte.memory = null;
            }
            if (freePage != null) {
                synchronized (freePages) {
                    freePages.add(freePage);
                }
            }
        }
        Closeables2.closeQuietly(raf, log);
    }
}
 

/**
 * {@inheritDoc}
 */
@Override
public void close() throws IOException {
    if (! closed) {
        Closeables2.closeQuietly(fileChannel, LOG);
        deleteIfExistsQuietly(tempFile);

        closed = true;
    }
}
 

/**
 * Just like mapWithCopy() except that this silently swallows any exception
 * that calling close() on the copy might throw.
 *
 * @param function The function to apply to the value of the local reference.
 * @param <Z> The type of object produced by the function.
 * @return The value that was produced by the supplied {@code function}.
 */
public @Nullable <Z> Z mapWithCopyQuietly(Function<T, Z> function) {
    final @Nullable SharedReference<T> localRef = getCopy();
    try {
        if (localRef == null) {
            return function.apply(null);
        } else {
            return function.apply(localRef.get());
        }
    } finally {
        if (localRef != null) Closeables2.closeQuietly(localRef, log);
    }
}
 

@Override
public synchronized void close() {
    if (!done) {
        done = true;
        Closeables2.closeAll(log, Closeables2.forArray(log, iterators), doneCallback);
    }
}
 

public Reader(long address) throws IOException {
    currentSegmentNum = (int)(address>>> segmentShift);
    final Option<SharedReference<BlockCompressedRecordFile<E>>> recordFile = fileCache.get(currentSegmentNum);
    if (recordFile.isNone()) {
        throw new IOException("address is invalid: "+address);
    }
    final SharedReference<BlockCompressedRecordFile<E>> reference = recordFile.some();
    try {
        currentReader = reference.get().reader(address & segmentMask);
    } finally {
        Closeables2.closeQuietly(reference, log);
    }
}
 

public Option<RecordFile.Reader<E>> getFileReader(final long segmentNum) throws IOException {
    final Option<SharedReference<BlockCompressedRecordFile<E>>> option = fileCache.get((int) segmentNum);
    if (option.isNone()) return Option.none();
    final SharedReference<BlockCompressedRecordFile<E>> reference = option.some();
    final RecordFile.Reader<E> segmentReader;
    try {
        segmentReader = reference.get().reader();
    } finally {
        Closeables2.closeQuietly(reference, log);
    }
    return Option.<RecordFile.Reader<E>>some(new RecordFile.Reader<E>() {

        final long segmentShift = 64-fileIndexBits;
        final long maxSegmentPosition = (1L << segmentShift)-1;

        @Override
        public boolean next() throws IOException {
            return segmentReader.next();
        }

        @Override
        public long getPosition() {
            final long segmentPosition = segmentReader.getPosition();
            if (segmentPosition > maxSegmentPosition) {
                throw new IllegalStateException("position in segment file"+segmentNum+" is too high to be addressable in record log directory with "+fileIndexBits+" file index bits");
            }
            return (segmentNum<<segmentShift)+segmentPosition;
        }

        @Override
        public E get() {
            return segmentReader.get();
        }

        @Override
        public void close() throws IOException {
            Closeables2.closeQuietly(segmentReader, log);
        }
    });
}
 

private void finishClose() throws IOException {
    try {
        final SharedReference<GenerationState<K, V>> state = generationState.getAndUnset();
        try {
            if (state != null) {
                final VolatileGeneration<K, V> volatileGeneration = state.get().volatileGeneration;
                if (volatileGeneration != null) volatileGeneration.closeWriter();
            }
        } finally {
            Closeables2.closeQuietly(state, log);
        }
    } finally {
        threadPool.shutdown();
    }
}
 

/**
 * Close the store, clean up lock files.
 *
 * @throws IOException  if an I/O error occurs
 */
@Override
public void close() throws IOException {
    if (closed) return;
    closed = true;
    if (compactor != null) {
        Closeables2.closeQuietly(compactor, log);
    } else {
        Closeables2.closeQuietly(generationState.getAndUnset(), log);
    }
    if (lockFile != null) {
        lockFile.delete();
    }
}
 

public Reader(Path path, Comparator<K> comparator, Serializer<K> keySerializer, Serializer<V> valueSerializer, final boolean mlockFiles) throws IOException {
    this.comparator = comparator;
    indexFile = path.resolve("index.bin");
    sizeInBytes = Files.size(indexFile);
    buffer = new MMapBuffer(indexFile, FileChannel.MapMode.READ_ONLY, ByteOrder.LITTLE_ENDIAN);
    try {
        stuffToClose = SharedReference.create((Closeable)buffer);
        final MemoryDataInput in = new MemoryDataInput(buffer.memory());
        if (sizeInBytes < Header.length()) {
            throw new IOException("file is less than header length bytes");
        }
        final byte[] headerBytes = new byte[Header.length()];
        in.seek(sizeInBytes - Header.length());
        in.readFully(headerBytes);
        final LittleEndianDataInputStream headerStream = new LittleEndianDataInputStream(new ByteArrayInputStream(headerBytes));
        final Header header = new HeaderSerializer().read(headerStream);
        hasDeletions = header.hasDeletions;
        size = header.size;
        if (header.fileLength != sizeInBytes) {
            log.error(header.fileLength);
            throw new IOException("file length written to last 8 bytes of file does not match file length, file is inconsistent");
        }
        rootLevel = Level.build(buffer.memory(), keySerializer, valueSerializer, comparator, header.hasDeletions, header.indexLevels);
        rootLevelStartAddress = header.rootLevelStartAddress;
        if (mlockFiles) buffer.mlock(0, buffer.memory().length());
    } catch (Throwable t) {
        Closeables2.closeQuietly(buffer, log);
        Throwables.propagateIfInstanceOf(t, IOException.class);
        throw Throwables.propagate(t);
    }
}
 

public synchronized void close() {
    try {
        Closeables2.closeAll(log, circularBuffer.getInputStream(), circularBuffer.getOutputStream(), iterator);
    } finally {
        try {
            executorService.shutdownNow();
            executorService.awaitTermination(Long.MAX_VALUE, TimeUnit.MILLISECONDS);
        } catch (InterruptedException e) {
            throw Throwables.propagate(e);
        } finally {
            Closeables2.closeAll(log, iterators);
        }
    }
}
 

@Override
public void close() {
    if (done.compareAndSet(false, true)) {
        try {
            if (Thread.currentThread() != runThread) {
                while (true) {
                    try {
                        runThread.interrupt();
                        runThread.join(10);
                        if (!runThread.isAlive()) break;
                    } catch (InterruptedException e) {
                        //ignore
                    }
                }
            }
        } finally {
            Closeables2.closeAll(log, iterator, Closeables2.forIterable(log, Iterables.transform(Arrays.asList(files), new Function<File, Closeable>() {
                public Closeable apply(final File input) {
                    return new Closeable() {
                        public void close() throws IOException {
                            input.delete();
                        }
                    };
                }
            })), Closeables2.forArray(log, outputs), Closeables2.forArray(log, ftgsIterators), Closeables2.forArray(log, outputStreams));
        }
    }
}
 

protected void preClose() {
    try {
        if (lastIterator != null) {
            Closeables2.closeQuietly(lastIterator, log);
            lastIterator = null;
        }
    } finally {
        getSplitBufferThreads.shutdown();
        mergeSplitBufferThreads.shutdown();
    }
}
 

@Override
public RawFTGSIterator getSubsetFTGSIteratorSplit(final Map<String, long[]> intFields, final Map<String, String[]> stringFields, final int splitIndex, final int numSplits) {
    final RawFTGSIterator[] splits = new RawFTGSIterator[sessions.length];
    try {
        executeSessions(splits, new ThrowingFunction<ImhotepSession, RawFTGSIterator>() {
            public RawFTGSIterator apply(final ImhotepSession imhotepSession) throws Exception {
                return imhotepSession.getSubsetFTGSIteratorSplit(intFields, stringFields, splitIndex, numSplits);
            }
        });
    } catch (Throwable t) {
        Closeables2.closeAll(log, splits);
        throw Throwables.propagate(t);
    }
    return new RawFTGSMerger(Arrays.asList(splits), numStats, null);
}
 

public RawFTGSIterator getFTGSIteratorSplit(final String[] intFields, final String[] stringFields, final int splitIndex, final int numSplits) {
    final RawFTGSIterator[] splits = new RawFTGSIterator[sessions.length];
    try {
        executeSessions(splits, new ThrowingFunction<ImhotepSession, RawFTGSIterator>() {
            public RawFTGSIterator apply(final ImhotepSession imhotepSession) throws Exception {
                return imhotepSession.getFTGSIteratorSplit(intFields, stringFields, splitIndex, numSplits);
            }
        });
    } catch (Throwable t) {
        Closeables2.closeAll(log, splits);
        throw Throwables.propagate(t);
    }
    return new RawFTGSMerger(Arrays.asList(splits), numStats, null);
}
 

public final DocIterator getDocIterator(String[] intFields, String[] stringFields) throws ImhotepOutOfMemoryException {
    final Closer closer = Closer.create();
    try {
        final List<DocIterator> docIterators = Lists.newArrayList();
        for (ImhotepSession session : sessions) {
            docIterators.add(closer.register(session.getDocIterator(intFields, stringFields)));
        }
        return new DocIteratorMerger(docIterators, intFields.length, stringFields.length);
    } catch (Throwable t) {
        Closeables2.closeQuietly(closer, log);
        throw Throwables2.propagate(t, ImhotepOutOfMemoryException.class);
    }
}
 

@Override
public synchronized void close() {
    if (!done) {
        done = true;
        Closeables2.closeAll(log, Closeables2.forArray(log, iterators), doneCallback);
    }
}