Java Code Examples for org.apache.cassandra.io.util.FileUtils#closeQuietly()

private IndexSummary buildSummaryAtLevel(int newSamplingLevel) throws IOException
{
    // we read the positions in a BRAF so we don't have to worry about an entry spanning a mmap boundary.
    RandomAccessReader primaryIndex = RandomAccessReader.open(new File(descriptor.filenameFor(Component.PRIMARY_INDEX)));
    try
    {
        long indexSize = primaryIndex.length();
        try (IndexSummaryBuilder summaryBuilder = new IndexSummaryBuilder(estimatedKeys(), metadata.getMinIndexInterval(), newSamplingLevel))
        {
            long indexPosition;
            while ((indexPosition = primaryIndex.getFilePointer()) != indexSize)
            {
                summaryBuilder.maybeAddEntry(partitioner.decorateKey(ByteBufferUtil.readWithShortLength(primaryIndex)), indexPosition);
                RowIndexEntry.Serializer.skip(primaryIndex);
            }

            return summaryBuilder.build(partitioner);
        }
    }
    finally
    {
        FileUtils.closeQuietly(primaryIndex);
    }
}
 

@Test
@Ignore
public void testHugeBFSerialization() throws IOException
{
    ByteBuffer test = ByteBuffer.wrap(new byte[] {0, 1});

    File file = FileUtils.createTempFile("bloomFilterTest-", ".dat");
    BloomFilter filter = (BloomFilter) FilterFactory.getFilter(((long)Integer.MAX_VALUE / 8) + 1, 0.01d, true);
    filter.add(test);
    DataOutputStreamAndChannel out = new DataOutputStreamAndChannel(new FileOutputStream(file));
    FilterFactory.serialize(filter, out);
    filter.bitset.serialize(out);
    out.close();
    filter.close();
    
    DataInputStream in = new DataInputStream(new FileInputStream(file));
    BloomFilter filter2 = (BloomFilter) FilterFactory.deserialize(in, true);
    Assert.assertTrue(filter2.isPresent(test));
    FileUtils.closeQuietly(in);
}
 

/**
 * Appends new component names to the TOC component.
 */
protected static void appendTOC(Descriptor descriptor, Collection<Component> components)
{
    File tocFile = new File(descriptor.filenameFor(Component.TOC));
    PrintWriter w = null;
    try
    {
        w = new PrintWriter(new FileWriter(tocFile, true));
        for (Component component : components)
            w.println(component.name);
    }
    catch (IOException e)
    {
        throw new FSWriteError(e, tocFile);
    }
    finally
    {
        FileUtils.closeQuietly(w);
    }
}
 

private static Properties loadProperties()
{
    Properties properties = new Properties();
    InputStream stream = InvertedIndex.class.getClassLoader().getResourceAsStream("InvertedIndex.properties");
    try
    {
        properties.load(stream);
    }
    catch (Exception e)
    {
        throw new RuntimeException(e);
    }
    finally
    {
        FileUtils.closeQuietly(stream);
    }
    logger.info("loaded property file, InvertedIndex.properties");
    return properties;
}
 

public static String getReleaseVersionString()
{
    InputStream in = null;
    try
    {
        in = FBUtilities.class.getClassLoader().getResourceAsStream("org/apache/cassandra/config/version.properties");
        if (in == null)
        {
            return System.getProperty("cassandra.releaseVersion", "Unknown");
        }
        Properties props = new Properties();
        props.load(in);
        return props.getProperty("CassandraVersion");
    }
    catch (Exception e)
    {
        JVMStabilityInspector.inspectThrowable(e);
        logger.warn("Unable to load version.properties", e);
        return "debug version";
    }
    finally
    {
        FileUtils.closeQuietly(in);
    }
}
 

/**
 * Finds and returns the first key beyond a given token in this SSTable or null if no such key exists.
 */
public DecoratedKey firstKeyBeyond(RowPosition token)
{
    if (token.compareTo(first) < 0)
        return first;

    long sampledPosition = getIndexScanPosition(token);

    Iterator<FileDataInput> segments = ifile.iterator(sampledPosition);
    while (segments.hasNext())
    {
        FileDataInput in = segments.next();
        try
        {
            while (!in.isEOF())
            {
                ByteBuffer indexKey = ByteBufferUtil.readWithShortLength(in);
                DecoratedKey indexDecoratedKey = partitioner.decorateKey(indexKey);
                if (indexDecoratedKey.compareTo(token) > 0)
                    return indexDecoratedKey;

                RowIndexEntry.Serializer.skip(in);
            }
        }
        catch (IOException e)
        {
            markSuspect();
            throw new CorruptSSTableException(e, in.getPath());
        }
        finally
        {
            FileUtils.closeQuietly(in);
        }
    }

    return null;
}
 

public void release()
{
    int n = references.decrementAndGet();
    if (n == 0)
    {
        FileUtils.closeQuietly(index);
        sstable.releaseReference();
        if (obsolete.get() || sstable.isMarkedCompacted())
            FileUtils.delete(index.getIndexPath());
    }
}
 

private void saveSummary(SegmentedFile.Builder ibuilder, SegmentedFile.Builder dbuilder, IndexSummary summary)
{
    File summariesFile = new File(descriptor.filenameFor(Component.SUMMARY));
    if (summariesFile.exists())
        FileUtils.deleteWithConfirm(summariesFile);

    DataOutputStreamAndChannel oStream = null;
    try
    {
        oStream = new DataOutputStreamAndChannel(new FileOutputStream(summariesFile));
        IndexSummary.serializer.serialize(summary, oStream, descriptor.version.hasSamplingLevel);
        ByteBufferUtil.writeWithLength(first.getKey(), oStream);
        ByteBufferUtil.writeWithLength(last.getKey(), oStream);
        ibuilder.serializeBounds(oStream);
        dbuilder.serializeBounds(oStream);
    }
    catch (IOException e)
    {
        logger.debug("Cannot save SSTable Summary: ", e);

        // corrupted hence delete it and let it load it now.
        if (summariesFile.exists())
            FileUtils.deleteWithConfirm(summariesFile);
    }
    finally
    {
        FileUtils.closeQuietly(oStream);
    }
}
 

@Test
public void testOpenWithoutPageBits() throws IOException
{
    File tmp = File.createTempFile("mapped-buffer", "tmp");
    tmp.deleteOnExit();

    RandomAccessFile file = new RandomAccessFile(tmp, "rw");

    long numValues = 1000;
    for (long i = 0; i < numValues; i++)
        file.writeLong(i);

    file.getFD().sync();

    try (MappedBuffer buffer = new MappedBuffer(file))
    {
        Assert.assertEquals(numValues * 8, buffer.limit());
        Assert.assertEquals(numValues * 8, buffer.capacity());

        for (long i = 0; i < numValues; i++)
        {
            Assert.assertEquals(i * 8, buffer.position());
            Assert.assertEquals(i, buffer.getLong());
        }
    }
    finally
    {
        FileUtils.closeQuietly(file);
    }
}
 

private static TokenTree generateTree(final long minToken, final long maxToken) throws IOException
{
    final SortedMap<Long, LongSet> toks = new TreeMap<Long, LongSet>()
    {{
            for (long i = minToken; i <= maxToken; i++)
            {
                LongSet offsetSet = new LongOpenHashSet();
                offsetSet.add(i);
                put(i, offsetSet);
            }
    }};

    final TokenTreeBuilder builder = new TokenTreeBuilder(toks).finish();
    final File treeFile = File.createTempFile("token-tree-get-test", "tt");
    treeFile.deleteOnExit();

    final SequentialWriter writer = new SequentialWriter(treeFile, 4096, false);
    builder.write(writer);
    writer.close();

    RandomAccessReader reader = null;

    try
    {
        reader = RandomAccessReader.open(treeFile);
        return new TokenTree(new MappedBuffer(reader));
    }
    finally
    {
        FileUtils.closeQuietly(reader);
    }
}
 

private MappedBuffer createTestFile(long numCount, int typeSize, int numPageBits, int padding) throws IOException
{
    final File testFile = File.createTempFile("mapped-buffer-test", "db");
    testFile.deleteOnExit();

    RandomAccessFile file = new RandomAccessFile(testFile, "rw");

    for (long i = 0; i < numCount; i++)
    {

        switch (typeSize)
        {
            case 1:
                file.write((byte) i);
                break;

            case 2:
                file.writeShort((short) i);
                break;

            case 4:
                file.writeInt((int) i);
                break;

            case 8:
                // bunch of longs
                file.writeLong(i);
                break;

            default:
                throw new IllegalArgumentException("unknown byte size: " + typeSize);
        }

        for (int j = 0; j < padding; j++)
            file.write(0);
    }

    file.getFD().sync();

    try
    {
        return new MappedBuffer(file, numPageBits);
    }
    finally
    {
        FileUtils.closeQuietly(file);
    }
}
 

/**
 * Build index summary(and optionally bloom filter) by reading through Index.db file.
 *
 * @param recreateBloomFilter true if recreate bloom filter
 * @param ibuilder
 * @param dbuilder
 * @param summaryLoaded true if index summary is already loaded and not need to build again
 * @throws IOException
 */
private void buildSummary(boolean recreateBloomFilter, SegmentedFile.Builder ibuilder, SegmentedFile.Builder dbuilder, boolean summaryLoaded, int samplingLevel) throws IOException
{
    // we read the positions in a BRAF so we don't have to worry about an entry spanning a mmap boundary.
    RandomAccessReader primaryIndex = RandomAccessReader.open(new File(descriptor.filenameFor(Component.PRIMARY_INDEX)));

    try
    {
        long indexSize = primaryIndex.length();
        long histogramCount = sstableMetadata.estimatedRowSize.count();
        long estimatedKeys = histogramCount > 0 && !sstableMetadata.estimatedRowSize.isOverflowed()
                             ? histogramCount
                             : estimateRowsFromIndex(primaryIndex); // statistics is supposed to be optional

        try(IndexSummaryBuilder summaryBuilder = summaryLoaded ? null : new IndexSummaryBuilder(estimatedKeys, metadata.getMinIndexInterval(), samplingLevel))
        {

            if (recreateBloomFilter)
                bf = FilterFactory.getFilter(estimatedKeys, metadata.getBloomFilterFpChance(), true);

            long indexPosition;
            while ((indexPosition = primaryIndex.getFilePointer()) != indexSize)
            {
                ByteBuffer key = ByteBufferUtil.readWithShortLength(primaryIndex);
                RowIndexEntry indexEntry = metadata.comparator.rowIndexEntrySerializer().deserialize(primaryIndex, descriptor.version);
                DecoratedKey decoratedKey = partitioner.decorateKey(key);
                if (first == null)
                    first = decoratedKey;
                last = decoratedKey;

                if (recreateBloomFilter)
                    bf.add(decoratedKey.getKey());

                // if summary was already read from disk we don't want to re-populate it using primary index
                if (!summaryLoaded)
                {
                    summaryBuilder.maybeAddEntry(decoratedKey, indexPosition);
                    ibuilder.addPotentialBoundary(indexPosition);
                    dbuilder.addPotentialBoundary(indexEntry.position);
                }
            }

            if (!summaryLoaded)
                indexSummary = summaryBuilder.build(partitioner);
        }
    }
    finally
    {
        FileUtils.closeQuietly(primaryIndex);
    }

    first = getMinimalKey(first);
    last = getMinimalKey(last);
}
 

/**
 * Stream file of specified sections to given channel.
 *
 * StreamWriter uses LZF compression on wire to decrease size to transfer.
 *
 * @param channel where this writes data to
 * @throws IOException on any I/O error
 */
public void write(WritableByteChannel channel) throws IOException
{
    long totalSize = totalSize();
    RandomAccessReader file = sstable.openDataReader();
    ChecksumValidator validator = new File(sstable.descriptor.filenameFor(Component.CRC)).exists()
                                ? DataIntegrityMetadata.checksumValidator(sstable.descriptor)
                                : null;
    transferBuffer = validator == null ? new byte[DEFAULT_CHUNK_SIZE] : new byte[validator.chunkSize];

    // setting up data compression stream
    compressedOutput = new LZFOutputStream(Channels.newOutputStream(channel));
    long progress = 0L;

    try
    {
        // stream each of the required sections of the file
        for (Pair<Long, Long> section : sections)
        {
            long start = validator == null ? section.left : validator.chunkStart(section.left);
            int readOffset = (int) (section.left - start);
            // seek to the beginning of the section
            file.seek(start);
            if (validator != null)
                validator.seek(start);

            // length of the section to read
            long length = section.right - start;
            // tracks write progress
            long bytesRead = 0;
            while (bytesRead < length)
            {
                long lastBytesRead = write(file, validator, readOffset, length, bytesRead);
                bytesRead += lastBytesRead;
                progress += (lastBytesRead - readOffset);
                session.progress(sstable.descriptor, ProgressInfo.Direction.OUT, progress, totalSize);
                readOffset = 0;
            }

            // make sure that current section is send
            compressedOutput.flush();
        }
    }
    finally
    {
        // no matter what happens close file
        FileUtils.closeQuietly(file);
        FileUtils.closeQuietly(validator);
    }
}
 

public void close()
{
    FileUtils.closeQuietly(dataFile);
    FileUtils.closeQuietly(indexFile);
}
 

/**
 * Validates that an index summary has full sampling, as expected when the serialization format does not support
 * persisting the sampling level.
 * @return true if the summary has full sampling, false otherwise
 */
private boolean validateSummarySamplingLevel()
{
    // We need to check index summary entries against the index to verify that none of them were dropped due to
    // downsampling.  Downsampling can drop any of the first BASE_SAMPLING_LEVEL entries (repeating that drop pattern
    // for the remainder of the summary).  Unfortunately, the first entry to be dropped is the entry at
    // index (BASE_SAMPLING_LEVEL - 1), so we need to check a full set of BASE_SAMPLING_LEVEL entries.
    Iterator<FileDataInput> segments = ifile.iterator(0);
    int i = 0;
    int summaryEntriesChecked = 0;
    int expectedIndexInterval = getMinIndexInterval();
    while (segments.hasNext())
    {
        FileDataInput in = segments.next();
        try
        {
            while (!in.isEOF())
            {
                ByteBuffer indexKey = ByteBufferUtil.readWithShortLength(in);
                if (i % expectedIndexInterval == 0)
                {
                    ByteBuffer summaryKey = ByteBuffer.wrap(indexSummary.getKey(i / expectedIndexInterval));
                    if (!summaryKey.equals(indexKey))
                        return false;
                    summaryEntriesChecked++;

                    if (summaryEntriesChecked == Downsampling.BASE_SAMPLING_LEVEL)
                        return true;
                }
                RowIndexEntry.Serializer.skip(in);
                i++;
            }
        }
        catch (IOException e)
        {
            markSuspect();
            throw new CorruptSSTableException(e, in.getPath());
        }
        finally
        {
            FileUtils.closeQuietly(in);
        }
    }

    return true;
}
 

@Test
public void testBuilder()
{
    RangeUnionIterator.Builder<Long, Token> builder = RangeUnionIterator.builder();

    Assert.assertNull(builder.getMinimum());
    Assert.assertNull(builder.getMaximum());
    Assert.assertEquals(0L, builder.getTokenCount());
    Assert.assertEquals(0L, builder.rangeCount());

    builder.add(new LongIterator(new long[] { 1L, 2L, 3L }));
    builder.add(new LongIterator(new long[] { 4L, 5L, 6L }));
    builder.add(new LongIterator(new long[] { 7L, 8L, 9L }));

    Assert.assertEquals(1L, (long) builder.getMinimum());
    Assert.assertEquals(9L, (long) builder.getMaximum());
    Assert.assertEquals(9L, builder.getTokenCount());
    Assert.assertEquals(3L, builder.rangeCount());
    Assert.assertFalse(builder.statistics.isDisjoint());

    Assert.assertEquals(1L, (long) builder.ranges.poll().getMinimum());
    Assert.assertEquals(4L, (long) builder.ranges.poll().getMinimum());
    Assert.assertEquals(7L, (long) builder.ranges.poll().getMinimum());

    RangeIterator<Long, Token> tokens = RangeUnionIterator.build(new ArrayList<RangeIterator<Long, Token>>()
    {{
        add(new LongIterator(new long[]{1L, 2L, 4L}));
        add(new LongIterator(new long[]{3L, 5L, 6L}));
    }});

    Assert.assertEquals(convert(1L, 2L, 3L, 4L, 5L, 6L), convert(tokens));

    FileUtils.closeQuietly(tokens);

    RangeIterator emptyTokens = RangeUnionIterator.builder().build();
    Assert.assertNull(emptyTokens);

    builder = RangeUnionIterator.builder();
    Assert.assertEquals(0L, builder.add((RangeIterator<Long, Token>) null).rangeCount());
    Assert.assertEquals(0L, builder.add((List<RangeIterator<Long, Token>>) null).getTokenCount());
    Assert.assertEquals(0L, builder.add(new LongIterator(new long[] {})).rangeCount());

    RangeIterator<Long, Token> single = new LongIterator(new long[] { 1L, 2L, 3L });
    RangeIterator<Long, Token> range = RangeIntersectionIterator.<Long, Token>builder().add(single).build();

    // because build should return first element if it's only one instead of building yet another iterator
    Assert.assertEquals(range, single);
}
 

private void testBuilder(Strategy strategy)
{
    RangeIterator.Builder<Long, Token> builder = RangeIntersectionIterator.builder(strategy);

    Assert.assertNull(builder.getMinimum());
    Assert.assertNull(builder.getMaximum());
    Assert.assertEquals(0L, builder.getTokenCount());
    Assert.assertEquals(0L, builder.rangeCount());

    builder.add(new LongIterator(new long[] { 1L, 2L, 6L }));
    builder.add(new LongIterator(new long[] { 4L, 5L, 6L }));
    builder.add(new LongIterator(new long[] { 6L, 8L, 9L }));

    Assert.assertEquals(6L, (long) builder.getMinimum());
    Assert.assertEquals(6L, (long) builder.getMaximum());
    Assert.assertEquals(9L, builder.getTokenCount());
    Assert.assertEquals(3L, builder.rangeCount());
    Assert.assertFalse(builder.statistics.isDisjoint());

    Assert.assertEquals(1L, (long) builder.ranges.poll().getMinimum());
    Assert.assertEquals(4L, (long) builder.ranges.poll().getMinimum());
    Assert.assertEquals(6L, (long) builder.ranges.poll().getMinimum());

    builder.add(new LongIterator(new long[] { 1L, 2L, 6L }));
    builder.add(new LongIterator(new long[] { 4L, 5L, 6L }));
    builder.add(new LongIterator(new long[] { 6L, 8L, 9L }));

    Assert.assertEquals(convert(6L), convert(builder.build()));

    builder = RangeIntersectionIterator.builder(strategy);
    builder.add(new LongIterator(new long[]{ 1L, 5L, 6L }));
    builder.add(new LongIterator(new long[]{ 3L, 5L, 6L }));

    RangeIterator<Long, Token> tokens = builder.build();

    Assert.assertEquals(convert(5L, 6L), convert(tokens));

    FileUtils.closeQuietly(tokens);

    RangeIterator emptyTokens = RangeIntersectionIterator.builder(strategy).build();
    Assert.assertNull(emptyTokens);

    builder = RangeIntersectionIterator.builder(strategy);
    Assert.assertEquals(0L, builder.add((RangeIterator<Long, Token>) null).rangeCount());
    Assert.assertEquals(0L, builder.add((List<RangeIterator<Long, Token>>) null).getTokenCount());
    Assert.assertEquals(0L, builder.add(new LongIterator(new long[] {})).rangeCount());

    RangeIterator<Long, Token> single = new LongIterator(new long[] { 1L, 2L, 3L });
    RangeIterator<Long, Token> range = RangeIntersectionIterator.<Long, Token>builder().add(single).build();

    // because build should return first element if it's only one instead of building yet another iterator
    Assert.assertEquals(range, single);

    // disjoint case
    builder = RangeIntersectionIterator.builder();
    builder.add(new LongIterator(new long[] { 1L, 2L, 3L }));
    builder.add(new LongIterator(new long[] { 4L, 5L, 6L }));

    Assert.assertTrue(builder.statistics.isDisjoint());

    RangeIterator<Long, Token> disjointIntersection = builder.build();
    Assert.assertNotNull(disjointIntersection);
    Assert.assertFalse(disjointIntersection.hasNext());

}
 

@Override
public void close() throws IOException
{
    FileUtils.closeQuietly(indexFile);
}
 

@Override
public void close() throws IOException
{
    for (RangeIterator<K, D> range : ranges)
        FileUtils.closeQuietly(range);
}
 

@Override
public void close() throws IOException
{
    for (RangeIterator<K, D> range : ranges)
        FileUtils.closeQuietly(range);
}