org.apache.cassandra.io.util.FileUtils Java Examples

/**
 * Differs from the above because it can work on any file instead of just existing
 * commit log segments managed by this manager.
 *
 * @param file segment file that is no longer in use.
 */
void recycleSegment(final File file)
{
    if (isCapExceeded()
        || CommitLogDescriptor.fromFileName(file.getName()).getMessagingVersion() != MessagingService.current_version)
    {
        // (don't decrease managed size, since this was never a "live" segment)
        logger.debug("(Unopened) segment {} is no longer needed and will be deleted now", file);
        FileUtils.deleteWithConfirm(file);
        return;
    }

    logger.debug("Recycling {}", file);
    // this wasn't previously a live segment, so add it to the managed size when we make it live
    size.addAndGet(DatabaseDescriptor.getCommitLogSegmentSize());
    segmentManagementTasks.add(new Callable<CommitLogSegment>()
    {
        public CommitLogSegment call()
        {
            return new CommitLogSegment(file.getPath());
        }
    });
}
 

public static String getReleaseVersionString()
{
    InputStream in = null;
    try
    {
        in = FBUtilities.class.getClassLoader().getResourceAsStream("org/apache/cassandra/config/version.properties");
        if (in == null)
        {
            return System.getProperty("cassandra.releaseVersion", "Unknown");
        }
        Properties props = new Properties();
        props.load(in);
        return props.getProperty("CassandraVersion");
    }
    catch (Exception e)
    {
        JVMStabilityInspector.inspectThrowable(e);
        logger.warn("Unable to load version.properties", e);
        return "debug version";
    }
    finally
    {
        FileUtils.closeQuietly(in);
    }
}
 

private IndexSummary buildSummaryAtLevel(int newSamplingLevel) throws IOException
{
    // we read the positions in a BRAF so we don't have to worry about an entry spanning a mmap boundary.
    RandomAccessReader primaryIndex = RandomAccessReader.open(new File(descriptor.filenameFor(Component.PRIMARY_INDEX)));
    try
    {
        long indexSize = primaryIndex.length();
        try (IndexSummaryBuilder summaryBuilder = new IndexSummaryBuilder(estimatedKeys(), metadata.getMinIndexInterval(), newSamplingLevel))
        {
            long indexPosition;
            while ((indexPosition = primaryIndex.getFilePointer()) != indexSize)
            {
                summaryBuilder.maybeAddEntry(partitioner.decorateKey(ByteBufferUtil.readWithShortLength(primaryIndex)), indexPosition);
                RowIndexEntry.Serializer.skip(primaryIndex);
            }

            return summaryBuilder.build(partitioner);
        }
    }
    finally
    {
        FileUtils.closeQuietly(primaryIndex);
    }
}
 

/**
 * We use a ReferenceQueue to manage deleting files that have been compacted
 * and for which no more SSTable references exist.  But this is not guaranteed
 * to run for each such file because of the semantics of the JVM gc.  So,
 * we write a marker to `compactedFilename` when a file is compacted;
 * if such a marker exists on startup, the file should be removed.
 *
 * This method will also remove SSTables that are marked as temporary.
 *
 * @return true if the file was deleted
 */
public static boolean delete(Descriptor desc, Set<Component> components)
{
    // remove the DATA component first if it exists
    if (components.contains(Component.DATA))
        FileUtils.deleteWithConfirm(desc.filenameFor(Component.DATA));
    for (Component component : components)
    {
        if (component.equals(Component.DATA) || component.equals(Component.SUMMARY))
            continue;

        FileUtils.deleteWithConfirm(desc.filenameFor(component));
    }
    FileUtils.delete(desc.filenameFor(Component.SUMMARY));

    logger.debug("Deleted {}", desc);
    return true;
}
 

private static Properties loadProperties()
{
    Properties properties = new Properties();
    InputStream stream = InvertedIndex.class.getClassLoader().getResourceAsStream("InvertedIndex.properties");
    try
    {
        properties.load(stream);
    }
    catch (Exception e)
    {
        throw new RuntimeException(e);
    }
    finally
    {
        FileUtils.closeQuietly(stream);
    }
    logger.info("loaded property file, InvertedIndex.properties");
    return properties;
}
 

/**
 * Appends new component names to the TOC component.
 */
protected static void appendTOC(Descriptor descriptor, Collection<Component> components)
{
    File tocFile = new File(descriptor.filenameFor(Component.TOC));
    PrintWriter w = null;
    try
    {
        w = new PrintWriter(new FileWriter(tocFile, true));
        for (Component component : components)
            w.println(component.name);
    }
    catch (IOException e)
    {
        throw new FSWriteError(e, tocFile);
    }
    finally
    {
        FileUtils.closeQuietly(w);
    }
}
 

/**
 * Skip the index
 * @param in the data input from which the index should be skipped
 * @throws IOException if an I/O error occurs.
 */
public static void skipIndex(DataInput in) throws IOException
{
    /* read only the column index list */
    int columnIndexSize = in.readInt();
    /* skip the column index data */
    if (in instanceof FileDataInput)
    {
        FileUtils.skipBytesFully(in, columnIndexSize);
    }
    else
    {
        // skip bytes
        byte[] skip = new byte[columnIndexSize];
        in.readFully(skip);
    }
}
 

@Test
@Ignore
public void testHugeBFSerialization() throws IOException
{
    ByteBuffer test = ByteBuffer.wrap(new byte[] {0, 1});

    File file = FileUtils.createTempFile("bloomFilterTest-", ".dat");
    BloomFilter filter = (BloomFilter) FilterFactory.getFilter(((long)Integer.MAX_VALUE / 8) + 1, 0.01d, true);
    filter.add(test);
    DataOutputStreamAndChannel out = new DataOutputStreamAndChannel(new FileOutputStream(file));
    FilterFactory.serialize(filter, out);
    filter.bitset.serialize(out);
    out.close();
    filter.close();
    
    DataInputStream in = new DataInputStream(new FileInputStream(file));
    BloomFilter filter2 = (BloomFilter) FilterFactory.deserialize(in, true);
    Assert.assertTrue(filter2.isPresent(test));
    FileUtils.closeQuietly(in);
}
 

/**
 * Dump the inter arrival times for examination if necessary.
 */
public void dumpInterArrivalTimes()
{
    File file = FileUtils.createTempFile("failuredetector-", ".dat");

    OutputStream os = null;
    try
    {
        os = new BufferedOutputStream(new FileOutputStream(file, true));
        os.write(toString().getBytes());
    }
    catch (IOException e)
    {
        throw new FSWriteError(e, file);
    }
    finally
    {
        FileUtils.closeQuietly(os);
    }
}
 

@Test
public void testSerialization() throws IOException
{
    Pair<List<DecoratedKey>, IndexSummary> random = generateRandomIndex(100, 1);
    DataOutputBuffer dos = new DataOutputBuffer();
    IndexSummary.serializer.serialize(random.right, dos, false);
    // write junk
    dos.writeUTF("JUNK");
    dos.writeUTF("JUNK");
    FileUtils.closeQuietly(dos);
    DataInputStream dis = new DataInputStream(new ByteArrayInputStream(dos.toByteArray()));
    IndexSummary is = IndexSummary.serializer.deserialize(dis, DatabaseDescriptor.getPartitioner(), false, 1, 1);
    for (int i = 0; i < 100; i++)
        assertEquals(i, is.binarySearch(random.left.get(i)));
    // read the junk
    assertEquals(dis.readUTF(), "JUNK");
    assertEquals(dis.readUTF(), "JUNK");
    is.close();
    FileUtils.closeQuietly(dis);
}
 

public static MemtablePool getMemtableAllocatorPool()
{
    long heapLimit = ((long) conf.memtable_heap_space_in_mb) << 20;
    long offHeapLimit = ((long) conf.memtable_offheap_space_in_mb) << 20;
    switch (conf.memtable_allocation_type)
    {
        case unslabbed_heap_buffers:
            return new HeapPool(heapLimit, conf.memtable_cleanup_threshold, new ColumnFamilyStore.FlushLargestColumnFamily());
        case heap_buffers:
            return new SlabPool(heapLimit, 0, conf.memtable_cleanup_threshold, new ColumnFamilyStore.FlushLargestColumnFamily());
        case offheap_buffers:
            if (!FileUtils.isCleanerAvailable())
            {
                logger.error("Could not free direct byte buffer: offheap_buffers is not a safe memtable_allocation_type without this ability, please adjust your config. This feature is only guaranteed to work on an Oracle JVM. Refusing to start.");
                System.exit(-1);
            }
            return new SlabPool(heapLimit, offHeapLimit, conf.memtable_cleanup_threshold, new ColumnFamilyStore.FlushLargestColumnFamily());
        case offheap_objects:
            return new NativePool(heapLimit, offHeapLimit, conf.memtable_cleanup_threshold, new ColumnFamilyStore.FlushLargestColumnFamily());
        default:
            throw new AssertionError();
    }
}
 

@Override
public void close()
{
    if (!FileUtils.isCleanerAvailable())
        return;

    /*
     * Try forcing the unmapping of pages using undocumented unsafe sun APIs.
     * If this fails (non Sun JVM), we'll have to wait for the GC to finalize the mapping.
     * If this works and a thread tries to access any page, hell will unleash on earth.
     */
    try
    {
        for (MappedByteBuffer segment : pages)
            FileUtils.clean(segment);
    }
    catch (Exception e)
    {
        // This is not supposed to happen
    }
}
 

@Override
protected void performSkipTo(K nextToken)
{
    List<RangeIterator<K, D>> changedRanges = new ArrayList<>();

    while (!ranges.isEmpty())
    {
        if (ranges.peek().getCurrent().compareTo(nextToken) >= 0)
            break;

        RangeIterator<K, D> head = ranges.poll();

        if (head.getMaximum().compareTo(nextToken) >= 0)
        {
            head.skipTo(nextToken);
            changedRanges.add(head);
            continue;
        }

        FileUtils.closeQuietly(head);
    }

    for (RangeIterator<K, D> range : changedRanges)
        ranges.add(range);
}
 

/**
 * Removes an SSTable directory for a given table path if the directory is empty
 * @param tablePath Path to remove if empty
 */
private void clearSSTableDirectoryIfEmpty(String tablePath) {
	String SSTablePath = this.defaultSSTableOutputPath + "/" + keyspaceDefinition.getName() + "/" + tablePath;
	File SSTableDirectory = new File(SSTablePath);

	if (SSTableDirectory.isDirectory() && SSTableDirectory.list().length == 0) {
		FileUtils.deleteRecursive(SSTableDirectory);
	}
}
 

private static void removeAllSSTables(String ks, List<String> tables)
{
    // clean up data directory which are stored as data directory/keyspace/data files
    for (File d : Directories.getKSChildDirectories(ks))
    {
        if (d.exists() && containsAny(d.getName(), tables))
            FileUtils.deleteRecursive(d);
    }
}
 

public void shutdown() {
    disconnect();
    if (zkServer != null) {
        zkServer.shutdown();
        zkServer = null;
    }
    FileUtils.deleteRecursive(logDir);
    FileUtils.deleteRecursive(dataDir);
}
 

protected static void cleanupSavedCaches()
{
    File cachesDir = new File(DatabaseDescriptor.getSavedCachesLocation());

    if (!cachesDir.exists() || !cachesDir.isDirectory())
        return;

    FileUtils.delete(cachesDir.listFiles());
}
 

/**
 * Creates an SSTable keyspace output directory at defaultSSTableOutputPath and table output directories for each SSTable,
 * and initializes each SSTableWriter for each static and index table in this keyspace.
 * @param sorted Defines if the SSTableWriters created by this should be set as sorted, which improves performance if
 *               rows are inserted in SSTable sort order, but throws exceptions if they are inserted in the wrong order.
 * @throws CQLGenerationException
 * @throws IOException
 */
public void initializeSSTableWriters(boolean sorted) throws CQLGenerationException, IOException {
	Map<String, CDefinition> definitions = this.keyspaceDefinition.getDefinitions();

	// Make sure the SSTableOutput directory exists and is clear
	String keyspacePath = this.defaultSSTableOutputPath + "/" + this.keyspaceDefinition.getName();
	File keyspaceDir = new File(keyspacePath);
	if (keyspaceDir.exists()) {
		FileUtils.deleteRecursive(new File(keyspacePath));
	}
	if (!new File(keyspacePath).mkdir()) {
		throw new IOException("Failed to create SSTable keyspace output directory at " + keyspacePath);
	}

	this.SSTableWriters.put(CObjectShardList.SHARD_INDEX_TABLE_NAME, Pair.create(this.buildSSTableWriterForShardIndexTable(sorted), (Map<CIndex, CQLSSTableWriter>) null));

	for (String defName : definitions.keySet()) {
		// Build the CQLSSTableWriter for the static table
		CQLSSTableWriter staticWriter = buildSSTableWriterForStaticTable(definitions.get(defName), sorted);

		// Build the CQLSSTableWriter for all the index tables
		List<CIndex> indexes = definitions.get(defName).getIndexesAsList();
		Map<CIndex, CQLSSTableWriter> indexWriters = Maps.newHashMap();
		for (CIndex index : indexes) {
			CQLSSTableWriter writer = buildSSTableWriterForWideTable(definitions.get(defName), index, sorted);
			indexWriters.put(index, writer);
		}
		this.SSTableWriters.put(defName, Pair.create(staticWriter, indexWriters));
	}
}
 

void skipUnsafe(long bytes) throws IOException {
    if (bytes <= 0) {
        return;
    }

    FileUtils.skipBytesFully(input, bytes);
}
 

void skipPromotedIndexes() throws IOException {
    int size = input.readInt();
    if (size <= 0) {
        return;
    }

    FileUtils.skipBytesFully(input, size);
}
 

public void maybeIncrementallyBackup(final SSTableReader sstable)
{
    if (!DatabaseDescriptor.isIncrementalBackupsEnabled())
        return;

    File backupsDir = Directories.getBackupsDirectory(sstable.descriptor);
    sstable.createLinks(FileUtils.getCanonicalPath(backupsDir));
}
 

private void saveSummary(SegmentedFile.Builder ibuilder, SegmentedFile.Builder dbuilder, IndexSummary summary)
{
    File summariesFile = new File(descriptor.filenameFor(Component.SUMMARY));
    if (summariesFile.exists())
        FileUtils.deleteWithConfirm(summariesFile);

    DataOutputStreamAndChannel oStream = null;
    try
    {
        oStream = new DataOutputStreamAndChannel(new FileOutputStream(summariesFile));
        IndexSummary.serializer.serialize(summary, oStream, descriptor.version.hasSamplingLevel);
        ByteBufferUtil.writeWithLength(first.getKey(), oStream);
        ByteBufferUtil.writeWithLength(last.getKey(), oStream);
        ibuilder.serializeBounds(oStream);
        dbuilder.serializeBounds(oStream);
    }
    catch (IOException e)
    {
        logger.debug("Cannot save SSTable Summary: ", e);

        // corrupted hence delete it and let it load it now.
        if (summariesFile.exists())
            FileUtils.deleteWithConfirm(summariesFile);
    }
    finally
    {
        FileUtils.closeQuietly(oStream);
    }
}
 

@Override
public void write(WritableByteChannel channel) throws IOException
{
    long totalSize = totalSize();
    RandomAccessReader file = sstable.openDataReader();
    FileChannel fc = file.getChannel();

    long progress = 0L;
    // calculate chunks to transfer. we want to send continuous chunks altogether.
    List<Pair<Long, Long>> sections = getTransferSections(compressionInfo.chunks);
    try
    {
        // stream each of the required sections of the file
        for (Pair<Long, Long> section : sections)
        {
            // length of the section to stream
            long length = section.right - section.left;
            // tracks write progress
            long bytesTransferred = 0;
            while (bytesTransferred < length)
            {
                int toTransfer = (int) Math.min(CHUNK_SIZE, length - bytesTransferred);
                limiter.acquire(toTransfer);
                long lastWrite = fc.transferTo(section.left + bytesTransferred, toTransfer, channel);
                bytesTransferred += lastWrite;
                progress += lastWrite;
                session.progress(sstable.descriptor, ProgressInfo.Direction.OUT, progress, totalSize);
            }
        }
    }
    finally
    {
        // no matter what happens close file
        FileUtils.closeQuietly(file);
    }
}
 

public CfAssumptions()
{
    assumptions = new HashMap<String, Map<String, Map<String, String>>>();
    assumptionsChanged = false;
    assumptionDirectory = FBUtilities.getToolsOutputDirectory();

    File oldAssumptionDir = new File(System.getProperty("user.home") + File.separator + ".cassandra-cli");
    if (oldAssumptionDir.exists())
    {
        File oldAssumptionFile = new File(oldAssumptionDir, ASSUMPTIONS_FILENAME);
        if (oldAssumptionFile.exists())
            FileUtils.renameWithConfirm(oldAssumptionFile, new File(assumptionDirectory, ASSUMPTIONS_FILENAME));
        FileUtils.deleteRecursive(oldAssumptionDir);
    }
}
 

private static File handleHistoryFiles()
{
    File outputDir = FBUtilities.getToolsOutputDirectory();
    File historyFile = new File(outputDir, HISTORYFILE);
    File oldHistoryFile = new File(System.getProperty("user.home"), OLD_HISTORYFILE);
    if(oldHistoryFile.exists())
        FileUtils.renameWithConfirm(oldHistoryFile, historyFile);

    return historyFile;
}
 

@Override
public void execute(NodeProbe probe)
{
    try
    {
        System.out.println("Snapshot Details: ");

        final Map<String,TabularData> snapshotDetails = probe.getSnapshotDetails();
        if (snapshotDetails.isEmpty())
        {
            System.out.printf("There are no snapshots");
            return;
        }

        final long trueSnapshotsSize = probe.trueSnapshotsSize();
        final String format = "%-20s%-29s%-29s%-19s%-19s%n";
        // display column names only once
        final List<String> indexNames = snapshotDetails.entrySet().iterator().next().getValue().getTabularType().getIndexNames();
        System.out.printf(format, (Object[]) indexNames.toArray(new String[indexNames.size()]));

        for (final Map.Entry<String, TabularData> snapshotDetail : snapshotDetails.entrySet())
        {
            Set<?> values = snapshotDetail.getValue().keySet();
            for (Object eachValue : values)
            {
                final List<?> value = (List<?>) eachValue;
                System.out.printf(format, value.toArray(new Object[value.size()]));
            }
        }

        System.out.println("\nTotal TrueDiskSpaceUsed: " + FileUtils.stringifyFileSize(trueSnapshotsSize) + "\n");
    }
    catch (Exception e)
    {
        throw new RuntimeException("Error during list snapshot", e);
    }
}
 

/**
 * Creates the base storage directories required for running a Cassandra instance.
 */
private static void setupCassandraDirectories() throws IOException {
    final Path root = Files.createTempDirectory("cassandra");
    final Path config = root.resolve("cassandra.yaml");
    Files.copy(ReactorExampleApplication.class.getResourceAsStream("/cassandra.yaml"), config);
    System.setProperty("cassandra.config", "file:" + config.toString());
    System.setProperty("cassandra.storagedir", root.toString());
    System.setProperty("cassandra-foreground", "true");
    Stream.of(DatabaseDescriptor.getAllDataFileLocations())
        .map(root::resolve)
        .map(Path::toFile)
        .forEach(FileUtils::createDirectory);
}
 

/**
 *
 * @return  Return a map of all snapshots to space being used
 * The pair for a snapshot has size on disk and true size.
 */
public Map<String, Pair<Long, Long>> getSnapshotDetails()
{
    final Map<String, Pair<Long, Long>> snapshotSpaceMap = new HashMap<>();
    for (final File dir : dataPaths)
    {
        final File snapshotDir = new File(dir,SNAPSHOT_SUBDIR);
        if (snapshotDir.exists() && snapshotDir.isDirectory())
        {
            final File[] snapshots  = snapshotDir.listFiles();
            if (snapshots != null)
            {
                for (final File snapshot : snapshots)
                {
                    if (snapshot.isDirectory())
                    {
                        final long sizeOnDisk = FileUtils.folderSize(snapshot);
                        final long trueSize = getTrueAllocatedSizeIn(snapshot);
                        Pair<Long,Long> spaceUsed = snapshotSpaceMap.get(snapshot.getName());
                        if (spaceUsed == null)
                            spaceUsed =  Pair.create(sizeOnDisk,trueSize);
                        else
                            spaceUsed = Pair.create(spaceUsed.left + sizeOnDisk, spaceUsed.right + trueSize);
                        snapshotSpaceMap.put(snapshot.getName(), spaceUsed);
                    }
                }
            }
        }
    }

    return snapshotSpaceMap;
}
 

public static void from(final String snapshot, final String ks, final String cf, Map.Entry<String, Pair<Long,Long>> snapshotDetail, TabularDataSupport result)
{
    try
    {
        final String totalSize = FileUtils.stringifyFileSize(snapshotDetail.getValue().left);
        final String liveSize =  FileUtils.stringifyFileSize(snapshotDetail.getValue().right);
        result.put(new CompositeDataSupport(COMPOSITE_TYPE, ITEM_NAMES,
                new Object[]{ snapshot, ks, cf, liveSize, totalSize }));
    }
    catch (OpenDataException e)
    {
        throw new RuntimeException(e);
    }
}
 

public static void clearSnapshot(String snapshotName, List<File> snapshotDirectories)
{
    // If snapshotName is empty or null, we will delete the entire snapshot directory
    String tag = snapshotName == null ? "" : snapshotName;
    for (File dir : snapshotDirectories)
    {
        File snapshotDir = new File(dir, join(SNAPSHOT_SUBDIR, tag));
        if (snapshotDir.exists())
        {
            if (logger.isDebugEnabled())
                logger.debug("Removing snapshot directory {}", snapshotDir);
            FileUtils.deleteRecursive(snapshotDir);
        }
    }
}