org.apache.cassandra.io.FSWriteError Java Examples

@VisibleForTesting
protected boolean finish(Descriptor descriptor, File file) throws FSWriteError
{
    // no terms means there is nothing to build
    if (terms.isEmpty())
        return false;

    // split terms into suffixes only if it's text, otherwise (even if SUFFIX is set) use terms in original form
    SA sa = ((termComparator instanceof UTF8Type || termComparator instanceof AsciiType) && mode == Mode.SUFFIX)
                ? new SuffixSA(termComparator, mode) : new IntegralSA(termComparator, mode);

    for (Map.Entry<ByteBuffer, TokenTreeBuilder> term : terms.entrySet())
        sa.add(term.getKey(), term.getValue());

    finish(descriptor, Pair.create(minKey, maxKey), file, sa.finish());
    return true;
}
 

public void writeHeader(CompressionParameters parameters) {
    try {
        writeUTF(parameters.sstableCompressor.getClass().getSimpleName());
        writeInt(parameters.otherOptions.size());
        for (Map.Entry<String, String> entry : parameters.otherOptions.entrySet()) {
            writeUTF(entry.getKey());
            writeUTF(entry.getValue());
        }

        // store the length of the chunk
        writeInt(parameters.chunkLength());
        // store position and reserve a place for uncompressed data length and chunks count
        dataLengthOffset = getFilePointer();
        writeLong(-1);
        writeInt(-1);
    } catch (IOException e) {
        throw new FSWriteError(e, filePath);
    }
}
 

public void writeFullChecksum(Descriptor descriptor)
{
    File outFile = new File(descriptor.filenameFor(Component.DIGEST));
    BufferedWriter out = null;
    try
    {
        out = Files.newBufferedWriter(outFile.toPath(), Charsets.UTF_8);
        out.write(String.valueOf(fullChecksum.getValue()));
    }
    catch (IOException e)
    {
        throw new FSWriteError(e, outFile);
    }
    finally
    {
        FileUtils.closeQuietly(out);
    }
}
 

public static void createHardLink(File from, File to)
{
    if (to.exists())
        throw new RuntimeException("Tried to create duplicate hard link to " + to);
    if (!from.exists())
        throw new RuntimeException("Tried to hard link to file that does not exist " + from);

    try
    {
        Files.createLink(to.toPath(), from.toPath());
    }
    catch (IOException e)
    {
        throw new FSWriteError(e, to);
    }
}
 

@Override
public void close()
{
    if (buffer == null)
        return; // already closed

    super.close();
    sstableMetadataCollector.addCompressionRatio(compressedSize, originalSize);
    try
    {
        metadataWriter.close(current, chunkCount);
    }
    catch (IOException e)
    {
        throw new FSWriteError(e, getPath());
    }
}
 

private void writeHeader(DataOutput out, long dataLength, int chunks)
{
    try
    {
        out.writeUTF(parameters.sstableCompressor.getClass().getSimpleName());
        out.writeInt(parameters.otherOptions.size());
        for (Map.Entry<String, String> entry : parameters.otherOptions.entrySet())
        {
            out.writeUTF(entry.getKey());
            out.writeUTF(entry.getValue());
        }

        // store the length of the chunk
        out.writeInt(parameters.chunkLength());
        // store position and reserve a place for uncompressed data length and chunks count
        out.writeLong(dataLength);
        out.writeInt(chunks);
    }
    catch (IOException e)
    {
        throw new FSWriteError(e, filePath);
    }
}
 

/**
 * Appends new component names to the TOC component.
 */
protected static void appendTOC(Descriptor descriptor, Collection<Component> components)
{
    File tocFile = new File(descriptor.filenameFor(Component.TOC));
    PrintWriter w = null;
    try
    {
        w = new PrintWriter(new FileWriter(tocFile, true));
        for (Component component : components)
            w.println(component.name);
    }
    catch (IOException e)
    {
        throw new FSWriteError(e, tocFile);
    }
    finally
    {
        FileUtils.closeQuietly(w);
    }
}
 

public void append(DecoratedKey key, RowIndexEntry indexEntry, long dataEnd)
{
    bf.add(key.getKey());
    long indexStart = indexFile.getFilePointer();
    try
    {
        ByteBufferUtil.writeWithShortLength(key.getKey(), indexFile.stream);
        metadata.comparator.rowIndexEntrySerializer().serialize(indexEntry, indexFile.stream);
    }
    catch (IOException e)
    {
        throw new FSWriteError(e, indexFile.getPath());
    }
    long indexEnd = indexFile.getFilePointer();

    if (logger.isTraceEnabled())
        logger.trace("wrote index entry: " + indexEntry + " at " + indexStart);

    summary.maybeAddEntry(key, indexStart, indexEnd, dataEnd);
    builder.addPotentialBoundary(indexStart);
}
 

private static void writeMetadata(Descriptor desc, Map<MetadataType, MetadataComponent> components)
{
    SequentialWriter out = SequentialWriter.open(new File(desc.filenameFor(Component.STATS)));
    try
    {
        desc.getMetadataSerializer().serialize(components, out.stream);
    }
    catch (IOException e)
    {
        throw new FSWriteError(e, out.getPath());
    }
    finally
    {
        out.close();
    }
}
 

/**
 * @param row
 * @return null if the row was compacted away entirely; otherwise, the PK index entry for this row
 */
public RowIndexEntry append(AbstractCompactedRow row)
{
    long startPosition = beforeAppend(row.key);
    RowIndexEntry entry;
    try
    {
        entry = row.write(startPosition, dataFile.stream);
        if (entry == null)
            return null;
    }
    catch (IOException e)
    {
        throw new FSWriteError(e, dataFile.getPath());
    }
    long endPosition = dataFile.getFilePointer();
    sstableMetadataCollector.update(endPosition - startPosition, row.columnStats());
    afterAppend(row.key, endPosition, entry);
    return entry;
}
 

/**
 * Dump the inter arrival times for examination if necessary.
 */
public void dumpInterArrivalTimes()
{
    File file = FileUtils.createTempFile("failuredetector-", ".dat");

    OutputStream os = null;
    try
    {
        os = new BufferedOutputStream(new FileOutputStream(file, true));
        os.write(toString().getBytes());
    }
    catch (IOException e)
    {
        throw new FSWriteError(e, file);
    }
    finally
    {
        FileUtils.closeQuietly(os);
    }
}
 

private void writeSnapshotManifest(final JSONArray filesJSONArr, final String snapshotName)
{
    final File manifestFile = directories.getSnapshotManifestFile(snapshotName);
    final JSONObject manifestJSON = new JSONObject();
    manifestJSON.put("files", filesJSONArr);

    try
    {
        if (!manifestFile.getParentFile().exists())
            manifestFile.getParentFile().mkdirs();
        PrintStream out = new PrintStream(manifestFile);
        out.println(manifestJSON.toJSONString());
        out.close();
    }
    catch (IOException e)
    {
        throw new FSWriteError(e, manifestFile);
    }
}
 

/**
 * Recycle processes an unneeded segment file for reuse.
 *
 * @return a new CommitLogSegment representing the newly reusable segment.
 */
CommitLogSegment recycle()
{
    try
    {
        sync();
    }
    catch (FSWriteError e)
    {
        logger.error("I/O error flushing {} {}", this, e.getMessage());
        throw e;
    }

    close();

    return new CommitLogSegment(getPath());
}
 

public void append(DecoratedKey decoratedKey, ColumnFamily cf)
{
    if (decoratedKey.getKey().remaining() > FBUtilities.MAX_UNSIGNED_SHORT)
    {
        logger.error("Key size {} exceeds maximum of {}, skipping row",
                     decoratedKey.getKey().remaining(),
                     FBUtilities.MAX_UNSIGNED_SHORT);
        return;
    }

    long startPosition = beforeAppend(decoratedKey);
    long endPosition;
    try
    {
        RowIndexEntry entry = rawAppend(cf, startPosition, decoratedKey, dataFile.stream);
        endPosition = dataFile.getFilePointer();
        afterAppend(decoratedKey, endPosition, entry);
    }
    catch (IOException e)
    {
        throw new FSWriteError(e, dataFile.getPath());
    }
    sstableMetadataCollector.update(endPosition - startPosition, cf.getColumnStats());
}
 

public static File createTempFile(String prefix, String suffix, File directory)
{
    try
    {
        return File.createTempFile(prefix, suffix, directory);
    }
    catch (IOException e)
    {
        throw new FSWriteError(e, directory);
    }
}
 

/**
 * Reset the writer so that the next chunk offset written will be the
 * one of {@code chunkIndex}.
 */
public void resetAndTruncate(int chunkIndex) {
    try {
        seek(dataLengthOffset
                + 8 // size reserved for uncompressed data length
                + 4 // size reserved for chunk count
                + (chunkIndex * 8L));
        getChannel().truncate(getFilePointer());
    } catch (IOException e) {
        throw new FSWriteError(e, filePath);
    }
}
 

@Test
public void testDiskFailurePolicy_best_effort()
{
    DiskFailurePolicy origPolicy = DatabaseDescriptor.getDiskFailurePolicy();
    
    try 
    {
        DatabaseDescriptor.setDiskFailurePolicy(DiskFailurePolicy.best_effort);
        // Fake a Directory creation failure
        if (Directories.dataDirectories.length > 0)
        {
            String[] path = new String[] {KS, "bad"};
            File dir = new File(Directories.dataDirectories[0].location, StringUtils.join(path, File.separator));
            FileUtils.handleFSError(new FSWriteError(new IOException("Unable to create directory " + dir), dir));
        }

        for (DataDirectory dd : Directories.dataDirectories)
        {
            File file = new File(dd.location, new File(KS, "bad").getPath());
            assertTrue(BlacklistedDirectories.isUnwritable(file));
        }
    } 
    finally 
    {
        DatabaseDescriptor.setDiskFailurePolicy(origPolicy);
    }
}
 

private void handle(Throwable t, boolean throwExceptions)
{
    if (!throwExceptions)
        logger.warn("Suppressing exception thrown while aborting writer", t);
    else
        throw new FSWriteError(t, getPath());
}
 

public void truncate(long toSize)
{
    try
    {
        out.getChannel().truncate(toSize);
    }
    catch (IOException e)
    {
        throw new FSWriteError(e, getPath());
    }
}
 

/**
 * Override this method instead of overriding flush()
 * @throws FSWriteError on any I/O error.
 */
protected void flushData()
{
    try
    {
        out.write(buffer, 0, validBufferBytes);
        lastFlushOffset += validBufferBytes;
    }
    catch (IOException e)
    {
        throw new FSWriteError(e, getPath());
    }
    if (runPostFlush != null)
        runPostFlush.run();
}
 

protected void syncDataOnlyInternal()
{
    try
    {
        out.getFD().sync();
    }
    catch (IOException e)
    {
        throw new FSWriteError(e, getPath());
    }
}
 

public static void createDirectory(File directory)
{
    if (!directory.exists())
    {
        if (!directory.mkdirs())
            throw new FSWriteError(new IOException("Failed to mkdirs " + directory), directory);
    }
}
 

void internalClose()
{
    try
    {
        if (FileUtils.isCleanerAvailable())
            FileUtils.clean(buffer);
        logFileAccessor.close();
    }
    catch (IOException e)
    {
        throw new FSWriteError(e, getPath());
    }
}
 

/**
 * Closes the index and bloomfilter, making the public state of this writer valid for consumption.
 */
public void close()
{
    if (components.contains(Component.FILTER))
    {
        String path = descriptor.filenameFor(Component.FILTER);
        try
        {
            // bloom filter
            FileOutputStream fos = new FileOutputStream(path);
            DataOutputStreamPlus stream = new DataOutputStreamPlus(new BufferedOutputStream(fos));
            FilterFactory.serialize(bf, stream);
            stream.flush();
            fos.getFD().sync();
            stream.close();
        }
        catch (IOException e)
        {
            throw new FSWriteError(e, path);
        }
    }

    // index
    long position = indexFile.getFilePointer();
    indexFile.close(); // calls force
    FileUtils.truncate(indexFile.getPath(), position);
}
 

private static File getOrCreate(File base, String... subdirs)
{
    File dir = subdirs == null || subdirs.length == 0 ? base : new File(base, join(subdirs));
    if (dir.exists())
    {
        if (!dir.isDirectory())
            throw new AssertionError(String.format("Invalid directory path %s: path exists but is not a directory", dir));
    }
    else if (!dir.mkdirs() && !(dir.exists() && dir.isDirectory()))
    {
        throw new FSWriteError(new IOException("Unable to create directory " + dir), dir);
    }
    return dir;
}
 

/**
 * Constructs a new segment file.
 *
 * @param filePath  if not null, recycles the existing file by renaming it and truncating it to CommitLog.SEGMENT_SIZE.
 */
CommitLogSegment(String filePath)
{
    id = getNextId();
    descriptor = new CommitLogDescriptor(id);
    logFile = new File(DatabaseDescriptor.getCommitLogLocation(), descriptor.fileName());
    boolean isCreating = true;

    try
    {
        if (filePath != null)
        {
            File oldFile = new File(filePath);

            if (oldFile.exists())
            {
                logger.debug("Re-using discarded CommitLog segment for {} from {}", id, filePath);
                if (!oldFile.renameTo(logFile))
                    throw new IOException("Rename from " + filePath + " to " + id + " failed");
                isCreating = false;
            }
        }

        // Open the initial the segment file
        logFileAccessor = new RandomAccessFile(logFile, "rw");

        if (isCreating)
            logger.debug("Creating new commit log segment {}", logFile.getPath());

        // Map the segment, extending or truncating it to the standard segment size.
        // (We may have restarted after a segment size configuration change, leaving "incorrectly"
        // sized segments on disk.)
        logFileAccessor.setLength(DatabaseDescriptor.getCommitLogSegmentSize());
        fd = CLibrary.getfd(logFileAccessor.getFD());

        buffer = logFileAccessor.getChannel().map(FileChannel.MapMode.READ_WRITE, 0, DatabaseDescriptor.getCommitLogSegmentSize());
        // write the header
        CommitLogDescriptor.writeHeader(buffer, descriptor);
        // mark the initial sync marker as uninitialised
        buffer.putInt(CommitLogDescriptor.HEADER_SIZE, 0);
        buffer.putLong(CommitLogDescriptor.HEADER_SIZE + 4, 0);
        allocatePosition.set(CommitLogDescriptor.HEADER_SIZE + SYNC_MARKER_SIZE);
        lastSyncedOffset = CommitLogDescriptor.HEADER_SIZE;
    }
    catch (IOException e)
    {
        throw new FSWriteError(e, logFile);
    }
}
 

/**
 * @throws IOException if a read from the DataInput fails
 * @throws FSWriteError if a write to the dataFile fails
 */
public long appendFromStream(DecoratedKey key, CFMetaData metadata, DataInput in, Descriptor.Version version) throws IOException
{
    long currentPosition = beforeAppend(key);

    ColumnStats.MaxLongTracker maxTimestampTracker = new ColumnStats.MaxLongTracker(Long.MAX_VALUE);
    ColumnStats.MinLongTracker minTimestampTracker = new ColumnStats.MinLongTracker(Long.MIN_VALUE);
    ColumnStats.MaxIntTracker maxDeletionTimeTracker = new ColumnStats.MaxIntTracker(Integer.MAX_VALUE);
    List<ByteBuffer> minColumnNames = Collections.emptyList();
    List<ByteBuffer> maxColumnNames = Collections.emptyList();
    StreamingHistogram tombstones = new StreamingHistogram(TOMBSTONE_HISTOGRAM_BIN_SIZE);
    boolean hasLegacyCounterShards = false;

    ColumnFamily cf = ArrayBackedSortedColumns.factory.create(metadata);
    cf.delete(DeletionTime.serializer.deserialize(in));

    ColumnIndex.Builder columnIndexer = new ColumnIndex.Builder(cf, key.getKey(), dataFile.stream);

    if (cf.deletionInfo().getTopLevelDeletion().localDeletionTime < Integer.MAX_VALUE)
    {
        tombstones.update(cf.deletionInfo().getTopLevelDeletion().localDeletionTime);
        maxDeletionTimeTracker.update(cf.deletionInfo().getTopLevelDeletion().localDeletionTime);
        minTimestampTracker.update(cf.deletionInfo().getTopLevelDeletion().markedForDeleteAt);
        maxTimestampTracker.update(cf.deletionInfo().getTopLevelDeletion().markedForDeleteAt);
    }

    Iterator<RangeTombstone> rangeTombstoneIterator = cf.deletionInfo().rangeIterator();
    while (rangeTombstoneIterator.hasNext())
    {
        RangeTombstone rangeTombstone = rangeTombstoneIterator.next();
        tombstones.update(rangeTombstone.getLocalDeletionTime());
        minTimestampTracker.update(rangeTombstone.timestamp());
        maxTimestampTracker.update(rangeTombstone.timestamp());
        maxDeletionTimeTracker.update(rangeTombstone.getLocalDeletionTime());
        minColumnNames = ColumnNameHelper.minComponents(minColumnNames, rangeTombstone.min, metadata.comparator);
        maxColumnNames = ColumnNameHelper.maxComponents(maxColumnNames, rangeTombstone.max, metadata.comparator);
    }

    Iterator<OnDiskAtom> iter = metadata.getOnDiskIterator(in, ColumnSerializer.Flag.PRESERVE_SIZE, Integer.MIN_VALUE, version);
    try
    {
        while (iter.hasNext())
        {
            OnDiskAtom atom = iter.next();
            if (atom == null)
                break;

            if (atom instanceof CounterCell)
            {
                atom = ((CounterCell) atom).markLocalToBeCleared();
                hasLegacyCounterShards = hasLegacyCounterShards || ((CounterCell) atom).hasLegacyShards();
            }

            int deletionTime = atom.getLocalDeletionTime();
            if (deletionTime < Integer.MAX_VALUE)
                tombstones.update(deletionTime);
            minTimestampTracker.update(atom.timestamp());
            maxTimestampTracker.update(atom.timestamp());
            minColumnNames = ColumnNameHelper.minComponents(minColumnNames, atom.name(), metadata.comparator);
            maxColumnNames = ColumnNameHelper.maxComponents(maxColumnNames, atom.name(), metadata.comparator);
            maxDeletionTimeTracker.update(atom.getLocalDeletionTime());

            columnIndexer.add(atom); // This write the atom on disk too
        }

        columnIndexer.maybeWriteEmptyRowHeader();
        dataFile.stream.writeShort(END_OF_ROW);
    }
    catch (IOException e)
    {
        throw new FSWriteError(e, dataFile.getPath());
    }

    sstableMetadataCollector.updateMinTimestamp(minTimestampTracker.get())
                            .updateMaxTimestamp(maxTimestampTracker.get())
                            .updateMaxLocalDeletionTime(maxDeletionTimeTracker.get())
                            .addRowSize(dataFile.getFilePointer() - currentPosition)
                            .addColumnCount(columnIndexer.writtenAtomCount())
                            .mergeTombstoneHistogram(tombstones)
                            .updateMinColumnNames(minColumnNames)
                            .updateMaxColumnNames(maxColumnNames)
                            .updateHasLegacyCounterShards(hasLegacyCounterShards);
    afterAppend(key, currentPosition, RowIndexEntry.create(currentPosition, cf.deletionInfo().getTopLevelDeletion(), columnIndexer.build()));
    return currentPosition;
}
 

/**
 * Finishes up index building process by creating/populating index file.
 *
 * @param indexFile The file to write index contents to.
 *
 * @return true if index was written successfully, false otherwise (e.g. if index was empty).
 *
 * @throws FSWriteError on I/O error.
 */
public boolean finish(File indexFile) throws FSWriteError
{
    return finish(Descriptor.CURRENT, indexFile);
}