Java Code Examples for org.apache.cassandra.config.DatabaseDescriptor#getCommitLogLocation()

void processLastModifiedCommitLog() throws IOException {
    LOGGER.warn("CommitLogProcessor will read the last modified commit log from the COMMIT LOG "
            + "DIRECTORY based on modified timestamp, NOT FROM THE CDC_RAW DIRECTORY. This method "
            + "should not be used in PRODUCTION!");
    File commitLogDir = new File(DatabaseDescriptor.getCommitLogLocation());
    File[] files = CommitLogUtil.getCommitLogs(commitLogDir);

    File lastModified = null;
    for (File file : files) {
        if (lastModified == null || lastModified.lastModified() < file.lastModified()) {
            lastModified = file;
        }
    }

    if (lastModified != null) {
        processCommitLog(lastModified);
    }
    else {
        LOGGER.info("No commit logs found in {}", DatabaseDescriptor.getCommitLogLocation());
    }
}
 

@Test
public void testProcessCommitLogs() throws Exception {
    int commitLogRowSize = 10;
    context.getCassandraClient().execute("CREATE TABLE IF NOT EXISTS " + keyspaceTable("cdc_table") + " (a int, b int, PRIMARY KEY(a)) WITH cdc = true;");
    context.getSchemaHolder().refreshSchemas();

    // programmatically add insertion and deletion events into commit log, this is because running an 'INSERT' or 'DELETE'
    // cql against the embedded Cassandra does not modify the commit log file on disk.
    CFMetaData cfMetaData = Schema.instance.getCFMetaData(TEST_KEYSPACE, "cdc_table");
    for (int i = 0; i < commitLogRowSize; i++) {
        SimpleBuilders.PartitionUpdateBuilder puBuilder = new SimpleBuilders.PartitionUpdateBuilder(cfMetaData, i);
        Row row = puBuilder.row().add("b", i).build();
        PartitionUpdate pu = PartitionUpdate.singleRowUpdate(cfMetaData, puBuilder.build().partitionKey(), row);
        Mutation m = new Mutation(pu);
        CommitLog.instance.add(m);
    }
    CommitLog.instance.sync(true);

    // check to make sure there are no records in the queue to begin with
    ChangeEventQueue<Event> queue = context.getQueue();
    assertEquals(queue.totalCapacity(), queue.remainingCapacity());

    // process the logs in commit log directory
    File cdcLoc = new File(DatabaseDescriptor.getCommitLogLocation());
    File[] commitLogs = CommitLogUtil.getCommitLogs(cdcLoc);
    for (File commitLog : commitLogs) {
        commitLogProcessor.processCommitLog(commitLog);
    }

    // verify the commit log has been processed and records have been enqueued
    List<Event> events = queue.poll();
    int eofEventSize = commitLogs.length;
    assertEquals(commitLogRowSize + eofEventSize, events.size());
    for (int i = 0; i < events.size(); i++) {
        Event event = events.get(i);
        if (event instanceof Record) {
            Record record = (Record) events.get(i);
            assertEquals(record.getEventType(), Event.EventType.CHANGE_EVENT);
            assertEquals(record.getSource().cluster, DatabaseDescriptor.getClusterName());
            assertFalse(record.getSource().snapshot);
            assertEquals(record.getSource().keyspaceTable.name(), keyspaceTable("cdc_table"));
            assertTrue(record.getSource().offsetPosition.fileName.contains(String.valueOf(CommitLog.instance.getCurrentPosition().segmentId)));
        }
        else if (event instanceof EOFEvent) {
            EOFEvent eofEvent = (EOFEvent) event;
            assertTrue(eofEvent.success);
        }
        else {
            throw new Exception("unexpected event type");
        }
    }

    deleteTestKeyspaceTables();
}
 

public void maybeRestoreArchive()
{
    if (Strings.isNullOrEmpty(restoreDirectories))
        return;

    for (String dir : restoreDirectories.split(DELIMITER))
    {
        File[] files = new File(dir).listFiles();
        if (files == null)
        {
            throw new RuntimeException("Unable to list directory " + dir);
        }
        for (File fromFile : files)
        {
            CommitLogDescriptor fromHeader = CommitLogDescriptor.fromHeader(fromFile);
            CommitLogDescriptor fromName = CommitLogDescriptor.isValid(fromFile.getName()) ? CommitLogDescriptor.fromFileName(fromFile.getName()) : null;
            CommitLogDescriptor descriptor;
            if (fromHeader == null && fromName == null)
                throw new IllegalStateException("Cannot safely construct descriptor for segment, either from its name or its header: " + fromFile.getPath());
            else if (fromHeader != null && fromName != null && !fromHeader.equals(fromName))
                throw new IllegalStateException(String.format("Cannot safely construct descriptor for segment, as name and header descriptors do not match (%s vs %s): %s", fromHeader, fromName, fromFile.getPath()));
            else if (fromName != null && fromHeader == null && fromName.version >= CommitLogDescriptor.VERSION_21)
                throw new IllegalStateException("Cannot safely construct descriptor for segment, as name descriptor implies a version that should contain a header descriptor, but that descriptor could not be read: " + fromFile.getPath());
            else if (fromHeader != null)
                descriptor = fromHeader;
            else descriptor = fromName;

            if (descriptor.version > CommitLogDescriptor.VERSION_21)
                throw new IllegalStateException("Unsupported commit log version: " + descriptor.version);

            File toFile = new File(DatabaseDescriptor.getCommitLogLocation(), descriptor.fileName());
            if (toFile.exists())
            {
                logger.debug("Skipping restore of archive {} as the segment already exists in the restore location {}",
                             fromFile.getPath(), toFile.getPath());
                continue;
            }

            String command = restoreCommand.replace("%from", fromFile.getPath());
            command = command.replace("%to", toFile.getPath());
            try
            {
                exec(command);
            }
            catch (IOException e)
            {
                throw new RuntimeException(e);
            }
        }
    }
}
 

/**
 * Constructs a new segment file.
 *
 * @param filePath  if not null, recycles the existing file by renaming it and truncating it to CommitLog.SEGMENT_SIZE.
 */
CommitLogSegment(String filePath)
{
    id = getNextId();
    descriptor = new CommitLogDescriptor(id);
    logFile = new File(DatabaseDescriptor.getCommitLogLocation(), descriptor.fileName());
    boolean isCreating = true;

    try
    {
        if (filePath != null)
        {
            File oldFile = new File(filePath);

            if (oldFile.exists())
            {
                logger.debug("Re-using discarded CommitLog segment for {} from {}", id, filePath);
                if (!oldFile.renameTo(logFile))
                    throw new IOException("Rename from " + filePath + " to " + id + " failed");
                isCreating = false;
            }
        }

        // Open the initial the segment file
        logFileAccessor = new RandomAccessFile(logFile, "rw");

        if (isCreating)
            logger.debug("Creating new commit log segment {}", logFile.getPath());

        // Map the segment, extending or truncating it to the standard segment size.
        // (We may have restarted after a segment size configuration change, leaving "incorrectly"
        // sized segments on disk.)
        logFileAccessor.setLength(DatabaseDescriptor.getCommitLogSegmentSize());
        fd = CLibrary.getfd(logFileAccessor.getFD());

        buffer = logFileAccessor.getChannel().map(FileChannel.MapMode.READ_WRITE, 0, DatabaseDescriptor.getCommitLogSegmentSize());
        // write the header
        CommitLogDescriptor.writeHeader(buffer, descriptor);
        // mark the initial sync marker as uninitialised
        buffer.putInt(CommitLogDescriptor.HEADER_SIZE, 0);
        buffer.putLong(CommitLogDescriptor.HEADER_SIZE + 4, 0);
        allocatePosition.set(CommitLogDescriptor.HEADER_SIZE + SYNC_MARKER_SIZE);
        lastSyncedOffset = CommitLogDescriptor.HEADER_SIZE;
    }
    catch (IOException e)
    {
        throw new FSWriteError(e, logFile);
    }
}
 

@Test
public void testNoCommitLog() throws Exception
{
    Mutation rm = new Mutation("Keyspace1", ByteBufferUtil.bytes("row"));
    rm.add("Standard1", Util.cellname("commit1"), ByteBufferUtil.bytes("abcd"), 0);
    rm.apply();

    rm = new Mutation("NoCommitlogSpace", ByteBufferUtil.bytes("row"));
    rm.add("Standard1", Util.cellname("commit2"), ByteBufferUtil.bytes("abcd"), 0);
    rm.apply();

    boolean commitLogMessageFound = false;
    boolean noCommitLogMessageFound = false;

    File commitLogDir = new File(DatabaseDescriptor.getCommitLogLocation());

    byte[] commitBytes = "commit".getBytes("UTF-8");

    for(String filename : commitLogDir.list())
    {
        BufferedInputStream is = null;
        try
        {
            is = new BufferedInputStream(new FileInputStream(commitLogDir.getAbsolutePath()+File.separator+filename));

            if (!isEmptyCommitLog(is))
            {
                while (findPatternInStream(commitBytes, is))
                {
                    char c = (char)is.read();

                    if (c == '1')
                        commitLogMessageFound = true;
                    else if (c == '2')
                        noCommitLogMessageFound = true;
                }
            }
        }
        finally
        {
            if (is != null)
                is.close();
        }
    }

    assertTrue(commitLogMessageFound);
    assertFalse(noCommitLogMessageFound);
}