Java Code Examples for org.apache.distributedlog.api.DistributedLogManager#getAsyncLogReader()

@Override
protected int runCmd() throws Exception {
    DistributedLogManager dlm = getNamespace().openLog(getStreamName());
    long totalCount = dlm.getLogRecordCount();
    try {
        AsyncLogReader reader;
        Object startOffset;
        try {
            DLSN lastDLSN = FutureUtils.result(dlm.getLastDLSNAsync());
            System.out.println("Last DLSN : " + lastDLSN);
            if (null == fromDLSN) {
                reader = dlm.getAsyncLogReader(fromTxnId);
                startOffset = fromTxnId;
            } else {
                reader = dlm.getAsyncLogReader(fromDLSN);
                startOffset = fromDLSN;
            }
        } catch (LogNotFoundException lee) {
            System.out.println("No stream found to dump records.");
            return 0;
        }
        try {
            System.out.println(String.format("Dump records for %s (from = %s, dump"
                    + " count = %d, total records = %d)", getStreamName(), startOffset, count, totalCount));

            dumpRecords(reader);
        } finally {
            Utils.close(reader);
        }
    } finally {
        dlm.close();
    }
    return 0;
}
 

void simpleAsyncReadTest(String name, DistributedLogConfiguration confLocal) throws Exception {
    confLocal.setOutputBufferSize(1024);
    confLocal.setReadAheadWaitTime(10);
    confLocal.setReadAheadBatchSize(10);
    DistributedLogManager dlm = createNewDLM(confLocal, name);

    int numLogSegments = 3;
    int numRecordsPerLogSegment = 10;

    // Write 30 records: 3 log segments, 10 records per log segment
    long txid = 1L;
    txid = writeRecords(dlm, numLogSegments, numRecordsPerLogSegment, txid, false);
    // Write another log segment with 5 records and flush every 2 records
    txid = writeLogSegment(dlm, 5, txid, 2, false);

    final AsyncLogReader reader = dlm.getAsyncLogReader(DLSN.InvalidDLSN);
    final CountDownLatch syncLatch = new CountDownLatch((int) (txid - 1));
    final CountDownLatch completionLatch = new CountDownLatch(1);
    final AtomicBoolean errorsFound = new AtomicBoolean(false);

    boolean monotonic = LogSegmentMetadata.supportsSequenceId(confLocal.getDLLedgerMetadataLayoutVersion());
    TestAsyncReaderWriter.readNext(
            reader,
            DLSN.InvalidDLSN,
            monotonic ? 0L : Long.MIN_VALUE,
            monotonic,
            syncLatch,
            completionLatch,
            errorsFound);

    completionLatch.await();
    assertFalse("Errors encountered on reading records", errorsFound.get());
    syncLatch.await();

    Utils.close(reader);
    dlm.close();
}
 

@Test(timeout = 60000)
public void testBulkAsyncReadWithWriteBatch() throws Exception {
    String name = "distrlog-bulkasyncread-with-writebatch";
    DistributedLogConfiguration confLocal = new DistributedLogConfiguration();
    confLocal.loadConf(conf);
    confLocal.setOutputBufferSize(1024000);
    confLocal.setReadAheadWaitTime(10);
    confLocal.setReadAheadMaxRecords(10000);
    confLocal.setReadAheadBatchSize(10);

    DistributedLogManager dlm = createNewDLM(confLocal, name);

    int numLogSegments = 3;
    int numRecordsPerLogSegment = 20;

    writeRecords(dlm, numLogSegments, numRecordsPerLogSegment, 1L, false);

    final AsyncLogReader reader = dlm.getAsyncLogReader(DLSN.InitialDLSN);
    int expectedTxID = 1;
    for (long i = 0; i < 3; i++) {
        // since we batched 20 entries into single bookkeeper entry
        // we should be able to read 20 entries as a batch.
        List<LogRecordWithDLSN> records = Utils.ioResult(reader.readBulk(20));
        assertEquals(20, records.size());
        for (LogRecordWithDLSN record : records) {
            assertEquals(expectedTxID, record.getTransactionId());
            ++expectedTxID;
        }
    }

    Utils.close(reader);
    dlm.close();
}
 

@Test(timeout = 60000)
public void testAsyncReadEmptyRecords() throws Exception {
    String name = "distrlog-simpleasyncreadempty";
    DistributedLogConfiguration confLocal = new DistributedLogConfiguration();
    confLocal.loadConf(testConf);
    confLocal.setOutputBufferSize(0);
    confLocal.setReadAheadWaitTime(10);
    confLocal.setReadAheadBatchSize(10);
    DistributedLogManager dlm = createNewDLM(confLocal, name);

    int numLogSegments = 3;
    int numRecordsPerLogSegment = 10;

    long txid = 1L;
    // write 3 log segments, 10 records per log segment
    txid = writeRecords(dlm, numLogSegments, numRecordsPerLogSegment, txid, true);
    // write another log segment with 5 records and flush every 2 records
    txid = writeLogSegment(dlm, 5, txid, 2, true);

    AsyncLogReader asyncReader = dlm.getAsyncLogReader(DLSN.InvalidDLSN);
    assertEquals("Expected stream name = " + name + " but " + asyncReader.getStreamName() + " found",
            name, asyncReader.getStreamName());
    long numTrans = 0;
    DLSN lastDLSN = DLSN.InvalidDLSN;
    LogRecordWithDLSN record = Utils.ioResult(asyncReader.readNext());
    while (null != record) {
        DLMTestUtil.verifyEmptyLogRecord(record);
        assertEquals(0, record.getDlsn().getSlotId());
        assertTrue(record.getDlsn().compareTo(lastDLSN) > 0);
        lastDLSN = record.getDlsn();
        numTrans++;
        if (numTrans >= (txid - 1)) {
            break;
        }
        record = Utils.ioResult(asyncReader.readNext());
    }
    assertEquals((txid - 1), numTrans);
    Utils.close(asyncReader);
    dlm.close();
}
 

@Test(timeout = 60000)
public void testReadBrokenEntries() throws Exception {
    String name = runtime.getMethodName();
    DistributedLogConfiguration confLocal = new DistributedLogConfiguration();
    confLocal.loadConf(testConf);

    confLocal.setOutputBufferSize(0);
    confLocal.setPeriodicFlushFrequencyMilliSeconds(0);
    confLocal.setImmediateFlushEnabled(true);
    confLocal.setReadAheadWaitTime(10);
    confLocal.setReadAheadBatchSize(1);
    confLocal.setPositionGapDetectionEnabled(false);
    confLocal.setReadAheadSkipBrokenEntries(true);
    confLocal.setEIInjectReadAheadBrokenEntries(true);
    DistributedLogManager dlm = createNewDLM(confLocal, name);

    int numLogSegments = 3;
    int numRecordsPerLogSegment = 10;

    long txid = 1L;
    txid = writeRecords(dlm, numLogSegments, numRecordsPerLogSegment, txid, false);

    AsyncLogReader reader = dlm.getAsyncLogReader(DLSN.InvalidDLSN);

    // 3 segments, 10 records each, immediate flush, batch size 1, so just the first
    // record in each ledger is discarded, for 30 - 3 = 27 records.
    for (int i = 0; i < 27; i++) {
        LogRecordWithDLSN record = Utils.ioResult(reader.readNext());
        assertFalse(record.getDlsn().getEntryId() % 10 == 0);
    }

    Utils.close(reader);
    dlm.close();
}
 

@Test(timeout = 60000)
public void testReadBrokenEntriesWithGapDetection() throws Exception {
    String name = runtime.getMethodName();
    DistributedLogConfiguration confLocal = new DistributedLogConfiguration();
    confLocal.loadConf(testConf);

    confLocal.setOutputBufferSize(0);
    confLocal.setPeriodicFlushFrequencyMilliSeconds(0);
    confLocal.setImmediateFlushEnabled(true);
    confLocal.setReadAheadWaitTime(10);
    confLocal.setReadAheadBatchSize(1);
    confLocal.setPositionGapDetectionEnabled(true);
    confLocal.setReadAheadSkipBrokenEntries(true);
    confLocal.setEIInjectReadAheadBrokenEntries(true);
    DistributedLogManager dlm = createNewDLM(confLocal, name);

    int numLogSegments = 1;
    int numRecordsPerLogSegment = 100;

    long txid = 1L;
    txid = writeRecords(dlm, numLogSegments, numRecordsPerLogSegment, txid, false);

    AsyncLogReader reader = dlm.getAsyncLogReader(DLSN.InvalidDLSN);

    try {
        // 3 segments, 10 records each, immediate flush, batch size 1, so just the first
        // record in each ledger is discarded, for 30 - 3 = 27 records.
        for (int i = 0; i < 30; i++) {
            LogRecordWithDLSN record = Utils.ioResult(reader.readNext());
            assertFalse(record.getDlsn().getEntryId() % 10 == 0);
        }
        fail("should have thrown");
    } catch (DLIllegalStateException e) {
    }

    Utils.close(reader);
    dlm.close();
}
 

@Test(timeout = 60000)
public void testReadBrokenEntriesAndLargeBatchSize() throws Exception {
    String name = runtime.getMethodName();
    DistributedLogConfiguration confLocal = new DistributedLogConfiguration();
    confLocal.loadConf(testConf);

    confLocal.setOutputBufferSize(0);
    confLocal.setPeriodicFlushFrequencyMilliSeconds(0);
    confLocal.setImmediateFlushEnabled(true);
    confLocal.setReadAheadWaitTime(10);
    confLocal.setReadAheadBatchSize(5);
    confLocal.setPositionGapDetectionEnabled(false);
    confLocal.setReadAheadSkipBrokenEntries(true);
    confLocal.setEIInjectReadAheadBrokenEntries(true);
    DistributedLogManager dlm = createNewDLM(confLocal, name);

    int numLogSegments = 1;
    int numRecordsPerLogSegment = 100;

    long txid = 1L;
    txid = writeRecords(dlm, numLogSegments, numRecordsPerLogSegment, txid, false);

    AsyncLogReader reader = dlm.getAsyncLogReader(DLSN.InvalidDLSN);

    // Every 10th record broken. Reading 5 at once, beginning from 0:
    // 1. range 0-4 will be corrupted and discarded
    // 2. ranges 1-5, 2-6, 3-7, 4-8, 5-9 will be ok
    // 3. ranges 6-10, 7-11, 8-12, 9-13 will be bad
    // And so on, so 5 records in each 10 will be discarded, for 50 good records.
    for (int i = 0; i < 50; i++) {
        LogRecordWithDLSN record = Utils.ioResult(reader.readNext());
        assertFalse(record.getDlsn().getEntryId() % 10 == 0);
    }

    Utils.close(reader);
    dlm.close();
}
 

@Test(timeout = 60000)
public void testReadBrokenEntriesAndLargeBatchSizeCrossSegment() throws Exception {
    String name = runtime.getMethodName();
    DistributedLogConfiguration confLocal = new DistributedLogConfiguration();
    confLocal.loadConf(testConf);

    confLocal.setOutputBufferSize(0);
    confLocal.setPeriodicFlushFrequencyMilliSeconds(0);
    confLocal.setImmediateFlushEnabled(true);
    confLocal.setReadAheadWaitTime(10);
    confLocal.setReadAheadBatchSize(8);
    confLocal.setPositionGapDetectionEnabled(false);
    confLocal.setReadAheadSkipBrokenEntries(true);
    confLocal.setEIInjectReadAheadBrokenEntries(true);
    DistributedLogManager dlm = createNewDLM(confLocal, name);

    int numLogSegments = 3;
    int numRecordsPerLogSegment = 5;

    long txid = 1L;
    txid = writeRecords(dlm, numLogSegments, numRecordsPerLogSegment, txid, false);

    AsyncLogReader reader = dlm.getAsyncLogReader(DLSN.InvalidDLSN);

    // Every 10th record broken. Reading 8 at once, beginning from 0:
    // 1. range 0-7 will be corrupted and discarded
    // 2. range 1-8 will be good, but only contain 4 records
    // And so on for the next segment, so 4 records in each segment, for 12 good records
    for (int i = 0; i < 12; i++) {
        LogRecordWithDLSN record = Utils.ioResult(reader.readNext());
        assertFalse(record.getDlsn().getEntryId() % 10 == 0);
    }

    Utils.close(reader);
    dlm.close();
}
 

@Test(timeout = 60000)
public void testBulkAsyncRead() throws Exception {
    String name = "distrlog-bulkasyncread";
    DistributedLogConfiguration confLocal = new DistributedLogConfiguration();
    confLocal.loadConf(conf);
    confLocal.setOutputBufferSize(0);
    confLocal.setImmediateFlushEnabled(true);
    confLocal.setReadAheadWaitTime(10);
    confLocal.setReadAheadMaxRecords(10000);
    confLocal.setReadAheadBatchSize(10);

    int numLogSegments = 3;
    int numRecordsPerLogSegment = 20;

    DistributedLogManager dlm = createNewDLM(confLocal, name);
    writeRecords(dlm, numLogSegments, numRecordsPerLogSegment, 1L, false);

    final AsyncLogReader reader = dlm.getAsyncLogReader(DLSN.InitialDLSN);
    int expectedTxID = 1;
    int numReads = 0;
    while (expectedTxID <= numLogSegments * numRecordsPerLogSegment) {
        if (expectedTxID == numLogSegments * numRecordsPerLogSegment) {
            break;
        }
        List<LogRecordWithDLSN> records = Utils.ioResult(reader.readBulk(20));
        LOG.info("Bulk read {} entries.", records.size());

        assertTrue(records.size() >= 1);
        for (LogRecordWithDLSN record : records) {
            assertEquals(expectedTxID, record.getTransactionId());
            ++expectedTxID;
        }
        ++numReads;
    }

    // we expect bulk read works
    assertTrue(numReads < 60);

    Utils.close(reader);
    dlm.close();
}
 

/**
 * Test Async Read by positioning to a given position in the log.
 * @throws Exception
 */
@Test(timeout = 60000)
public void testSimpleAsyncReadPosition() throws Exception {
    String name = runtime.getMethodName();
    DistributedLogConfiguration confLocal = new DistributedLogConfiguration();
    confLocal.loadConf(testConf);
    confLocal.setOutputBufferSize(1024);
    confLocal.setReadAheadWaitTime(10);
    confLocal.setReadAheadBatchSize(10);
    DistributedLogManager dlm = createNewDLM(confLocal, name);

    int numLogSegments = 3;
    int numRecordsPerLogSegment = 10;

    long txid = 1L;
    // write 3 log segments, 10 records per log segment
    txid = writeRecords(dlm, numLogSegments, numRecordsPerLogSegment, txid, false);
    // write another log segment with 5 records
    txid = writeLogSegment(dlm, 5, txid, Integer.MAX_VALUE, false);

    final CountDownLatch syncLatch = new CountDownLatch((int) (txid - 14));
    final CountDownLatch doneLatch = new CountDownLatch(1);
    final AtomicBoolean errorsFound = new AtomicBoolean(false);
    final AsyncLogReader reader = dlm.getAsyncLogReader(new DLSN(2, 2, 4));
    assertEquals(name, reader.getStreamName());

    boolean monotonic = LogSegmentMetadata.supportsSequenceId(confLocal.getDLLedgerMetadataLayoutVersion());
    TestAsyncReaderWriter.readNext(
            reader,
            new DLSN(2, 3, 0),
            monotonic ? 13L : Long.MIN_VALUE,
            monotonic,
            syncLatch,
            doneLatch,
            errorsFound);

    doneLatch.await();
    assertFalse("Errors found on reading records", errorsFound.get());
    syncLatch.await();

    Utils.close(reader);
    dlm.close();
}
 

void testSimpleAsyncReadWriteInternal(String name, boolean immediateFlush,
                                      int logSegmentVersion) throws Exception {
    DistributedLogConfiguration confLocal = new DistributedLogConfiguration();
    confLocal.loadConf(testConf);
    confLocal.setReadAheadWaitTime(10);
    confLocal.setReadAheadBatchSize(10);
    confLocal.setOutputBufferSize(1024);
    confLocal.setDLLedgerMetadataLayoutVersion(logSegmentVersion);
    confLocal.setImmediateFlushEnabled(immediateFlush);
    DistributedLogManager dlm = createNewDLM(confLocal, name);

    int numLogSegments = 3;
    int numRecordsPerLogSegment = 10;

    final CountDownLatch readLatch = new CountDownLatch(numLogSegments * numRecordsPerLogSegment);
    final CountDownLatch readDoneLatch = new CountDownLatch(1);
    final AtomicBoolean readErrors = new AtomicBoolean(false);
    final CountDownLatch writeLatch = new CountDownLatch(numLogSegments * numRecordsPerLogSegment);
    final AtomicBoolean writeErrors = new AtomicBoolean(false);
    final AsyncLogReader reader = dlm.getAsyncLogReader(DLSN.InvalidDLSN);
    assertEquals(name, reader.getStreamName());

    int txid = 1;
    for (long i = 0; i < 3; i++) {
        final long currentLogSegmentSeqNo = i + 1;
        BKAsyncLogWriter writer = (BKAsyncLogWriter) (dlm.startAsyncLogSegmentNonPartitioned());
        for (long j = 0; j < 10; j++) {
            final long currentEntryId = j;
            final LogRecord record = DLMTestUtil.getLargeLogRecordInstance(txid++);
            CompletableFuture<DLSN> dlsnFuture = writer.write(record);
            dlsnFuture.whenComplete(new WriteFutureEventListener(
                    record, currentLogSegmentSeqNo, currentEntryId, writeLatch, writeErrors, true));
            if (i == 0 && j == 0) {
                boolean monotonic = LogSegmentMetadata.supportsSequenceId(logSegmentVersion);
                TestAsyncReaderWriter.readNext(
                        reader,
                        DLSN.InvalidDLSN,
                        monotonic ? 0L : Long.MIN_VALUE,
                        monotonic,
                        readLatch,
                        readDoneLatch,
                        readErrors);
            }
        }
        writer.closeAndComplete();
    }

    writeLatch.await();
    assertFalse("All writes should succeed", writeErrors.get());

    readDoneLatch.await();
    assertFalse("All reads should succeed", readErrors.get());
    readLatch.await();

    Utils.close(reader);
    dlm.close();
}
 

@Test(timeout = 60000)
public void testSimpleAsyncReadWritePiggyBack() throws Exception {
    String name = runtime.getMethodName();

    DistributedLogConfiguration confLocal = new DistributedLogConfiguration();
    confLocal.loadConf(testConf);
    confLocal.setEnableReadAhead(true);
    confLocal.setReadAheadWaitTime(500);
    confLocal.setReadAheadBatchSize(10);
    confLocal.setReadAheadMaxRecords(100);
    confLocal.setOutputBufferSize(1024);
    confLocal.setPeriodicFlushFrequencyMilliSeconds(100);
    DistributedLogManager dlm = createNewDLM(confLocal, name);

    final AsyncLogReader reader = dlm.getAsyncLogReader(DLSN.InvalidDLSN);

    int numLogSegments = 3;
    int numRecordsPerLogSegment = 10;

    final CountDownLatch readLatch = new CountDownLatch(30);
    final CountDownLatch readDoneLatch = new CountDownLatch(1);
    final AtomicBoolean readErrors = new AtomicBoolean(false);
    final CountDownLatch writeLatch = new CountDownLatch(30);
    final AtomicBoolean writeErrors = new AtomicBoolean(false);

    int txid = 1;
    for (long i = 0; i < numLogSegments; i++) {
        final long currentLogSegmentSeqNo = i + 1;
        BKAsyncLogWriter writer = (BKAsyncLogWriter) (dlm.startAsyncLogSegmentNonPartitioned());
        for (long j = 0; j < numRecordsPerLogSegment; j++) {
            Thread.sleep(50);
            final LogRecord record = DLMTestUtil.getLargeLogRecordInstance(txid++);
            CompletableFuture<DLSN> dlsnFuture = writer.write(record);
            dlsnFuture.whenComplete(new WriteFutureEventListener(
                    record, currentLogSegmentSeqNo, j, writeLatch, writeErrors, false));
            if (i == 0 && j == 0) {
                boolean monotonic =
                        LogSegmentMetadata.supportsSequenceId(confLocal.getDLLedgerMetadataLayoutVersion());
                TestAsyncReaderWriter.readNext(
                        reader,
                        DLSN.InvalidDLSN,
                        monotonic ? 0L : Long.MIN_VALUE,
                        monotonic,
                        readLatch,
                        readDoneLatch,
                        readErrors);
            }
        }
        writer.closeAndComplete();
    }

    writeLatch.await();
    assertFalse("All writes should succeed", writeErrors.get());

    readDoneLatch.await();
    assertFalse("All reads should succeed", readErrors.get());
    readLatch.await();

    Utils.close(reader);
    dlm.close();
}
 

@Test(timeout = 60000)
public void testMaxReadAheadRecords() throws Exception {
    int maxRecords = 1;
    int batchSize = 8;
    int maxAllowedCachedRecords = maxRecords + batchSize - 1;

    String name = runtime.getMethodName();
    DistributedLogConfiguration confLocal = new DistributedLogConfiguration();
    confLocal.addConfiguration(testConf);
    confLocal.setOutputBufferSize(0);
    confLocal.setImmediateFlushEnabled(false);
    confLocal.setPeriodicFlushFrequencyMilliSeconds(Integer.MAX_VALUE);
    confLocal.setReadAheadMaxRecords(maxRecords);
    confLocal.setReadAheadBatchSize(batchSize);

    DistributedLogManager dlm = createNewDLM(confLocal, name);
    AsyncLogWriter writer = dlm.startAsyncLogSegmentNonPartitioned();

    int numRecords = 40;
    for (int i = 1; i <= numRecords; i++) {
        Utils.ioResult(writer.write(DLMTestUtil.getLogRecordInstance(i)));
        assertEquals("last tx id should become " + i,
                i, writer.getLastTxId());
    }
    LogRecord record = DLMTestUtil.getLogRecordInstance(numRecords);
    record.setControl();
    Utils.ioResult(writer.write(record));

    BKAsyncLogReader reader = (BKAsyncLogReader) dlm.getAsyncLogReader(DLSN.InitialDLSN);
    record = Utils.ioResult(reader.readNext());
    LOG.info("Read record {}", record);
    assertEquals(1L, record.getTransactionId());

    assertNotNull(reader.getReadAheadReader());
    assertTrue(reader.getReadAheadReader().getNumCachedEntries() <= maxAllowedCachedRecords);

    for (int i = 2; i <= numRecords; i++) {
        record = Utils.ioResult(reader.readNext());
        LOG.info("Read record {}", record);
        assertEquals((long) i, record.getTransactionId());
        TimeUnit.MILLISECONDS.sleep(20);
        int numCachedEntries = reader.getReadAheadReader().getNumCachedEntries();
        assertTrue("Should cache less than " + batchSize + " records but already found "
                + numCachedEntries + " records when reading " + i + "th record",
                numCachedEntries <= maxAllowedCachedRecords);
    }
    Utils.close(reader);
}