Java Code Examples for org.apache.distributedlog.api.DistributedLogManager#getInputStream()

@Test(timeout = 15000)
public void testHandleInconsistentMetadataNonBlocking() throws Exception {
    String name = "distrlog-inconsistent-metadata-nonblocking-read";
    long numRecordsWritten = createStreamWithInconsistentMetadata(name);

    DistributedLogManager dlm = createNewDLM(conf, name);
    try {
        LogReader reader = dlm.getInputStream(45);
        long numRecordsRead = 0;
        long lastTxId = -1;
        while (numRecordsRead < (numRecordsWritten / 2)) {
            LogRecord record = reader.readNext(false);
            if (record != null) {
                DLMTestUtil.verifyLogRecord(record);
                Assert.assertTrue(lastTxId < record.getTransactionId());
                lastTxId = record.getTransactionId();
                numRecordsRead++;
            } else {
                Thread.sleep(1);
            }
        }
        reader.close();
    } finally {
        dlm.close();
    }
}
 

@Test(timeout = 15000)
public void testHandleInconsistentMetadataDLSNNonBlocking() throws Exception {
    String name = "distrlog-inconsistent-metadata-nonblocking-read-dlsn";
    long numRecordsWritten = createStreamWithInconsistentMetadata(name);

    DistributedLogManager dlm = createNewDLM(conf, name);
    try {
        LogReader reader = dlm.getInputStream(DLSN.InitialDLSN);
        long numRecordsRead = 0;
        long lastTxId = -1;
        while (numRecordsRead < numRecordsWritten) {
            LogRecord record = reader.readNext(false);
            if (record != null) {
                DLMTestUtil.verifyLogRecord(record);
                Assert.assertTrue(lastTxId < record.getTransactionId());
                lastTxId = record.getTransactionId();
                numRecordsRead++;
            } else {
                Thread.sleep(1);
            }
        }
        reader.close();
    } finally {
        dlm.close();
    }
}
 

@Test(timeout = 60000)
public void testToolTruncateStream() throws Exception {
    DistributedLogConfiguration confLocal = new DistributedLogConfiguration();
    confLocal.addConfiguration(conf);
    confLocal.setLogSegmentCacheEnabled(false);
    DistributedLogManager dlm = DLMTestUtil.createNewDLM("testToolTruncateStream", confLocal, defaultUri);
    DLMTestUtil.generateCompletedLogSegments(dlm, confLocal, 3, 1000);

    DLSN dlsn = new DLSN(2, 1, 0);
    TruncateStreamCommand cmd = new TruncateStreamCommand();
    cmd.setDlsn(dlsn);
    cmd.setUri(defaultUri);
    cmd.setStreamName("testToolTruncateStream");
    cmd.setForce(true);

    assertEquals(0, cmd.runCmd());

    LogReader reader = dlm.getInputStream(0);
    LogRecordWithDLSN record = reader.readNext(false);
    assertEquals(dlsn, record.getDlsn());

    reader.close();
    dlm.close();
}
 

private void verifyEntries(String name, long readFromTxId, long startTxId, int numEntries) throws Exception {
    DistributedLogManager dlm = createNewDLM(conf, name);
    LogReader reader = dlm.getInputStream(readFromTxId);

    long txid = startTxId;
    int numRead = 0;
    LogRecord r = reader.readNext(false);
    while (null != r) {
        DLMTestUtil.verifyLogRecord(r);
        assertEquals(txid++, r.getTransactionId());
        ++numRead;
        r = reader.readNext(false);
    }
    assertEquals(numEntries, numRead);
    reader.close();
    dlm.close();
}
 

/**
 * Construct ledger input stream.
 *
 * @param dlm the Distributed Log Manager to access the stream
 */
AppendOnlyStreamReader(DistributedLogManager dlm)
    throws IOException {
    this.dlm = dlm;
    reader = dlm.getInputStream(0);
    currentPosition = 0;
}
 

int countFromStartToEnd(DistributedLogManager dlm) throws Exception {
    int count = 0;
    try {
        LogReader reader = dlm.getInputStream(startDLSN);
        try {
            LogRecordWithDLSN record = reader.readNext(false);
            LogRecordWithDLSN preRecord = record;
            System.out.println("first record : " + record);
            while (null != record) {
                if (record.getDlsn().compareTo(endDLSN) > 0) {
                    break;
                }
                ++count;
                if (count % 1000 == 0) {
                    logger.info("read {} records from {}...", count, getStreamName());
                }
                preRecord = record;
                record = reader.readNext(false);
            }
            System.out.println("last record : " + preRecord);
        } finally {
            reader.close();
        }
    } finally {
        dlm.close();
    }
    return count;
}
 

static long getNumberofLogRecords(DistributedLogManager bkdlm, long startTxId) throws IOException {
    long numLogRecs = 0;
    LogReader reader = bkdlm.getInputStream(startTxId);
    LogRecord record = reader.readNext(false);
    while (null != record) {
        numLogRecs++;
        verifyLogRecord(record);
        record = reader.readNext(false);
    }
    reader.close();
    return numLogRecs;
}
 

@Test(timeout = 60000)
public void testReadingFromEmptyLog() throws Exception {
    String name = testName.getMethodName();
    DistributedLogConfiguration confLocal = new DistributedLogConfiguration();
    confLocal.addConfiguration(conf);
    confLocal.setOutputBufferSize(0);
    confLocal.setPeriodicFlushFrequencyMilliSeconds(Integer.MAX_VALUE);

    DistributedLogManager dlm = createNewDLM(confLocal, name);
    BKSyncLogWriter out = (BKSyncLogWriter) dlm.startLogSegmentNonPartitioned();
    // write a record but not commit
    LogRecord op = DLMTestUtil.getLogRecordInstance(1L);
    out.write(op);

    LogReader reader = dlm.getInputStream(1L);
    assertNull(reader.readNext(true));
    assertNull(reader.readNext(false));

    op = DLMTestUtil.getLogRecordInstance(2L);
    out.write(op);

    // reader is able to read first record
    LogRecord record = waitForNextRecord(reader);
    assertNotNull(record);
    assertEquals(1L, record.getTransactionId());
    DLMTestUtil.verifyLogRecord(record);

    assertNull(reader.readNext(true));

    out.close();
    reader.close();
    dlm.close();
}
 

@Test(timeout = 60000)
public void testHandleInconsistentMetadata() throws Exception {
    String name = "distrlog-inconsistent-metadata-blocking-read";
    long numRecordsWritten = createStreamWithInconsistentMetadata(name);

    DistributedLogManager dlm = createNewDLM(conf, name);
    try {
        LogReader reader = dlm.getInputStream(45);
        long numRecordsRead = 0;
        LogRecord record = reader.readNext(false);
        long lastTxId = -1;
        while (numRecordsRead < numRecordsWritten / 2) {
            if (null != record) {
                DLMTestUtil.verifyLogRecord(record);
                Assert.assertTrue(lastTxId < record.getTransactionId());
                lastTxId = record.getTransactionId();
                numRecordsRead++;
            } else {
                Thread.sleep(1);
            }
            record = reader.readNext(false);
        }
        reader.close();
        assertEquals(numRecordsWritten / 2, numRecordsRead);
    } finally {
        dlm.close();
    }
}
 

@Test(timeout = 60000)
public void testMultiReaders() throws Exception {
    String name = "distrlog-multireaders";
    final RateLimiter limiter = RateLimiter.create(1000);

    DistributedLogConfiguration confLocal = new DistributedLogConfiguration();
    confLocal.setOutputBufferSize(0);
    confLocal.setImmediateFlushEnabled(true);

    DistributedLogManager dlmwrite = createNewDLM(confLocal, name);

    final AsyncLogWriter writer = dlmwrite.startAsyncLogSegmentNonPartitioned();
    Utils.ioResult(writer.write(DLMTestUtil.getLogRecordInstance(0)));
    Utils.ioResult(writer.write(DLMTestUtil.getLogRecordInstance(1)));
    final AtomicInteger writeCount = new AtomicInteger(2);

    DistributedLogManager dlmread = createNewDLM(conf, name);

    BKSyncLogReader reader0 = (BKSyncLogReader) dlmread.getInputStream(0);

    try {
        ReaderThread[] readerThreads = new ReaderThread[1];
        readerThreads[0] = new ReaderThread("reader0-non-blocking", reader0, false);
        // readerThreads[1] = new ReaderThread("reader1-non-blocking", reader0, false);

        final AtomicBoolean running = new AtomicBoolean(true);
        Thread writerThread = new Thread("WriteThread") {
            @Override
            public void run() {
                try {
                    long txid = 2;
                    DLSN dlsn = DLSN.InvalidDLSN;
                    while (running.get()) {
                        limiter.acquire();
                        long curTxId = txid++;
                        dlsn = Utils.ioResult(writer.write(DLMTestUtil.getLogRecordInstance(curTxId)));
                        writeCount.incrementAndGet();
                        if (curTxId % 1000 == 0) {
                            LOG.info("writer write {}", curTxId);
                        }
                    }
                    LOG.info("Completed writing record at {}", dlsn);
                    Utils.close(writer);
                } catch (DLInterruptedException die) {
                    Thread.currentThread().interrupt();
                } catch (Exception e) {

                }
            }
        };

        for (ReaderThread rt : readerThreads) {
            rt.start();
        }

        writerThread.start();

        TimeUnit.SECONDS.sleep(5);

        LOG.info("Stopping writer");

        running.set(false);
        writerThread.join();

        LOG.info("Writer stopped after writing {} records, waiting for reader to complete",
                writeCount.get());
        while (writeCount.get() > (readerThreads[0].getReadCount())) {
            LOG.info("Write Count = {}, Read Count = {}",
                    new Object[] { writeCount.get(), readerThreads[0].getReadCount() });
            TimeUnit.MILLISECONDS.sleep(100);
        }
        assertEquals(writeCount.get(),
                (readerThreads[0].getReadCount()));

        for (ReaderThread readerThread : readerThreads) {
            readerThread.stopReading();
        }
    } finally {
        dlmwrite.close();
        reader0.close();
        dlmread.close();
    }
}
 

@Test(timeout = 60000)
public void testInterleavedReadersWithCleanup() throws Exception {
    String name = "distrlog-interleaved-cleanup";
    BKDistributedLogManager dlmwrite0 = createNewDLM(conf, name + "-0");
    BKDistributedLogManager dlmwrite1 = createNewDLM(conf, name + "-1");
    long txid = 1;
    Long retentionPeriodOverride = null;

    BKAsyncLogWriter writer0 = dlmwrite0.startAsyncLogSegmentNonPartitioned();
    BKAsyncLogWriter writer1 = dlmwrite1.startAsyncLogSegmentNonPartitioned();
    for (long j = 1; j <= 4; j++) {
        for (int k = 1; k <= 10; k++) {
            if (k == 5) {
                writer0.setForceRolling(true);
                writer0.overRideMinTimeStampToKeep(retentionPeriodOverride);
                writer1.setForceRolling(true);
                writer1.overRideMinTimeStampToKeep(retentionPeriodOverride);
            }
            DLSN dlsn1 = Utils.ioResult(writer1.write(DLMTestUtil.getLogRecordInstance(txid++)));
            LOG.info("writer1 write record {}", dlsn1);
            DLSN dlsn0 = Utils.ioResult(writer0.write(DLMTestUtil.getLogRecordInstance(txid++)));
            LOG.info("writer0 write record {}", dlsn0);
            if (k == 5) {
                writer0.setForceRolling(false);
                writer1.setForceRolling(false);
                retentionPeriodOverride = System.currentTimeMillis();
            }
            Thread.sleep(5);
        }
        Utils.ioResult(writer1.writeControlRecord(DLMTestUtil.getLogRecordInstance(txid - 1)));
        Utils.ioResult(writer0.writeControlRecord(DLMTestUtil.getLogRecordInstance(txid - 1)));
    }
    writer0.close();
    writer1.close();

    DistributedLogManager dlmreader0 = createNewDLM(conf, name + "-0");
    DistributedLogManager dlmreader1 = createNewDLM(conf, name + "-1");
    LogReader reader0 = dlmreader0.getInputStream(1);
    LogReader reader1 = dlmreader1.getInputStream(1);
    int numTrans = drainStreams(reader0, 15, reader1, 15);
    assertEquals(30, numTrans);
    reader0.close();
    reader1.close();
    dlmreader0.close();
    dlmwrite0.close();
    dlmreader1.close();
    dlmwrite1.close();
}
 

@Test(timeout = 60000)
public void testUnableToRollLogSegments() throws Exception {
    String name = "distrlog-unable-to-roll-log-segments";
    DistributedLogConfiguration confLocal = new DistributedLogConfiguration();
    confLocal.loadConf(conf);
    confLocal.setImmediateFlushEnabled(true);
    confLocal.setOutputBufferSize(0);
    confLocal.setLogSegmentRollingIntervalMinutes(0);
    confLocal.setMaxLogSegmentBytes(1);

    DistributedLogManager dlm = createNewDLM(confLocal, name);
    BKAsyncLogWriter writer = (BKAsyncLogWriter) dlm.startAsyncLogSegmentNonPartitioned();

    long txId = 1L;

    // Create Log Segments
    Utils.ioResult(writer.write(DLMTestUtil.getLogRecordInstance(txId)));

    FailpointUtils.setFailpoint(FailpointUtils.FailPointName.FP_StartLogSegmentBeforeLedgerCreate,
            FailpointUtils.FailPointActions.FailPointAction_Throw);

    try {
        // If we couldn't open new log segment, we should keep using the old one
        final int numRecords = 10;
        final CountDownLatch latch = new CountDownLatch(numRecords);
        for (int i = 0; i < numRecords; i++) {
            writer.write(DLMTestUtil.getLogRecordInstance(++txId)).whenComplete(new FutureEventListener<DLSN>() {
                @Override
                public void onSuccess(DLSN value) {
                    logger.info("Completed entry : {}.", value);
                    latch.countDown();
                }
                @Override
                public void onFailure(Throwable cause) {
                    logger.error("Failed to write entries : ", cause);
                }
            });
        }

        latch.await();

        writer.close();

        List<LogSegmentMetadata> segments = dlm.getLogSegments();
        logger.info("LogSegments: {}", segments);

        assertEquals(1, segments.size());

        long expectedTxID = 1L;
        LogReader reader = dlm.getInputStream(DLSN.InitialDLSN);
        LogRecordWithDLSN record = reader.readNext(false);
        while (null != record) {
            DLMTestUtil.verifyLogRecord(record);
            assertEquals(expectedTxID++, record.getTransactionId());
            assertEquals(record.getTransactionId() - 1, record.getSequenceId());

            record = reader.readNext(false);
        }

        assertEquals(12L, expectedTxID);

        reader.close();

        dlm.close();
    } finally {
        FailpointUtils.removeFailpoint(FailpointUtils.FailPointName.FP_StartLogSegmentBeforeLedgerCreate);
    }
}
 

@Test(timeout = 60000)
public void deleteDuringRead() throws Exception {
    String name = "distrlog-delete-with-reader";
    DistributedLogManager dlm = createNewDLM(conf, name);

    long txid = 1;
    for (long i = 0; i < 3; i++) {
        long start = txid;
        BKSyncLogWriter writer = (BKSyncLogWriter) dlm.startLogSegmentNonPartitioned();
        for (long j = 1; j <= DEFAULT_SEGMENT_SIZE; j++) {
            writer.write(DLMTestUtil.getLogRecordInstance(txid++));
        }

        BKLogSegmentWriter perStreamLogWriter = writer.getCachedLogWriter();

        writer.closeAndComplete();
        BKLogWriteHandler blplm = ((BKDistributedLogManager) (dlm)).createWriteHandler(true);
        assertNotNull(zkc.exists(blplm.completedLedgerZNode(start, txid - 1,
                perStreamLogWriter.getLogSegmentSequenceNumber()), false));
        Utils.ioResult(blplm.asyncClose());
    }

    LogReader reader = dlm.getInputStream(1);
    LogRecord record = reader.readNext(false);
    assert (null != record);
    DLMTestUtil.verifyLogRecord(record);
    long lastTxId = record.getTransactionId();

    dlm.delete();

    boolean exceptionEncountered;
    try {
        record = reader.readNext(false);
        while (null != record) {
            DLMTestUtil.verifyLogRecord(record);
            assert (lastTxId < record.getTransactionId());
            lastTxId = record.getTransactionId();
            record = reader.readNext(false);
        }
        // make sure the exception is thrown from readahead
        while (true) {
            reader.readNext(false);
        }
    } catch (LogReadException | LogNotFoundException | DLIllegalStateException e) {
        exceptionEncountered = true;
    }
    assertTrue(exceptionEncountered);
    reader.close();
}
 

/**
 * Construct DistributedLog input stream.
 *
 * @param dlm the Distributed Log Manager to access the stream
 */
public DLInputStream(DistributedLogManager dlm) throws IOException {
  this.dlm = dlm;
  reader = dlm.getInputStream(DLSN.InitialDLSN);
}
 

/**
 * Construct distributedlog input stream
 *
 * @param dlm the Distributed Log Manager to access the stream
 */
public DLInputStream(DistributedLogManager dlm) throws IOException {
  this.dlm = dlm;
  reader = dlm.getInputStream(DLSN.InitialDLSN);
}