Java Code Examples for org.apache.distributedlog.api.LogReader#readNext()

The following examples show how to use org.apache.distributedlog.api.LogReader#readNext() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: TestNonBlockingReads.java    From distributedlog with Apache License 2.0 6 votes vote down vote up 
@Test(timeout = 15000)
public void testHandleInconsistentMetadataNonBlocking() throws Exception {
    String name = "distrlog-inconsistent-metadata-nonblocking-read";
    long numRecordsWritten = createStreamWithInconsistentMetadata(name);

    DistributedLogManager dlm = createNewDLM(conf, name);
    try {
        LogReader reader = dlm.getInputStream(45);
        long numRecordsRead = 0;
        long lastTxId = -1;
        while (numRecordsRead < (numRecordsWritten / 2)) {
            LogRecord record = reader.readNext(false);
            if (record != null) {
                DLMTestUtil.verifyLogRecord(record);
                Assert.assertTrue(lastTxId < record.getTransactionId());
                lastTxId = record.getTransactionId();
                numRecordsRead++;
            } else {
                Thread.sleep(1);
            }
        }
        reader.close();
    } finally {
        dlm.close();
    }
}
Example 2
Source File: TestNonBlockingReads.java    From distributedlog with Apache License 2.0 6 votes vote down vote up 
@Test(timeout = 15000)
public void testHandleInconsistentMetadataDLSNNonBlocking() throws Exception {
    String name = "distrlog-inconsistent-metadata-nonblocking-read-dlsn";
    long numRecordsWritten = createStreamWithInconsistentMetadata(name);

    DistributedLogManager dlm = createNewDLM(conf, name);
    try {
        LogReader reader = dlm.getInputStream(DLSN.InitialDLSN);
        long numRecordsRead = 0;
        long lastTxId = -1;
        while (numRecordsRead < numRecordsWritten) {
            LogRecord record = reader.readNext(false);
            if (record != null) {
                DLMTestUtil.verifyLogRecord(record);
                Assert.assertTrue(lastTxId < record.getTransactionId());
                lastTxId = record.getTransactionId();
                numRecordsRead++;
            } else {
                Thread.sleep(1);
            }
        }
        reader.close();
    } finally {
        dlm.close();
    }
}
Example 3
Source File: TestDistributedLogTool.java    From distributedlog with Apache License 2.0 6 votes vote down vote up 
@Test(timeout = 60000)
public void testToolTruncateStream() throws Exception {
    DistributedLogConfiguration confLocal = new DistributedLogConfiguration();
    confLocal.addConfiguration(conf);
    confLocal.setLogSegmentCacheEnabled(false);
    DistributedLogManager dlm = DLMTestUtil.createNewDLM("testToolTruncateStream", confLocal, defaultUri);
    DLMTestUtil.generateCompletedLogSegments(dlm, confLocal, 3, 1000);

    DLSN dlsn = new DLSN(2, 1, 0);
    TruncateStreamCommand cmd = new TruncateStreamCommand();
    cmd.setDlsn(dlsn);
    cmd.setUri(defaultUri);
    cmd.setStreamName("testToolTruncateStream");
    cmd.setForce(true);

    assertEquals(0, cmd.runCmd());

    LogReader reader = dlm.getInputStream(0);
    LogRecordWithDLSN record = reader.readNext(false);
    assertEquals(dlsn, record.getDlsn());

    reader.close();
    dlm.close();
}
Example 4
Source File: TestTruncate.java    From distributedlog with Apache License 2.0 6 votes vote down vote up 
private void verifyEntries(String name, long readFromTxId, long startTxId, int numEntries) throws Exception {
    DistributedLogManager dlm = createNewDLM(conf, name);
    LogReader reader = dlm.getInputStream(readFromTxId);

    long txid = startTxId;
    int numRead = 0;
    LogRecord r = reader.readNext(false);
    while (null != r) {
        DLMTestUtil.verifyLogRecord(r);
        assertEquals(txid++, r.getTransactionId());
        ++numRead;
        r = reader.readNext(false);
    }
    assertEquals(numEntries, numRead);
    reader.close();
    dlm.close();
}
Example 5
Source File: AppendOnlyStreamReader.java    From distributedlog with Apache License 2.0 5 votes vote down vote up 
private static LogRecordWithInputStream nextLogRecord(LogReader reader) throws IOException {
    LogRecordWithDLSN record = reader.readNext(false);

    if (null != record) {
        return new LogRecordWithInputStream(record);
    } else {
        record = reader.readNext(false);
        if (null != record) {
            return new LogRecordWithInputStream(record);
        } else {
            LOG.debug("No record");
            return null;
        }
    }
}
Example 6
Source File: DistributedLogTool.java    From distributedlog with Apache License 2.0 5 votes vote down vote up 
int countFromStartToEnd(DistributedLogManager dlm) throws Exception {
    int count = 0;
    try {
        LogReader reader = dlm.getInputStream(startDLSN);
        try {
            LogRecordWithDLSN record = reader.readNext(false);
            LogRecordWithDLSN preRecord = record;
            System.out.println("first record : " + record);
            while (null != record) {
                if (record.getDlsn().compareTo(endDLSN) > 0) {
                    break;
                }
                ++count;
                if (count % 1000 == 0) {
                    logger.info("read {} records from {}...", count, getStreamName());
                }
                preRecord = record;
                record = reader.readNext(false);
            }
            System.out.println("last record : " + preRecord);
        } finally {
            reader.close();
        }
    } finally {
        dlm.close();
    }
    return count;
}
Example 7
Source File: TestInterleavedReaders.java    From distributedlog with Apache License 2.0 5 votes vote down vote up 
private int drainStreams(LogReader reader0, int num0, LogReader reader1, int num1)
        throws Exception {
    // Allow time for watches to fire
    Thread.sleep(15);
    int numTrans = 0;
    LogRecord record;
    int i = 0;
    while (i < num0) {
        record = reader0.readNext(false);
        if (null != record) {
            assertTrue((record.getTransactionId() % 2 == 0));
            DLMTestUtil.verifyLogRecord(record);
            numTrans++;
            i++;
            LOG.info("Read record {}", record);
        }
    }
    i = 0;
    while (i < num1) {
        record = reader1.readNext(false);
        if (null != record) {
            assertTrue((record.getTransactionId() % 2 == 1));
            DLMTestUtil.verifyLogRecord(record);
            numTrans++;
            i++;
            LOG.info("Read record {}", record);
        }
    }
    return numTrans;
}
Example 8
Source File: TestDistributedLogBase.java    From distributedlog with Apache License 2.0 5 votes vote down vote up 
protected LogRecord waitForNextRecord(LogReader reader) throws Exception {
    LogRecord record = reader.readNext(false);
    while (null == record) {
        record = reader.readNext(false);
    }
    return record;
}
Example 9
Source File: DLMTestUtil.java    From distributedlog with Apache License 2.0 5 votes vote down vote up 
static long getNumberofLogRecords(DistributedLogManager bkdlm, long startTxId) throws IOException {
    long numLogRecs = 0;
    LogReader reader = bkdlm.getInputStream(startTxId);
    LogRecord record = reader.readNext(false);
    while (null != record) {
        numLogRecs++;
        verifyLogRecord(record);
        record = reader.readNext(false);
    }
    reader.close();
    return numLogRecs;
}
Example 10
Source File: TestNonBlockingReads.java    From distributedlog with Apache License 2.0 5 votes vote down vote up 
@Test(timeout = 60000)
public void testHandleInconsistentMetadata() throws Exception {
    String name = "distrlog-inconsistent-metadata-blocking-read";
    long numRecordsWritten = createStreamWithInconsistentMetadata(name);

    DistributedLogManager dlm = createNewDLM(conf, name);
    try {
        LogReader reader = dlm.getInputStream(45);
        long numRecordsRead = 0;
        LogRecord record = reader.readNext(false);
        long lastTxId = -1;
        while (numRecordsRead < numRecordsWritten / 2) {
            if (null != record) {
                DLMTestUtil.verifyLogRecord(record);
                Assert.assertTrue(lastTxId < record.getTransactionId());
                lastTxId = record.getTransactionId();
                numRecordsRead++;
            } else {
                Thread.sleep(1);
            }
            record = reader.readNext(false);
        }
        reader.close();
        assertEquals(numRecordsWritten / 2, numRecordsRead);
    } finally {
        dlm.close();
    }
}
Example 11
Source File: DLInputStream.java    From distributedlog with Apache License 2.0 5 votes vote down vote up 
private static RecordStream nextRecordStream(LogReader reader) throws IOException {
    LogRecordWithDLSN record = reader.readNext(false);
    if (null != record) {
        return new RecordStream(record);
    }
    return null;
}
Example 12
Source File: DLInputStream.java    From pulsar with Apache License 2.0 5 votes vote down vote up 
private static LogRecordWithInputStream nextLogRecord(LogReader reader) throws IOException {
  LogRecordWithDLSN record = reader.readNext(false);

  if (null != record) {
    return new LogRecordWithInputStream(record);
  } else {
    record = reader.readNext(false);
    if (null != record) {
      return new LogRecordWithInputStream(record);
    } else {
      return null;
    }
  }
}
Example 13
Source File: DLInputStream.java    From incubator-heron with Apache License 2.0 5 votes vote down vote up 
private static LogRecordWithInputStream nextLogRecord(LogReader reader) throws IOException {
  LogRecordWithDLSN record = reader.readNext(false);

  if (null != record) {
    return new LogRecordWithInputStream(record);
  } else {
    record = reader.readNext(false);
    if (null != record) {
      return new LogRecordWithInputStream(record);
    } else {
      return null;
    }
  }
}
Example 14
Source File: TestRollLogSegments.java    From distributedlog with Apache License 2.0 4 votes vote down vote up 
@Test(timeout = 60000)
public void testUnableToRollLogSegments() throws Exception {
    String name = "distrlog-unable-to-roll-log-segments";
    DistributedLogConfiguration confLocal = new DistributedLogConfiguration();
    confLocal.loadConf(conf);
    confLocal.setImmediateFlushEnabled(true);
    confLocal.setOutputBufferSize(0);
    confLocal.setLogSegmentRollingIntervalMinutes(0);
    confLocal.setMaxLogSegmentBytes(1);

    DistributedLogManager dlm = createNewDLM(confLocal, name);
    BKAsyncLogWriter writer = (BKAsyncLogWriter) dlm.startAsyncLogSegmentNonPartitioned();

    long txId = 1L;

    // Create Log Segments
    Utils.ioResult(writer.write(DLMTestUtil.getLogRecordInstance(txId)));

    FailpointUtils.setFailpoint(FailpointUtils.FailPointName.FP_StartLogSegmentBeforeLedgerCreate,
            FailpointUtils.FailPointActions.FailPointAction_Throw);

    try {
        // If we couldn't open new log segment, we should keep using the old one
        final int numRecords = 10;
        final CountDownLatch latch = new CountDownLatch(numRecords);
        for (int i = 0; i < numRecords; i++) {
            writer.write(DLMTestUtil.getLogRecordInstance(++txId)).whenComplete(new FutureEventListener<DLSN>() {
                @Override
                public void onSuccess(DLSN value) {
                    logger.info("Completed entry : {}.", value);
                    latch.countDown();
                }
                @Override
                public void onFailure(Throwable cause) {
                    logger.error("Failed to write entries : ", cause);
                }
            });
        }

        latch.await();

        writer.close();

        List<LogSegmentMetadata> segments = dlm.getLogSegments();
        logger.info("LogSegments: {}", segments);

        assertEquals(1, segments.size());

        long expectedTxID = 1L;
        LogReader reader = dlm.getInputStream(DLSN.InitialDLSN);
        LogRecordWithDLSN record = reader.readNext(false);
        while (null != record) {
            DLMTestUtil.verifyLogRecord(record);
            assertEquals(expectedTxID++, record.getTransactionId());
            assertEquals(record.getTransactionId() - 1, record.getSequenceId());

            record = reader.readNext(false);
        }

        assertEquals(12L, expectedTxID);

        reader.close();

        dlm.close();
    } finally {
        FailpointUtils.removeFailpoint(FailpointUtils.FailPointName.FP_StartLogSegmentBeforeLedgerCreate);
    }
}
Example 15
Source File: TestBKDistributedLogManager.java    From distributedlog with Apache License 2.0 4 votes vote down vote up 
@Test(timeout = 60000)
public void deleteDuringRead() throws Exception {
    String name = "distrlog-delete-with-reader";
    DistributedLogManager dlm = createNewDLM(conf, name);

    long txid = 1;
    for (long i = 0; i < 3; i++) {
        long start = txid;
        BKSyncLogWriter writer = (BKSyncLogWriter) dlm.startLogSegmentNonPartitioned();
        for (long j = 1; j <= DEFAULT_SEGMENT_SIZE; j++) {
            writer.write(DLMTestUtil.getLogRecordInstance(txid++));
        }

        BKLogSegmentWriter perStreamLogWriter = writer.getCachedLogWriter();

        writer.closeAndComplete();
        BKLogWriteHandler blplm = ((BKDistributedLogManager) (dlm)).createWriteHandler(true);
        assertNotNull(zkc.exists(blplm.completedLedgerZNode(start, txid - 1,
                perStreamLogWriter.getLogSegmentSequenceNumber()), false));
        Utils.ioResult(blplm.asyncClose());
    }

    LogReader reader = dlm.getInputStream(1);
    LogRecord record = reader.readNext(false);
    assert (null != record);
    DLMTestUtil.verifyLogRecord(record);
    long lastTxId = record.getTransactionId();

    dlm.delete();

    boolean exceptionEncountered;
    try {
        record = reader.readNext(false);
        while (null != record) {
            DLMTestUtil.verifyLogRecord(record);
            assert (lastTxId < record.getTransactionId());
            lastTxId = record.getTransactionId();
            record = reader.readNext(false);
        }
        // make sure the exception is thrown from readahead
        while (true) {
            reader.readNext(false);
        }
    } catch (LogReadException | LogNotFoundException | DLIllegalStateException e) {
        exceptionEncountered = true;
    }
    assertTrue(exceptionEncountered);
    reader.close();
}