Java Code Examples for org.apache.distributedlog.api.DistributedLogManager#getLogSegments()

boolean processLog(String logName) throws Exception {
    DistributedLogManager dlm = getNamespace().openLog(logName);
    try {
        List<LogSegmentMetadata> segments = dlm.getLogSegments();
        for (LogSegmentMetadata segment : segments) {
            if (getLedgerID() == segment.getLogSegmentId()) {
                System.out.println("Found ledger " + getLedgerID() + " at log segment "
                        + segment + " for stream '" + logName + "'");
                return true;
            }
        }
        return false;
    } finally {
        dlm.close();
    }
}
 

private List<Long> collectLedgersFromStream(Namespace namespace,
                                            String stream,
                                            Set<Long> ledgers)
        throws IOException {
    DistributedLogManager dlm = namespace.openLog(stream);
    try {
        List<LogSegmentMetadata> segments = dlm.getLogSegments();
        List<Long> sLedgers = new ArrayList<Long>();
        for (LogSegmentMetadata segment : segments) {
            synchronized (ledgers) {
                ledgers.add(segment.getLogSegmentId());
            }
            sLedgers.add(segment.getLogSegmentId());
        }
        return sLedgers;
    } finally {
        dlm.close();
    }
}
 

private void printMetadata(DistributedLogManager dlm) throws Exception {
    printHeader(dlm);
    if (listSegments) {
        System.out.println("Ledgers : ");
        List<LogSegmentMetadata> segments = dlm.getLogSegments();
        for (LogSegmentMetadata segment : segments) {
            if (include(segment)) {
                printLedgerRow(segment);
            }
        }
    }
}
 

@Test(timeout = 60000)
public void testCloseReaderToCancelPendingReads() throws Exception {
    DistributedLogConfiguration confLocal = new DistributedLogConfiguration();
    confLocal.addConfiguration(conf);
    confLocal.setNumPrefetchEntriesPerLogSegment(10);
    confLocal.setMaxPrefetchEntriesPerLogSegment(10);
    DistributedLogManager dlm = createNewDLM(confLocal, runtime.getMethodName());
    DLMTestUtil.generateCompletedLogSegments(dlm, confLocal, 1, 20);
    List<LogSegmentMetadata> segments = dlm.getLogSegments();
    assertEquals(segments.size() + " log segments found, expected to be only one",
            1, segments.size());

    BKLogSegmentEntryReader reader = createEntryReader(segments.get(0), 0, confLocal);
    List<CompletableFuture<List<Entry.Reader>>> futures = Lists.newArrayList();
    for (int i = 0; i < 5; i++) {
        futures.add(reader.readNext(1));
    }
    assertFalse("Reader should not be closed yet", reader.isClosed());
    Utils.close(reader);
    for (CompletableFuture<List<Entry.Reader>> future : futures) {
        try {
            Utils.ioResult(future);
            fail("The read request should be cancelled");
        } catch (ReadCancelledException rce) {
            // expected
        }
    }
    assertFalse(reader.hasCaughtUpOnInprogress());
    assertTrue("Reader should be closed yet", reader.isClosed());
}
 

@Test(timeout = 60000)
public void testReleaseLockAfterFailedToRecover() throws Exception {
    String name = "release-lock-after-failed-to-recover";
    DistributedLogConfiguration confLocal = new DistributedLogConfiguration();
    confLocal.addConfiguration(testConf);
    confLocal.setLockTimeout(0);
    confLocal.setImmediateFlushEnabled(true);
    confLocal.setOutputBufferSize(0);

    DistributedLogManager dlm = createNewDLM(confLocal, name);
    BKAsyncLogWriter writer =
            (BKAsyncLogWriter) (dlm.startAsyncLogSegmentNonPartitioned());

    Utils.ioResult(writer.write(DLMTestUtil.getLogRecordInstance(1L)));
    writer.abort();

    for (int i = 0; i < 2; i++) {
        FailpointUtils.setFailpoint(
                FailpointUtils.FailPointName.FP_RecoverIncompleteLogSegments,
                FailpointUtils.FailPointActions.FailPointAction_Throw);

        try {
            dlm.startAsyncLogSegmentNonPartitioned();
            fail("Should fail during recovering incomplete log segments");
        } catch (IOException ioe) {
            // expected;
        } finally {
            FailpointUtils.removeFailpoint(FailpointUtils.FailPointName.FP_RecoverIncompleteLogSegments);
        }
    }

    writer = (BKAsyncLogWriter) (dlm.startAsyncLogSegmentNonPartitioned());

    List<LogSegmentMetadata> segments = dlm.getLogSegments();
    assertEquals(1, segments.size());
    assertFalse(segments.get(0).isInProgress());

    writer.close();
    dlm.close();
}
 

public static void fenceStream(DistributedLogConfiguration conf, URI uri, String name) throws Exception {
    DistributedLogManager dlm = createNewDLM(name, conf, uri);
    try {
        List<LogSegmentMetadata> logSegmentList = dlm.getLogSegments();
        LogSegmentMetadata lastSegment = logSegmentList.get(logSegmentList.size() - 1);
        LogSegmentEntryStore entryStore =
                dlm.getNamespaceDriver().getLogSegmentEntryStore(NamespaceDriver.Role.READER);
        Utils.close(Utils.ioResult(entryStore.openRandomAccessReader(lastSegment, true)));
    } finally {
        dlm.close();
    }
}
 

static Map<Long, LogSegmentMetadata> getLogSegments(DistributedLogManager dlm) throws Exception {
    Map<Long, LogSegmentMetadata> logSegmentMap =
            new HashMap<Long, LogSegmentMetadata>();
    List<LogSegmentMetadata> segments = dlm.getLogSegments();
    for (LogSegmentMetadata segment : segments) {
        logSegmentMap.put(segment.getLogSegmentSequenceNumber(), segment);
    }
    return logSegmentMap;
}
 

private static StreamCandidate checkStream(
        final Namespace namespace,
        final String streamName,
        final OrderedScheduler scheduler) throws IOException {
    DistributedLogManager dlm = namespace.openLog(streamName);
    try {
        List<LogSegmentMetadata> segments = dlm.getLogSegments();
        if (segments.isEmpty()) {
            return null;
        }
        List<CompletableFuture<LogSegmentCandidate>> futures =
                new ArrayList<CompletableFuture<LogSegmentCandidate>>(segments.size());
        for (LogSegmentMetadata segment : segments) {
            futures.add(checkLogSegment(namespace, streamName, segment, scheduler));
        }
        List<LogSegmentCandidate> segmentCandidates;
        try {
            segmentCandidates = FutureUtils.result(FutureUtils.collect(futures));
        } catch (Exception e) {
            throw new IOException("Failed on checking stream " + streamName, e);
        }
        StreamCandidate streamCandidate = new StreamCandidate(streamName);
        for (LogSegmentCandidate segmentCandidate: segmentCandidates) {
            if (null != segmentCandidate) {
                streamCandidate.addLogSegmentCandidate(segmentCandidate);
            }
        }
        if (streamCandidate.segmentCandidates.isEmpty()) {
            return null;
        }
        return streamCandidate;
    } finally {
        dlm.close();
    }
}
 

/**
 * Create Log Segment while max sequence number isn't match with list of log segments.
 */
@Test(timeout = 60000)
public void testCreateLogSegmentUnmatchMaxSequenceNumber() throws Exception {
    URI uri = createURI();
    String streamName = testName.getMethodName();
    DistributedLogConfiguration conf = new DistributedLogConfiguration()
            .setLockTimeout(99999)
            .setOutputBufferSize(0)
            .setImmediateFlushEnabled(true)
            .setEnableLedgerAllocatorPool(true)
            .setLedgerAllocatorPoolName("test");
    Namespace namespace = NamespaceBuilder.newBuilder().conf(conf).uri(uri).build();

    namespace.createLog(streamName);
    MaxLogSegmentSequenceNo max1 = getMaxLogSegmentSequenceNo(getZooKeeperClient(namespace), uri, streamName, conf);
    assertEquals(DistributedLogConstants.UNASSIGNED_LOGSEGMENT_SEQNO, max1.getSequenceNumber());
    DistributedLogManager dlm = namespace.openLog(streamName);
    final int numSegments = 3;
    for (int i = 0; i < numSegments; i++) {
        BKSyncLogWriter out = (BKSyncLogWriter) dlm.startLogSegmentNonPartitioned();
        out.write(DLMTestUtil.getLogRecordInstance(i));
        out.closeAndComplete();
    }
    MaxLogSegmentSequenceNo max2 = getMaxLogSegmentSequenceNo(getZooKeeperClient(namespace), uri, streamName, conf);
    assertEquals(3, max2.getSequenceNumber());

    // update the max ledger sequence number
    updateMaxLogSegmentSequenceNo(getZooKeeperClient(namespace), uri, streamName, conf,
            DLUtils.serializeLogSegmentSequenceNumber(99));

    DistributedLogManager dlm1 = namespace.openLog(streamName);
    try {
        BKSyncLogWriter out1 = (BKSyncLogWriter) dlm1.startLogSegmentNonPartitioned();
        out1.write(DLMTestUtil.getLogRecordInstance(numSegments + 1));
        out1.closeAndComplete();
        fail("Should fail creating new log segment when encountered unmatch max ledger sequence number");
    } catch (DLIllegalStateException lse) {
        // expected
    } finally {
        dlm1.close();
    }

    DistributedLogManager dlm2 = namespace.openLog(streamName);
    List<LogSegmentMetadata> segments = dlm2.getLogSegments();
    try {
        assertEquals(3, segments.size());
        assertEquals(1L, segments.get(0).getLogSegmentSequenceNumber());
        assertEquals(2L, segments.get(1).getLogSegmentSequenceNumber());
        assertEquals(3L, segments.get(2).getLogSegmentSequenceNumber());
    } finally {
        dlm2.close();
    }

    dlm.close();
    namespace.close();
}
 

@Test(timeout = 600000)
public void testLastDLSNInRollingLogSegments() throws Exception {
    final Map<Long, DLSN> lastDLSNs = new HashMap<Long, DLSN>();
    String name = "distrlog-lastdlsn-in-rolling-log-segments";
    DistributedLogConfiguration confLocal = new DistributedLogConfiguration();
    confLocal.loadConf(conf);
    confLocal.setImmediateFlushEnabled(true);
    confLocal.setOutputBufferSize(0);
    confLocal.setLogSegmentRollingIntervalMinutes(0);
    confLocal.setMaxLogSegmentBytes(40);

    int numEntries = 100;

    DistributedLogManager dlm = createNewDLM(confLocal, name);
    BKAsyncLogWriter writer = (BKAsyncLogWriter) dlm.startAsyncLogSegmentNonPartitioned();

    final CountDownLatch latch = new CountDownLatch(numEntries);

    // send requests in parallel to have outstanding requests
    for (int i = 1; i <= numEntries; i++) {
        final int entryId = i;
        CompletableFuture<DLSN> writeFuture =
            writer.write(DLMTestUtil.getLogRecordInstance(entryId))
                .whenComplete(new FutureEventListener<DLSN>() {

            @Override
            public void onSuccess(DLSN value) {
                logger.info("Completed entry {} : {}.", entryId, value);
                synchronized (lastDLSNs) {
                    DLSN lastDLSN = lastDLSNs.get(value.getLogSegmentSequenceNo());
                    if (null == lastDLSN || lastDLSN.compareTo(value) < 0) {
                        lastDLSNs.put(value.getLogSegmentSequenceNo(), value);
                    }
                }
                latch.countDown();
            }

            @Override
            public void onFailure(Throwable cause) {

            }
        });
        if (i == 1) {
            // wait for first log segment created
            Utils.ioResult(writeFuture);
        }
    }
    latch.await();

    // make sure all ensure blocks were executed.
    writer.closeAndComplete();

    List<LogSegmentMetadata> segments = dlm.getLogSegments();
    logger.info("lastDLSNs after writes {} {}", lastDLSNs.size(), lastDLSNs);
    logger.info("segments after writes {} {}", segments.size(), segments);
    assertTrue(segments.size() >= 2);
    assertTrue(lastDLSNs.size() >= 2);
    assertEquals(lastDLSNs.size(), segments.size());
    for (LogSegmentMetadata segment : segments) {
        DLSN dlsnInMetadata = segment.getLastDLSN();
        DLSN dlsnSeen = lastDLSNs.get(segment.getLogSegmentSequenceNumber());
        assertNotNull(dlsnInMetadata);
        assertNotNull(dlsnSeen);
        if (dlsnInMetadata.compareTo(dlsnSeen) != 0) {
            logger.error("Last dlsn recorded in log segment {} is different from the one already seen {}.",
                         dlsnInMetadata, dlsnSeen);
        }
        assertEquals(0, dlsnInMetadata.compareTo(dlsnSeen));
    }

    dlm.close();
}
 

@Test(timeout = 60000)
public void testUnableToRollLogSegments() throws Exception {
    String name = "distrlog-unable-to-roll-log-segments";
    DistributedLogConfiguration confLocal = new DistributedLogConfiguration();
    confLocal.loadConf(conf);
    confLocal.setImmediateFlushEnabled(true);
    confLocal.setOutputBufferSize(0);
    confLocal.setLogSegmentRollingIntervalMinutes(0);
    confLocal.setMaxLogSegmentBytes(1);

    DistributedLogManager dlm = createNewDLM(confLocal, name);
    BKAsyncLogWriter writer = (BKAsyncLogWriter) dlm.startAsyncLogSegmentNonPartitioned();

    long txId = 1L;

    // Create Log Segments
    Utils.ioResult(writer.write(DLMTestUtil.getLogRecordInstance(txId)));

    FailpointUtils.setFailpoint(FailpointUtils.FailPointName.FP_StartLogSegmentBeforeLedgerCreate,
            FailpointUtils.FailPointActions.FailPointAction_Throw);

    try {
        // If we couldn't open new log segment, we should keep using the old one
        final int numRecords = 10;
        final CountDownLatch latch = new CountDownLatch(numRecords);
        for (int i = 0; i < numRecords; i++) {
            writer.write(DLMTestUtil.getLogRecordInstance(++txId)).whenComplete(new FutureEventListener<DLSN>() {
                @Override
                public void onSuccess(DLSN value) {
                    logger.info("Completed entry : {}.", value);
                    latch.countDown();
                }
                @Override
                public void onFailure(Throwable cause) {
                    logger.error("Failed to write entries : ", cause);
                }
            });
        }

        latch.await();

        writer.close();

        List<LogSegmentMetadata> segments = dlm.getLogSegments();
        logger.info("LogSegments: {}", segments);

        assertEquals(1, segments.size());

        long expectedTxID = 1L;
        LogReader reader = dlm.getInputStream(DLSN.InitialDLSN);
        LogRecordWithDLSN record = reader.readNext(false);
        while (null != record) {
            DLMTestUtil.verifyLogRecord(record);
            assertEquals(expectedTxID++, record.getTransactionId());
            assertEquals(record.getTransactionId() - 1, record.getSequenceId());

            record = reader.readNext(false);
        }

        assertEquals(12L, expectedTxID);

        reader.close();

        dlm.close();
    } finally {
        FailpointUtils.removeFailpoint(FailpointUtils.FailPointName.FP_StartLogSegmentBeforeLedgerCreate);
    }
}
 

@Test(timeout = 60000)
public void testRollingLogSegments() throws Exception {
    logger.info("start testRollingLogSegments");
    String name = "distrlog-rolling-logsegments-hightraffic";
    DistributedLogConfiguration confLocal = new DistributedLogConfiguration();
    confLocal.loadConf(conf);
    confLocal.setImmediateFlushEnabled(true);
    confLocal.setOutputBufferSize(0);
    confLocal.setLogSegmentRollingIntervalMinutes(0);
    confLocal.setMaxLogSegmentBytes(1);
    confLocal.setLogSegmentRollingConcurrency(Integer.MAX_VALUE);

    int numLogSegments = 10;

    DistributedLogManager dlm = createNewDLM(confLocal, name);
    BKAsyncLogWriter writer = (BKAsyncLogWriter) dlm.startAsyncLogSegmentNonPartitioned();

    final CountDownLatch latch = new CountDownLatch(numLogSegments);
    long startTime = System.currentTimeMillis();
    // send requests in parallel to have outstanding requests
    for (int i = 1; i <= numLogSegments; i++) {
        final int entryId = i;
        CompletableFuture<DLSN> writeFuture = writer.write(DLMTestUtil.getLogRecordInstance(entryId))
                .whenComplete(new FutureEventListener<DLSN>() {
            @Override
            public void onSuccess(DLSN value) {
                logger.info("Completed entry {} : {}.", entryId, value);
                latch.countDown();
            }
            @Override
            public void onFailure(Throwable cause) {
                logger.error("Failed to write entries : {}", cause);
            }
        });
        if (i == 1) {
            // wait for first log segment created
            Utils.ioResult(writeFuture);
        }
    }
    latch.await();

    logger.info("Took {} ms to completed all requests.", System.currentTimeMillis() - startTime);

    List<LogSegmentMetadata> segments = dlm.getLogSegments();
    logger.info("LogSegments : {}", segments);

    assertTrue(segments.size() >= 2);
    ensureOnlyOneInprogressLogSegments(segments);

    int numSegmentsAfterAsyncWrites = segments.size();

    // writer should work after rolling log segments
    // there would be (numLogSegments/2) segments based on current rolling policy
    for (int i = 1; i <= numLogSegments; i++) {
        DLSN newDLSN = Utils.ioResult(writer.write(DLMTestUtil.getLogRecordInstance(numLogSegments + i)));
        logger.info("Completed entry {} : {}", numLogSegments + i, newDLSN);
    }

    segments = dlm.getLogSegments();
    logger.info("LogSegments : {}", segments);

    assertEquals(numSegmentsAfterAsyncWrites + numLogSegments / 2, segments.size());
    ensureOnlyOneInprogressLogSegments(segments);

    writer.close();
    dlm.close();
}
 

@Test(timeout = 60000)
public void testCreateLogSegmentAfterLoseLock() throws Exception {
    URI uri = createDLMURI("/LogSegmentCreation");
    String name = "distrlog-createlogsegment-afterloselock";
    DistributedLogConfiguration conf = new DistributedLogConfiguration()
            .setLockTimeout(99999)
            .setOutputBufferSize(0)
            .setImmediateFlushEnabled(true)
            .setEnableLedgerAllocatorPool(true)
            .setLedgerAllocatorPoolName("test");
    Namespace namespace = NamespaceBuilder.newBuilder()
            .conf(conf).uri(uri).build();
    DistributedLogManager dlm = namespace.openLog(name);
    final int numSegments = 3;
    for (int i = 0; i < numSegments; i++) {
        BKSyncLogWriter out = (BKSyncLogWriter) dlm.startLogSegmentNonPartitioned();
        out.write(DLMTestUtil.getLogRecordInstance(i));
        out.closeAndComplete();
    }

    List<LogSegmentMetadata> segments = dlm.getLogSegments();
    LOG.info("Segments : {}", segments);
    assertEquals(3, segments.size());

    final DistributedLogManager dlm1 = namespace.openLog(name);
    final DistributedLogManager dlm2 = namespace.openLog(name);

    BKAsyncLogWriter writer1 = (BKAsyncLogWriter) dlm1.startAsyncLogSegmentNonPartitioned();
    LOG.info("Created writer 1.");
    BKSyncLogWriter writer2 = (BKSyncLogWriter) dlm2.startLogSegmentNonPartitioned();
    LOG.info("Created writer 2.");
    writer2.write(DLMTestUtil.getLogRecordInstance(numSegments));
    writer2.closeAndComplete();

    try {
        Utils.ioResult(writer1.write(DLMTestUtil.getLogRecordInstance(numSegments + 1)));
        fail("Should fail on writing new log records.");
    } catch (Throwable t) {
        LOG.error("Failed to write entry : ", t);
    }

    segments = dlm.getLogSegments();

    boolean hasInprogress = false;
    boolean hasDuplicatedSegment = false;
    long nextSeqNo = segments.get(0).getLogSegmentSequenceNumber();
    for (int i = 1; i < segments.size(); i++) {
        LogSegmentMetadata segment = segments.get(i);
        assertTrue(segment.getLogSegmentSequenceNumber() >= nextSeqNo);
        if (segment.getLogSegmentSequenceNumber() == nextSeqNo) {
            hasDuplicatedSegment = true;
        }
        nextSeqNo = segment.getLogSegmentSequenceNumber();
        if (segment.isInProgress()) {
            hasInprogress = true;
        }
    }
    assertEquals(4, segments.size());
    assertFalse(hasInprogress);
    assertFalse(hasDuplicatedSegment);

    LOG.info("Segments : duplicated = {}, inprogress = {}, {}",
             new Object[] { hasDuplicatedSegment, hasInprogress, segments });

    dlm1.close();
    dlm2.close();
    dlm.close();

    namespace.close();
}
 

@Test(timeout = 60000)
public void testCreateLogStreamWithDifferentReplicationFactor() throws Exception {
    String name = runtime.getMethodName();
    DistributedLogConfiguration confLocal = new DistributedLogConfiguration();
    confLocal.addConfiguration(testConf);
    confLocal.setOutputBufferSize(0);
    confLocal.setImmediateFlushEnabled(false);
    confLocal.setPeriodicFlushFrequencyMilliSeconds(0);

    ConcurrentBaseConfiguration baseConf = new ConcurrentConstConfiguration(confLocal);
    DynamicDistributedLogConfiguration dynConf = new DynamicDistributedLogConfiguration(baseConf);
    dynConf.setProperty(DistributedLogConfiguration.BKDL_BOOKKEEPER_ENSEMBLE_SIZE,
            DistributedLogConfiguration.BKDL_BOOKKEEPER_ENSEMBLE_SIZE_DEFAULT - 1);

    URI uri = createDLMURI("/" + name);
    ensureURICreated(uri);
    Namespace namespace = NamespaceBuilder.newBuilder()
            .conf(confLocal).uri(uri).build();

    // use the pool
    DistributedLogManager dlm = namespace.openLog(name + "-pool");
    AsyncLogWriter writer = dlm.startAsyncLogSegmentNonPartitioned();
    Utils.ioResult(writer.write(DLMTestUtil.getLogRecordInstance(1L)));
    List<LogSegmentMetadata> segments = dlm.getLogSegments();
    assertEquals(1, segments.size());
    long ledgerId = segments.get(0).getLogSegmentId();
    LedgerHandle lh = ((BKNamespaceDriver) namespace.getNamespaceDriver()).getReaderBKC().get()
            .openLedgerNoRecovery(ledgerId, BookKeeper.DigestType.CRC32, confLocal.getBKDigestPW().getBytes(UTF_8));
    LedgerMetadata metadata = BookKeeperAccessor.getLedgerMetadata(lh);
    assertEquals(DistributedLogConfiguration.BKDL_BOOKKEEPER_ENSEMBLE_SIZE_DEFAULT, metadata.getEnsembleSize());
    lh.close();
    Utils.close(writer);
    dlm.close();

    // use customized configuration
    dlm = namespace.openLog(
            name + "-custom",
            java.util.Optional.empty(),
            java.util.Optional.of(dynConf),
            java.util.Optional.empty());
    writer = dlm.startAsyncLogSegmentNonPartitioned();
    Utils.ioResult(writer.write(DLMTestUtil.getLogRecordInstance(1L)));
    segments = dlm.getLogSegments();
    assertEquals(1, segments.size());
    ledgerId = segments.get(0).getLogSegmentId();
    lh = ((BKNamespaceDriver) namespace.getNamespaceDriver()).getReaderBKC().get()
            .openLedgerNoRecovery(ledgerId, BookKeeper.DigestType.CRC32, confLocal.getBKDigestPW().getBytes(UTF_8));
    metadata = BookKeeperAccessor.getLedgerMetadata(lh);
    assertEquals(DistributedLogConfiguration.BKDL_BOOKKEEPER_ENSEMBLE_SIZE_DEFAULT - 1, metadata.getEnsembleSize());
    lh.close();
    Utils.close(writer);
    dlm.close();
    namespace.close();
}
 

@Test(timeout = 60000)
public void testReadEntriesFromCompleteLogSegment() throws Exception {
    DistributedLogConfiguration confLocal = new DistributedLogConfiguration();
    confLocal.addConfiguration(conf);
    confLocal.setOutputBufferSize(0);
    confLocal.setPeriodicFlushFrequencyMilliSeconds(0);
    confLocal.setImmediateFlushEnabled(false);
    confLocal.setNumPrefetchEntriesPerLogSegment(10);
    confLocal.setMaxPrefetchEntriesPerLogSegment(10);
    DistributedLogManager dlm = createNewDLM(confLocal, runtime.getMethodName());
    generateCompletedLogSegments(dlm, confLocal, 1, 20);
    List<LogSegmentMetadata> segments = dlm.getLogSegments();
    assertEquals(segments.size() + " log segments found, expected to be only one",
            1, segments.size());

    BKLogSegmentEntryReader reader = createEntryReader(segments.get(0), 0, confLocal);
    reader.start();
    boolean done = false;
    long txId = 1L;
    long entryId = 0L;
    while (!done) {
        Entry.Reader entryReader;
        try {
            entryReader = Utils.ioResult(reader.readNext(1)).get(0);
        } catch (EndOfLogSegmentException eol) {
            done = true;
            continue;
        }
        LogRecordWithDLSN record = entryReader.nextRecord();
        while (null != record) {
            if (!record.isControl()) {
                DLMTestUtil.verifyLogRecord(record);
                assertEquals(txId, record.getTransactionId());
                ++txId;
            }
            DLSN dlsn = record.getDlsn();
            assertEquals(1L, dlsn.getLogSegmentSequenceNo());
            assertEquals(entryId, dlsn.getEntryId());
            record = entryReader.nextRecord();
        }
        ++entryId;
    }
    assertEquals(21, txId);
    assertFalse(reader.hasCaughtUpOnInprogress());
    Utils.close(reader);
}
 

@Test(timeout = 60000)
public void testMaxPrefetchEntriesSmallBatch() throws Exception {
    DistributedLogConfiguration confLocal = new DistributedLogConfiguration();
    confLocal.addConfiguration(conf);
    confLocal.setOutputBufferSize(0);
    confLocal.setPeriodicFlushFrequencyMilliSeconds(0);
    confLocal.setImmediateFlushEnabled(false);
    confLocal.setNumPrefetchEntriesPerLogSegment(2);
    confLocal.setMaxPrefetchEntriesPerLogSegment(10);
    DistributedLogManager dlm = createNewDLM(confLocal, runtime.getMethodName());
    generateCompletedLogSegments(dlm, confLocal, 1, 20);
    List<LogSegmentMetadata> segments = dlm.getLogSegments();
    assertEquals(segments.size() + " log segments found, expected to be only one",
            1, segments.size());

    BKLogSegmentEntryReader reader = createEntryReader(segments.get(0), 0, confLocal);
    reader.start();

    // wait for the read ahead entries to become available
    while (reader.readAheadEntries.size() < 10) {
        TimeUnit.MILLISECONDS.sleep(10);
    }

    long txId = 1L;
    long entryId = 0L;

    assertEquals(10, reader.readAheadEntries.size());
    assertEquals(10, reader.getNextEntryId());
    assertFalse(reader.hasCaughtUpOnInprogress());
    // read first entry
    Entry.Reader entryReader = Utils.ioResult(reader.readNext(1)).get(0);
    LogRecordWithDLSN record = entryReader.nextRecord();
    while (null != record) {
        if (!record.isControl()) {
            DLMTestUtil.verifyLogRecord(record);
            assertEquals(txId, record.getTransactionId());
            ++txId;
        }
        DLSN dlsn = record.getDlsn();
        assertEquals(1L, dlsn.getLogSegmentSequenceNo());
        assertEquals(entryId, dlsn.getEntryId());
        record = entryReader.nextRecord();
    }
    ++entryId;
    assertEquals(2L, txId);
    // wait for the read ahead entries to become 10 again
    while (reader.readAheadEntries.size() < 10) {
        TimeUnit.MILLISECONDS.sleep(10);
    }

    assertEquals(10, reader.readAheadEntries.size());
    assertEquals(11, reader.getNextEntryId());
    assertFalse(reader.hasCaughtUpOnInprogress());

    Utils.close(reader);
}
 

@Test(timeout = 60000)
public void testMaxPrefetchEntriesLargeBatch() throws Exception {
    DistributedLogConfiguration confLocal = new DistributedLogConfiguration();
    confLocal.addConfiguration(conf);
    confLocal.setOutputBufferSize(0);
    confLocal.setPeriodicFlushFrequencyMilliSeconds(0);
    confLocal.setImmediateFlushEnabled(false);
    confLocal.setNumPrefetchEntriesPerLogSegment(10);
    confLocal.setMaxPrefetchEntriesPerLogSegment(5);
    DistributedLogManager dlm = createNewDLM(confLocal, runtime.getMethodName());
    generateCompletedLogSegments(dlm, confLocal, 1, 20);
    List<LogSegmentMetadata> segments = dlm.getLogSegments();
    assertEquals(segments.size() + " log segments found, expected to be only one",
            1, segments.size());

    BKLogSegmentEntryReader reader = createEntryReader(segments.get(0), 0, confLocal);
    reader.start();

    // wait for the read ahead entries to become available
    while (reader.readAheadEntries.size() < 5) {
        TimeUnit.MILLISECONDS.sleep(10);
    }

    long txId = 1L;
    long entryId = 0L;

    assertEquals(5, reader.readAheadEntries.size());
    assertEquals(5, reader.getNextEntryId());
    // read first entry
    Entry.Reader entryReader = Utils.ioResult(reader.readNext(1)).get(0);
    LogRecordWithDLSN record = entryReader.nextRecord();
    while (null != record) {
        if (!record.isControl()) {
            DLMTestUtil.verifyLogRecord(record);
            assertEquals(txId, record.getTransactionId());
            ++txId;
        }
        DLSN dlsn = record.getDlsn();
        assertEquals(1L, dlsn.getLogSegmentSequenceNo());
        assertEquals(entryId, dlsn.getEntryId());
        record = entryReader.nextRecord();
    }
    ++entryId;
    assertEquals(2L, txId);
    // wait for the read ahead entries to become 10 again
    while (reader.readAheadEntries.size() < 5) {
        TimeUnit.MILLISECONDS.sleep(10);
    }

    assertEquals(5, reader.readAheadEntries.size());
    assertEquals(6, reader.getNextEntryId());
    assertFalse(reader.hasCaughtUpOnInprogress());

    Utils.close(reader);
}
 

@Test(timeout = 60000)
public void testMaxPrefetchEntriesSmallSegment() throws Exception {
    DistributedLogConfiguration confLocal = new DistributedLogConfiguration();
    confLocal.addConfiguration(conf);
    confLocal.setOutputBufferSize(0);
    confLocal.setPeriodicFlushFrequencyMilliSeconds(0);
    confLocal.setImmediateFlushEnabled(false);
    confLocal.setNumPrefetchEntriesPerLogSegment(10);
    confLocal.setMaxPrefetchEntriesPerLogSegment(20);
    DistributedLogManager dlm = createNewDLM(confLocal, runtime.getMethodName());
    generateCompletedLogSegments(dlm, confLocal, 1, 5);
    List<LogSegmentMetadata> segments = dlm.getLogSegments();
    assertEquals(segments.size() + " log segments found, expected to be only one",
            1, segments.size());

    BKLogSegmentEntryReader reader = createEntryReader(segments.get(0), 0, confLocal);
    reader.start();

    // wait for the read ahead entries to become available
    while (reader.readAheadEntries.size() < (reader.getLastAddConfirmed() + 1)) {
        TimeUnit.MILLISECONDS.sleep(10);
    }

    long txId = 1L;
    long entryId = 0L;

    assertEquals((reader.getLastAddConfirmed() + 1), reader.readAheadEntries.size());
    assertEquals((reader.getLastAddConfirmed() + 1), reader.getNextEntryId());
    // read first entry
    Entry.Reader entryReader = Utils.ioResult(reader.readNext(1)).get(0);
    LogRecordWithDLSN record = entryReader.nextRecord();
    while (null != record) {
        if (!record.isControl()) {
            DLMTestUtil.verifyLogRecord(record);
            assertEquals(txId, record.getTransactionId());
            ++txId;
        }
        DLSN dlsn = record.getDlsn();
        assertEquals(1L, dlsn.getLogSegmentSequenceNo());
        assertEquals(entryId, dlsn.getEntryId());
        record = entryReader.nextRecord();
    }
    ++entryId;
    assertEquals(2L, txId);
    assertEquals(reader.getLastAddConfirmed(), reader.readAheadEntries.size());
    assertEquals((reader.getLastAddConfirmed() + 1), reader.getNextEntryId());
    assertFalse(reader.hasCaughtUpOnInprogress());

    Utils.close(reader);
}
 

@Test(timeout = 60000)
public void testPurgeLogs() throws Exception {
    String name = "distrlog-purge-logs";
    URI uri = createDLMURI("/" + name);

    populateData(new HashMap<Long, DLSN>(), conf, name, 10, 10, false);

    DistributedLogManager distributedLogManager = createNewDLM(conf, name);

    List<LogSegmentMetadata> segments = distributedLogManager.getLogSegments();
    LOG.info("Segments before modifying completion time : {}", segments);

    ZooKeeperClient zkc = TestZooKeeperClientBuilder.newBuilder(conf)
            .uri(uri)
            .build();

    // Update completion time of first 5 segments
    long newTimeMs = System.currentTimeMillis() - 60 * 60 * 1000 * 2;
    for (int i = 0; i < 5; i++) {
        LogSegmentMetadata segment = segments.get(i);
        updateCompletionTime(zkc, segment, newTimeMs + i);
    }
    zkc.close();

    segments = distributedLogManager.getLogSegments();
    LOG.info("Segments after modifying completion time : {}", segments);

    DistributedLogConfiguration confLocal = new DistributedLogConfiguration();
    confLocal.loadConf(conf);
    confLocal.setRetentionPeriodHours(1);
    confLocal.setExplicitTruncationByApplication(false);

    DistributedLogManager dlm = createNewDLM(confLocal, name);
    AsyncLogWriter writer = dlm.startAsyncLogSegmentNonPartitioned();
    long txid = 1 + 10 * 10;
    for (int j = 1; j <= 10; j++) {
        Utils.ioResult(writer.write(DLMTestUtil.getLogRecordInstance(txid++)));
    }

    // to make sure the truncation task is executed
    DLSN lastDLSN = Utils.ioResult(dlm.getLastDLSNAsync());
    LOG.info("Get last dlsn of stream {} : {}", name, lastDLSN);

    assertEquals(6, distributedLogManager.getLogSegments().size());

    Utils.close(writer);
    dlm.close();

    distributedLogManager.close();
}
 

@Test(timeout = 60000)
public void testPartiallyTruncateTruncatedSegments() throws Exception {
    String name = "distrlog-partially-truncate-truncated-segments";
    URI uri = createDLMURI("/" + name);

    DistributedLogConfiguration confLocal = new DistributedLogConfiguration();
    confLocal.addConfiguration(conf);
    confLocal.setExplicitTruncationByApplication(true);

    // populate
    Map<Long, DLSN> dlsnMap = new HashMap<Long, DLSN>();
    populateData(dlsnMap, confLocal, name, 4, 10, false);

    DistributedLogManager dlm = createNewDLM(confLocal, name);
    List<LogSegmentMetadata> segments = dlm.getLogSegments();
    LOG.info("Segments before modifying segment status : {}", segments);

    ZooKeeperClient zkc = TestZooKeeperClientBuilder.newBuilder(conf)
            .uri(uri)
            .build();
    for (int i = 0; i < 4; i++) {
        LogSegmentMetadata segment = segments.get(i);
        setTruncationStatus(zkc, segment, TruncationStatus.TRUNCATED);
    }

    List<LogSegmentMetadata> newSegments = dlm.getLogSegments();
    LOG.info("Segments after changing truncation status : {}", newSegments);

    dlm.close();

    DistributedLogManager newDLM = createNewDLM(confLocal, name);
    AsyncLogWriter newWriter = newDLM.startAsyncLogSegmentNonPartitioned();
    Utils.ioResult(newWriter.truncate(dlsnMap.get(15L)));

    List<LogSegmentMetadata> newSegments2 = newDLM.getLogSegments();
    assertArrayEquals(newSegments.toArray(new LogSegmentMetadata[4]),
                      newSegments2.toArray(new LogSegmentMetadata[4]));

    Utils.close(newWriter);
    newDLM.close();
    zkc.close();
}