Java Code Examples for org.apache.distributedlog.api.DistributedLogManager#startLogSegmentNonPartitioned()

private void initDlogMeta(String dlNamespace, String un, String streamName) throws Exception {
    URI uri = createDLMURI(dlNamespace);
    DistributedLogConfiguration newConf = new DistributedLogConfiguration();
    newConf.addConfiguration(conf);
    newConf.setCreateStreamIfNotExists(true);
    newConf.setZkAclId(un);
    Namespace namespace = NamespaceBuilder.newBuilder()
            .conf(newConf).uri(uri).build();
    DistributedLogManager dlm = namespace.openLog(streamName);
    LogWriter writer = dlm.startLogSegmentNonPartitioned();
    for (int i = 0; i < 10; i++) {
        writer.write(DLMTestUtil.getLogRecordInstance(1L));
    }
    writer.close();
    dlm.close();
    namespace.close();
}
 

private void createLogPathTest(String logName) throws Exception {
    URI uri = createDLMURI("/" + runtime.getMethodName());
    ensureURICreated(zooKeeperClient.get(), uri);
    DistributedLogConfiguration newConf = new DistributedLogConfiguration();
    newConf.addConfiguration(conf);
    newConf.setCreateStreamIfNotExists(false);
    Namespace namespace = NamespaceBuilder.newBuilder()
        .conf(newConf).uri(uri).build();
    DistributedLogManager dlm = namespace.openLog(logName);
    LogWriter writer;
    try {
        writer = dlm.startLogSegmentNonPartitioned();
        writer.write(DLMTestUtil.getLogRecordInstance(1L));
        writer.commit();
        fail("Should fail to write data if stream doesn't exist.");
    } catch (IOException ioe) {
        // expected
    }
    dlm.close();
}
 

/**
 * Write <code>numRecords</code> records to the log, starting with <code>startTxId</code>.
 * It flushes every <code>flushPerNumRecords</code> records.
 *
 * @param dlm
 *          distributedlog manager
 * @param numRecords
 *          num records to write
 * @param startTxId
 *          start tx id
 * @param flushPerNumRecords
 *          number records to flush
 * @return next tx id
 * @throws IOException
 */
private static long writeLogSegment(DistributedLogManager dlm,
                                    int numRecords,
                                    long startTxId,
                                    int flushPerNumRecords,
                                    boolean emptyRecord) throws IOException {
    long txid = startTxId;
    LogWriter writer = dlm.startLogSegmentNonPartitioned();
    for (long j = 1; j <= numRecords; j++) {
        if (emptyRecord) {
            writer.write(DLMTestUtil.getEmptyLogRecordInstance(txid++));
        } else {
            writer.write(DLMTestUtil.getLargeLogRecordInstance(txid++));
        }
        if (j % flushPerNumRecords == 0) {
            writer.flush();
            writer.commit();
        }
    }
    writer.flush();
    writer.commit();
    writer.close();
    return txid;
}
 

/**
 * Write records into <i>numLogSegments</i> log segments.
 * Each log segment has <i>numRecordsPerLogSegment</i> records.
 *
 * @param dlm
 *          distributedlog manager
 * @param numLogSegments
 *          number of log segments
 * @param numRecordsPerLogSegment
 *          number records per log segment
 * @param startTxId
 *          start tx id
 * @return next tx id
 */
private static long writeRecords(DistributedLogManager dlm,
                                 int numLogSegments,
                                 int numRecordsPerLogSegment,
                                 long startTxId,
                                 boolean emptyRecord) throws IOException {
    long txid = startTxId;
    for (long i = 0; i < numLogSegments; i++) {
        BKSyncLogWriter writer = (BKSyncLogWriter) dlm.startLogSegmentNonPartitioned();
        for (long j = 1; j <= numRecordsPerLogSegment; j++) {
            if (emptyRecord) {
                writer.write(DLMTestUtil.getEmptyLogRecordInstance(txid++));
            } else {
                writer.write(DLMTestUtil.getLargeLogRecordInstance(txid++));
            }
        }
        writer.closeAndComplete();
    }
    return txid;
}
 

@Test(timeout = 60000)
public void testWriteFailsAfterMarkEndOfStream() throws Exception {
    String name = "distrlog-mark-end-failure";
    DistributedLogManager dlm = createNewDLM(conf, name);

    long txid = 1;
    txid = writeAndMarkEndOfStream(dlm, txid);

    assertEquals(txid - 1, dlm.getLastTxId());
    LogRecord last = dlm.getLastLogRecord();
    assertEquals(txid - 1, last.getTransactionId());
    DLMTestUtil.verifyLogRecord(last);
    assertTrue(dlm.isEndOfStreamMarked());

    LogWriter writer = null;
    boolean exceptionEncountered = false;
    try {
        writer = dlm.startLogSegmentNonPartitioned();
        for (long j = 1; j <= DEFAULT_SEGMENT_SIZE / 2; j++) {
            writer.write(DLMTestUtil.getLogRecordInstance(txid++));
        }
    } catch (EndOfStreamException exc) {
        exceptionEncountered = true;
    }
    writer.close();
    assertTrue(exceptionEncountered);
}
 

@Test(timeout = 60000)
public void testCreateIfNotExists() throws Exception {
    URI uri = createDLMURI("/" + runtime.getMethodName());
    ensureURICreated(zooKeeperClient.get(), uri);
    DistributedLogConfiguration newConf = new DistributedLogConfiguration();
    newConf.addConfiguration(conf);
    newConf.setCreateStreamIfNotExists(false);
    String streamName = "test-stream";
    Namespace namespace = NamespaceBuilder.newBuilder()
            .conf(newConf).uri(uri).build();
    DistributedLogManager dlm = namespace.openLog(streamName);
    LogWriter writer;
    try {
        writer = dlm.startLogSegmentNonPartitioned();
        writer.write(DLMTestUtil.getLogRecordInstance(1L));
        fail("Should fail to write data if stream doesn't exist.");
    } catch (IOException ioe) {
        // expected
    }
    dlm.close();

    // create the stream
    namespace.createLog(streamName);

    DistributedLogManager newDLM = namespace.openLog(streamName);
    LogWriter newWriter = newDLM.startLogSegmentNonPartitioned();
    newWriter.write(DLMTestUtil.getLogRecordInstance(1L));
    newWriter.close();
    newDLM.close();
}
 

@Test(timeout = 60000)
public void testReadingFromEmptyLog() throws Exception {
    String name = testName.getMethodName();
    DistributedLogConfiguration confLocal = new DistributedLogConfiguration();
    confLocal.addConfiguration(conf);
    confLocal.setOutputBufferSize(0);
    confLocal.setPeriodicFlushFrequencyMilliSeconds(Integer.MAX_VALUE);

    DistributedLogManager dlm = createNewDLM(confLocal, name);
    BKSyncLogWriter out = (BKSyncLogWriter) dlm.startLogSegmentNonPartitioned();
    // write a record but not commit
    LogRecord op = DLMTestUtil.getLogRecordInstance(1L);
    out.write(op);

    LogReader reader = dlm.getInputStream(1L);
    assertNull(reader.readNext(true));
    assertNull(reader.readNext(false));

    op = DLMTestUtil.getLogRecordInstance(2L);
    out.write(op);

    // reader is able to read first record
    LogRecord record = waitForNextRecord(reader);
    assertNotNull(record);
    assertEquals(1L, record.getTransactionId());
    DLMTestUtil.verifyLogRecord(record);

    assertNull(reader.readNext(true));

    out.close();
    reader.close();
    dlm.close();
}
 

@Test(timeout = 60000)
public void testDeleteLog() throws Exception {
    String name = "delete-log-should-delete-ledgers";
    DistributedLogManager dlm = createNewDLM(conf, name);
    long txid = 1;
    // Create the log and write some records
    BKSyncLogWriter writer = (BKSyncLogWriter) dlm.startLogSegmentNonPartitioned();
    for (long j = 1; j <= DEFAULT_SEGMENT_SIZE; j++) {
        writer.write(DLMTestUtil.getLogRecordInstance(txid++));
    }
    BKLogSegmentWriter perStreamLogWriter = writer.getCachedLogWriter();
    writer.closeAndComplete();
    BKLogWriteHandler blplm = ((BKDistributedLogManager) (dlm)).createWriteHandler(true);
    assertNotNull(zkc.exists(blplm.completedLedgerZNode(txid, txid - 1,
        perStreamLogWriter.getLogSegmentSequenceNumber()), false));
    Utils.ioResult(blplm.asyncClose());

    // Should be able to open the underline ledger using BK client
    long ledgerId = perStreamLogWriter.getLogSegmentId();
    BKNamespaceDriver driver = (BKNamespaceDriver) dlm.getNamespaceDriver();
    driver.getReaderBKC().get().openLedgerNoRecovery(ledgerId,
        BookKeeper.DigestType.CRC32, conf.getBKDigestPW().getBytes(UTF_8));
    // Delete the log and we shouldn't be able the open the ledger
    dlm.delete();
    try {
        driver.getReaderBKC().get().openLedgerNoRecovery(ledgerId,
            BookKeeper.DigestType.CRC32, conf.getBKDigestPW().getBytes(UTF_8));
        fail("Should fail to open ledger after we delete the log");
    } catch (BKException.BKNoSuchLedgerExistsException e) {
        // ignore
    }
    // delete again should not throw any exception
    try {
        dlm.delete();
    } catch (IOException ioe) {
        fail("Delete log twice should not throw any exception");
    }
}
 

private void prepareLogSegmentsNonPartitioned(String name,
                                              int numSegments, int numEntriesPerSegment) throws Exception {
    DistributedLogManager dlm = createNewDLM(conf, name);
    long txid = 1;
    for (int sid = 0; sid < numSegments; ++sid) {
        LogWriter out = dlm.startLogSegmentNonPartitioned();
        for (int eid = 0; eid < numEntriesPerSegment; ++eid) {
            LogRecord record = DLMTestUtil.getLargeLogRecordInstance(txid);
            out.write(record);
            ++txid;
        }
        out.close();
    }
    dlm.close();
}
 

@Test(timeout = 60000)
public void testCheckLogExists() throws Exception {
    String name = "distrlog-check-log-exists";
    DistributedLogManager dlm = createNewDLM(conf, name);

    long txid = 1;
    LogWriter writer = dlm.startLogSegmentNonPartitioned();
    for (long j = 1; j <= DEFAULT_SEGMENT_SIZE / 2; j++) {
        writer.write(DLMTestUtil.getLogRecordInstance(txid++));
    }
    writer.flush();
    writer.commit();
    writer.close();
    dlm.close();

    URI uri = createDLMURI("/" + name);
    Namespace namespace = NamespaceBuilder.newBuilder()
            .conf(conf).uri(uri).build();
    assertTrue(namespace.logExists(name));
    assertFalse(namespace.logExists("non-existent-log"));
    URI nonExistentUri = createDLMURI("/" + "non-existent-ns");
    Namespace nonExistentNS = NamespaceBuilder.newBuilder()
            .conf(conf).uri(nonExistentUri).build();
    assertFalse(nonExistentNS.logExists(name));

    int logCount = 0;
    Iterator<String> logIter = namespace.getLogs();
    while (logIter.hasNext()) {
        String log = logIter.next();
        logCount++;
        assertEquals(name, log);
    }
    assertEquals(1, logCount);

    namespace.close();
}
 

@Test(timeout = 60000)
public void testSimpleRead() throws Exception {
    String name = "distrlog-simpleread";
    DistributedLogManager dlm = createNewDLM(conf, name);
    final long numTransactions = 10000;
    BKSyncLogWriter out = (BKSyncLogWriter) dlm.startLogSegmentNonPartitioned();
    for (long i = 1; i <= numTransactions; i++) {
        LogRecord op = DLMTestUtil.getLogRecordInstance(i);
        out.write(op);
    }
    out.closeAndComplete();

    assertEquals(numTransactions, DLMTestUtil.getNumberofLogRecords(createNewDLM(conf, name), 1));
    dlm.close();
}
 

@Test(timeout = 60000)
public void testNumberOfTransactions() throws Exception {
    String name = "distrlog-txncount";
    DistributedLogManager dlm = createNewDLM(conf, name);
    BKSyncLogWriter out = (BKSyncLogWriter) dlm.startLogSegmentNonPartitioned();
    for (long i = 1; i <= 100; i++) {
        LogRecord op = DLMTestUtil.getLogRecordInstance(i);
        out.write(op);
    }
    out.closeAndComplete();

    long numTrans = DLMTestUtil.getNumberofLogRecords(createNewDLM(conf, name), 1);
    assertEquals(100, numTrans);
    dlm.close();
}
 

@Test(timeout = 60000)
public void testCompleteLogSegmentConflicts() throws Exception {
    URI uri = createURI();
    String streamName = testName.getMethodName();
    DistributedLogConfiguration conf = new DistributedLogConfiguration()
            .setLockTimeout(99999)
            .setOutputBufferSize(0)
            .setImmediateFlushEnabled(true)
            .setEnableLedgerAllocatorPool(true)
            .setLedgerAllocatorPoolName("test");
    Namespace namespace = NamespaceBuilder.newBuilder().conf(conf).uri(uri).build();

    namespace.createLog(streamName);
    DistributedLogManager dlm1 = namespace.openLog(streamName);
    DistributedLogManager dlm2 = namespace.openLog(streamName);

    // dlm1 is writing
    BKSyncLogWriter out1 = (BKSyncLogWriter) dlm1.startLogSegmentNonPartitioned();
    out1.write(DLMTestUtil.getLogRecordInstance(1));
    // before out1 complete, out2 is in on recovery
    // it completed the log segments which bump the version of /ledgers znode
    BKAsyncLogWriter out2 = (BKAsyncLogWriter) dlm2.startAsyncLogSegmentNonPartitioned();

    try {
        out1.closeAndComplete();
        fail("Should fail closeAndComplete since other people already completed it.");
    } catch (IOException ioe) {
    }
}
 

/**
 * Create Log Segment for an pre-create stream. No max ledger sequence number recorded.
 */
@Test(timeout = 60000)
public void testCreateLogSegmentOnPrecreatedStream() throws Exception {
    URI uri = createURI();
    String streamName = testName.getMethodName();
    DistributedLogConfiguration conf = new DistributedLogConfiguration()
            .setLockTimeout(99999)
            .setOutputBufferSize(0)
            .setImmediateFlushEnabled(true)
            .setEnableLedgerAllocatorPool(true)
            .setLedgerAllocatorPoolName("test");
    Namespace namespace = NamespaceBuilder.newBuilder().conf(conf).uri(uri).build();

    namespace.createLog(streamName);
    MaxLogSegmentSequenceNo max1 = getMaxLogSegmentSequenceNo(getZooKeeperClient(namespace), uri, streamName, conf);
    assertEquals(DistributedLogConstants.UNASSIGNED_LOGSEGMENT_SEQNO, max1.getSequenceNumber());
    DistributedLogManager dlm = namespace.openLog(streamName);
    final int numSegments = 3;
    for (int i = 0; i < numSegments; i++) {
        BKSyncLogWriter out = (BKSyncLogWriter) dlm.startLogSegmentNonPartitioned();
        out.write(DLMTestUtil.getLogRecordInstance(i));
        out.closeAndComplete();
    }
    MaxLogSegmentSequenceNo max2 = getMaxLogSegmentSequenceNo(getZooKeeperClient(namespace), uri, streamName, conf);
    assertEquals(3, max2.getSequenceNumber());
    dlm.close();
    namespace.close();
}
 

@Test(timeout = 60000)
public void deleteDuringRead() throws Exception {
    String name = "distrlog-delete-with-reader";
    DistributedLogManager dlm = createNewDLM(conf, name);

    long txid = 1;
    for (long i = 0; i < 3; i++) {
        long start = txid;
        BKSyncLogWriter writer = (BKSyncLogWriter) dlm.startLogSegmentNonPartitioned();
        for (long j = 1; j <= DEFAULT_SEGMENT_SIZE; j++) {
            writer.write(DLMTestUtil.getLogRecordInstance(txid++));
        }

        BKLogSegmentWriter perStreamLogWriter = writer.getCachedLogWriter();

        writer.closeAndComplete();
        BKLogWriteHandler blplm = ((BKDistributedLogManager) (dlm)).createWriteHandler(true);
        assertNotNull(zkc.exists(blplm.completedLedgerZNode(start, txid - 1,
                perStreamLogWriter.getLogSegmentSequenceNumber()), false));
        Utils.ioResult(blplm.asyncClose());
    }

    LogReader reader = dlm.getInputStream(1);
    LogRecord record = reader.readNext(false);
    assert (null != record);
    DLMTestUtil.verifyLogRecord(record);
    long lastTxId = record.getTransactionId();

    dlm.delete();

    boolean exceptionEncountered;
    try {
        record = reader.readNext(false);
        while (null != record) {
            DLMTestUtil.verifyLogRecord(record);
            assert (lastTxId < record.getTransactionId());
            lastTxId = record.getTransactionId();
            record = reader.readNext(false);
        }
        // make sure the exception is thrown from readahead
        while (true) {
            reader.readNext(false);
        }
    } catch (LogReadException | LogNotFoundException | DLIllegalStateException e) {
        exceptionEncountered = true;
    }
    assertTrue(exceptionEncountered);
    reader.close();
}
 

/**
 * Create Log Segment while max sequence number isn't match with list of log segments.
 */
@Test(timeout = 60000)
public void testCreateLogSegmentUnmatchMaxSequenceNumber() throws Exception {
    URI uri = createURI();
    String streamName = testName.getMethodName();
    DistributedLogConfiguration conf = new DistributedLogConfiguration()
            .setLockTimeout(99999)
            .setOutputBufferSize(0)
            .setImmediateFlushEnabled(true)
            .setEnableLedgerAllocatorPool(true)
            .setLedgerAllocatorPoolName("test");
    Namespace namespace = NamespaceBuilder.newBuilder().conf(conf).uri(uri).build();

    namespace.createLog(streamName);
    MaxLogSegmentSequenceNo max1 = getMaxLogSegmentSequenceNo(getZooKeeperClient(namespace), uri, streamName, conf);
    assertEquals(DistributedLogConstants.UNASSIGNED_LOGSEGMENT_SEQNO, max1.getSequenceNumber());
    DistributedLogManager dlm = namespace.openLog(streamName);
    final int numSegments = 3;
    for (int i = 0; i < numSegments; i++) {
        BKSyncLogWriter out = (BKSyncLogWriter) dlm.startLogSegmentNonPartitioned();
        out.write(DLMTestUtil.getLogRecordInstance(i));
        out.closeAndComplete();
    }
    MaxLogSegmentSequenceNo max2 = getMaxLogSegmentSequenceNo(getZooKeeperClient(namespace), uri, streamName, conf);
    assertEquals(3, max2.getSequenceNumber());

    // update the max ledger sequence number
    updateMaxLogSegmentSequenceNo(getZooKeeperClient(namespace), uri, streamName, conf,
            DLUtils.serializeLogSegmentSequenceNumber(99));

    DistributedLogManager dlm1 = namespace.openLog(streamName);
    try {
        BKSyncLogWriter out1 = (BKSyncLogWriter) dlm1.startLogSegmentNonPartitioned();
        out1.write(DLMTestUtil.getLogRecordInstance(numSegments + 1));
        out1.closeAndComplete();
        fail("Should fail creating new log segment when encountered unmatch max ledger sequence number");
    } catch (DLIllegalStateException lse) {
        // expected
    } finally {
        dlm1.close();
    }

    DistributedLogManager dlm2 = namespace.openLog(streamName);
    List<LogSegmentMetadata> segments = dlm2.getLogSegments();
    try {
        assertEquals(3, segments.size());
        assertEquals(1L, segments.get(0).getLogSegmentSequenceNumber());
        assertEquals(2L, segments.get(1).getLogSegmentSequenceNumber());
        assertEquals(3L, segments.get(2).getLogSegmentSequenceNumber());
    } finally {
        dlm2.close();
    }

    dlm.close();
    namespace.close();
}
 

@Test(timeout = 60000)
public void testCreateLogSegmentAfterLoseLock() throws Exception {
    URI uri = createDLMURI("/LogSegmentCreation");
    String name = "distrlog-createlogsegment-afterloselock";
    DistributedLogConfiguration conf = new DistributedLogConfiguration()
            .setLockTimeout(99999)
            .setOutputBufferSize(0)
            .setImmediateFlushEnabled(true)
            .setEnableLedgerAllocatorPool(true)
            .setLedgerAllocatorPoolName("test");
    Namespace namespace = NamespaceBuilder.newBuilder()
            .conf(conf).uri(uri).build();
    DistributedLogManager dlm = namespace.openLog(name);
    final int numSegments = 3;
    for (int i = 0; i < numSegments; i++) {
        BKSyncLogWriter out = (BKSyncLogWriter) dlm.startLogSegmentNonPartitioned();
        out.write(DLMTestUtil.getLogRecordInstance(i));
        out.closeAndComplete();
    }

    List<LogSegmentMetadata> segments = dlm.getLogSegments();
    LOG.info("Segments : {}", segments);
    assertEquals(3, segments.size());

    final DistributedLogManager dlm1 = namespace.openLog(name);
    final DistributedLogManager dlm2 = namespace.openLog(name);

    BKAsyncLogWriter writer1 = (BKAsyncLogWriter) dlm1.startAsyncLogSegmentNonPartitioned();
    LOG.info("Created writer 1.");
    BKSyncLogWriter writer2 = (BKSyncLogWriter) dlm2.startLogSegmentNonPartitioned();
    LOG.info("Created writer 2.");
    writer2.write(DLMTestUtil.getLogRecordInstance(numSegments));
    writer2.closeAndComplete();

    try {
        Utils.ioResult(writer1.write(DLMTestUtil.getLogRecordInstance(numSegments + 1)));
        fail("Should fail on writing new log records.");
    } catch (Throwable t) {
        LOG.error("Failed to write entry : ", t);
    }

    segments = dlm.getLogSegments();

    boolean hasInprogress = false;
    boolean hasDuplicatedSegment = false;
    long nextSeqNo = segments.get(0).getLogSegmentSequenceNumber();
    for (int i = 1; i < segments.size(); i++) {
        LogSegmentMetadata segment = segments.get(i);
        assertTrue(segment.getLogSegmentSequenceNumber() >= nextSeqNo);
        if (segment.getLogSegmentSequenceNumber() == nextSeqNo) {
            hasDuplicatedSegment = true;
        }
        nextSeqNo = segment.getLogSegmentSequenceNumber();
        if (segment.isInProgress()) {
            hasInprogress = true;
        }
    }
    assertEquals(4, segments.size());
    assertFalse(hasInprogress);
    assertFalse(hasDuplicatedSegment);

    LOG.info("Segments : duplicated = {}, inprogress = {}, {}",
             new Object[] { hasDuplicatedSegment, hasInprogress, segments });

    dlm1.close();
    dlm2.close();
    dlm.close();

    namespace.close();
}