Java Code Examples for org.apache.bookkeeper.client.LedgerHandle#close()

@Test(timeout = 60000, expected = BKException.ZKException.class)
public void testOpenLedgerWhenZkClosed() throws Exception {
    ZooKeeperClient newZkc = TestZooKeeperClientBuilder.newBuilder()
            .name("zkc-openledger-when-zk-closed")
            .zkServers(zkServers)
            .build();
    BookKeeperClient newBkc = BookKeeperClientBuilder.newBuilder()
            .name("bkc-openledger-when-zk-closed")
            .zkc(newZkc)
            .ledgersPath(ledgersPath)
            .dlConfig(conf)
            .build();
    try {
        LedgerHandle lh = newBkc.get().createLedger(BookKeeper.DigestType.CRC32, "zkcClosed".getBytes(UTF_8));
        lh.close();
        newZkc.close();
        LedgerHandleCache cache =
                LedgerHandleCache.newBuilder().bkc(newBkc).conf(conf).build();
        // open ledger after zkc closed
        cache.openLedger(new LogSegmentMetadata.LogSegmentMetadataBuilder("",
                2, lh.getId(), 1).setLogSegmentSequenceNo(lh.getId()).build(), false);
    } finally {
        newBkc.close();
    }
}
 

@Test
void whenWriteAndReadEntries_thenSuccess() throws Exception {
    LedgerHandle lh = createLedger(bk, "myledger", ledgerPassword);
    long start = System.currentTimeMillis();
    for (int i = 0; i < 1000; i++) {
        byte[] data = new String("message-" + i).getBytes();
        lh.append(data);
    }
    lh.close();
    long elapsed = System.currentTimeMillis() - start;
    LOG.info("Entries added to ledgerId " + lh.getId() + ", elapsed=" + elapsed);

    Long ledgerId = findLedgerByName(bk, "myledger").orElse(null);
    assertNotNull(ledgerId);
    lh = bk.openLedger(ledgerId, BookKeeper.DigestType.MAC, ledgerPassword);
    long lastId = lh.readLastConfirmed();
    Enumeration<LedgerEntry> entries = lh.readEntries(0, lastId);
    while (entries.hasMoreElements()) {
        LedgerEntry entry = entries.nextElement();
        String msg = new String(entry.getEntry());
        LOG.info("Entry: id=" + entry.getEntryId() + ", data=" + msg);
    }
}
 

@Override
protected int runCmd() throws Exception {
    LedgerHandle lh = getBookKeeperClient().get().openLedgerNoRecovery(
            getLedgerID(), BookKeeper.DigestType.CRC32, dlConf.getBKDigestPW().getBytes(UTF_8));
    final CountDownLatch doneLatch = new CountDownLatch(1);
    final AtomicInteger resultHolder = new AtomicInteger(-1234);
    BookkeeperInternalCallbacks.GenericCallback<Void> recoverCb =
            new BookkeeperInternalCallbacks.GenericCallback<Void>() {
        @Override
        public void operationComplete(int rc, Void result) {
            resultHolder.set(rc);
            doneLatch.countDown();
        }
    };
    try {
        BookKeeperAccessor.forceRecoverLedger(lh, recoverCb);
        doneLatch.await();
        if (BKException.Code.OK != resultHolder.get()) {
            throw BKException.create(resultHolder.get());
        }
    } finally {
        lh.close();
    }
    return 0;
}
 

@Override
protected int runCmd() throws Exception {
    LedgerHandle lh = getBookKeeperClient().get().openLedgerNoRecovery(
            getLedgerID(), BookKeeper.DigestType.CRC32, dlConf.getBKDigestPW().getBytes(UTF_8));
    final CountDownLatch doneLatch = new CountDownLatch(1);
    final AtomicInteger resultHolder = new AtomicInteger(-1234);
    BookkeeperInternalCallbacks.GenericCallback<Void> recoverCb =
            new BookkeeperInternalCallbacks.GenericCallback<Void>() {
        @Override
        public void operationComplete(int rc, Void result) {
            resultHolder.set(rc);
            doneLatch.countDown();
        }
    };
    try {
        BookKeeperAccessor.forceRecoverLedger(lh, recoverCb);
        doneLatch.await();
        if (BKException.Code.OK != resultHolder.get()) {
            throw BKException.create(resultHolder.get());
        }
    } finally {
        lh.close();
    }
    return 0;
}
 

@Override
protected int runCmd() throws Exception {
    LedgerHandle lh = getBookKeeperClient().get().openLedgerNoRecovery(
            getLedgerID(), BookKeeper.DigestType.CRC32, dlConf.getBKDigestPW().getBytes(UTF_8));
    try {
        long lac = lh.readLastConfirmed();
        System.out.println("LastAddConfirmed: " + lac);
    } finally {
        lh.close();
    }
    return 0;
}
 

@Override
protected int runCmd() throws Exception {
    LedgerHandle lh = getBookKeeperClient().get().openLedgerNoRecovery(getLedgerID(), BookKeeper.DigestType.CRC32,
            dlConf.getBKDigestPW().getBytes(UTF_8));
    try {
        if (null == fromEntryId) {
            fromEntryId = 0L;
        }
        if (null == untilEntryId) {
            untilEntryId = lh.readLastConfirmed();
        }
        if (untilEntryId >= fromEntryId) {
            if (readAllBookies) {
                LedgerReader lr = new LedgerReader(getBookKeeperClient().get());
                if (readLac) {
                    readLacsFromAllBookies(lr, lh, fromEntryId, untilEntryId);
                } else {
                    readEntriesFromAllBookies(lr, lh, fromEntryId, untilEntryId);
                }
            } else {
                simpleReadEntries(lh, fromEntryId, untilEntryId);
            }
        } else {
            System.out.println("No entries.");
        }
    } finally {
        lh.close();
    }
    return 0;
}
 

@Override
protected int runCmd() throws Exception {
    LedgerHandle lh = getBookKeeperClient().get()
            .openLedgerNoRecovery(getLedgerID(), BookKeeper.DigestType.CRC32,
                    dlConf.getBKDigestPW().getBytes(UTF_8));
    try {
        if (null == fromEntryId) {
            fromEntryId = 0L;
        }
        if (null == untilEntryId) {
            untilEntryId = lh.readLastConfirmed();
        }
        if (untilEntryId >= fromEntryId) {
            if (readAllBookies) {
                LedgerReader lr = new LedgerReader(getBookKeeperClient().get());
                if (readLac) {
                    readLacsFromAllBookies(lr, lh, fromEntryId, untilEntryId);
                } else {
                    readEntriesFromAllBookies(lr, lh, fromEntryId, untilEntryId);
                }
            } else {
                simpleReadEntries(lh, fromEntryId, untilEntryId);
            }
        } else {
            System.out.println("No entries.");
        }
    } finally {
        lh.close();
    }
    return 0;
}
 

@Test(timeout = 60000)
public void testBadVersionOnTwoAllocators() throws Exception {
    String allocationPath = "/allocation-bad-version";
    zkc.get().create(allocationPath, new byte[0], ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
    Stat stat = new Stat();
    byte[] data = zkc.get().getData(allocationPath, false, stat);
    Versioned<byte[]> allocationData = new Versioned<byte[]>(data, new ZkVersion(stat.getVersion()));

    SimpleLedgerAllocator allocator1 =
            new SimpleLedgerAllocator(allocationPath, allocationData, newQuorumConfigProvider(dlConf), zkc, bkc);
    SimpleLedgerAllocator allocator2 =
            new SimpleLedgerAllocator(allocationPath, allocationData, newQuorumConfigProvider(dlConf), zkc, bkc);
    allocator1.allocate();
    // wait until allocated
    ZKTransaction txn1 = newTxn();
    LedgerHandle lh = FutureUtils.result(allocator1.tryObtain(txn1, NULL_LISTENER));
    allocator2.allocate();
    ZKTransaction txn2 = newTxn();
    try {
        FutureUtils.result(allocator2.tryObtain(txn2, NULL_LISTENER));
        fail("Should fail allocating on second allocator as allocator1 is starting allocating something.");
    } catch (ZKException zke) {
        assertEquals(KeeperException.Code.BADVERSION, zke.getKeeperExceptionCode());
    }
    FutureUtils.result(txn1.execute());
    Utils.close(allocator1);
    Utils.close(allocator2);

    long eid = lh.addEntry("hello world".getBytes());
    lh.close();
    LedgerHandle readLh = bkc.get().openLedger(lh.getId(), BookKeeper.DigestType.CRC32, dlConf.getBKDigestPW().getBytes());
    Enumeration<LedgerEntry> entries = readLh.readEntries(eid, eid);
    int i = 0;
    while (entries.hasMoreElements()) {
        LedgerEntry entry = entries.nextElement();
        assertEquals("hello world", new String(entry.getEntry(), UTF_8));
        ++i;
    }
    assertEquals(1, i);
}
 

public static void fenceStream(DistributedLogConfiguration conf, URI uri, String name) throws Exception {
    BKDistributedLogManager dlm = (BKDistributedLogManager) createNewDLM(name, conf, uri);
    try {
        BKLogReadHandler readHandler = dlm.createReadHandler();
        List<LogSegmentMetadata> ledgerList = readHandler.getFullLedgerList(true, true);
        LogSegmentMetadata lastSegment = ledgerList.get(ledgerList.size() - 1);
        BookKeeperClient bkc = dlm.getWriterBKC();
        LedgerHandle lh = bkc.get().openLedger(lastSegment.getLedgerId(),
                BookKeeper.DigestType.CRC32, conf.getBKDigestPW().getBytes(UTF_8));
        lh.close();
    } finally {
        dlm.close();
    }
}
 

private ReadHandle buildReadHandle(int maxBlockSize, int blockCount) throws Exception {
    Assert.assertTrue(maxBlockSize > DataBlockHeaderImpl.getDataStartOffset());

    LedgerHandle lh = bk.createLedger(1,1,1, BookKeeper.DigestType.CRC32, "foobar".getBytes());

    int i = 0;
    int bytesWrittenCurrentBlock = DataBlockHeaderImpl.getDataStartOffset();
    int blocksWritten = 1;
    int entries = 0;

    while (blocksWritten < blockCount
           || bytesWrittenCurrentBlock < maxBlockSize/2) {
        byte[] entry = ("foobar"+i).getBytes();
        int sizeInBlock = entry.length + 12 /* ENTRY_HEADER_SIZE */;

        if (bytesWrittenCurrentBlock + sizeInBlock > maxBlockSize) {
            bytesWrittenCurrentBlock = DataBlockHeaderImpl.getDataStartOffset();
            blocksWritten++;
            entries = 0;
        }
        entries++;

        lh.addEntry(entry);
        bytesWrittenCurrentBlock += sizeInBlock;
        i++;
    }

    lh.close();

    return bk.newOpenLedgerOp().withLedgerId(lh.getId())
        .withPassword("foobar".getBytes()).withDigestType(DigestType.CRC32).execute().get();
}
 

@Test
public void testPlacement() throws Exception {
    for (int i = 0; i < NUM_BOOKIES; i++) {
        String bookie = bookies.get(i).getLocalAddress().toString();

        // Place bookie-1 in "rack-1" and the rest in "rack-2"
        int rackId = i == 0 ? 1 : 2;
        BookieInfo bi = new BookieInfo("rack-" + rackId, "bookie-" + (i + 1));
        log.info("setting rack for bookie at {} -- {}", bookie, bi);
        admin.bookies().updateBookieRackInfo(bookie, "default", bi);
    }

    // Make sure the racks cache gets updated through the ZK watch
    Thread.sleep(1000);

    BookKeeper bkc = this.pulsar.getBookKeeperClient();

    // Create few ledgers and verify all of them should have a copy in the first bookie
    BookieSocketAddress fistBookie = bookies.get(0).getLocalAddress();
    for (int i = 0; i < 100; i++) {
        LedgerHandle lh = bkc.createLedger(2, 2, DigestType.DUMMY, new byte[0]);
        log.info("Ledger: {} -- Ensemble: {}", i, lh.getLedgerMetadata().getEnsembleAt(0));
        assertTrue(lh.getLedgerMetadata().getEnsembleAt(0).contains(fistBookie),
                "first bookie in rack 0 not included in ensemble");
        lh.close();
    }
}
 

@Test
void whenWriteEntries_thenSuccess() throws Exception {
    LedgerHandle lh = createLedger(bk, "myledger", ledgerPassword);
    long start = System.currentTimeMillis();
    for (int i = 0; i < 1000; i++) {
        byte[] data = new String("message-" + i).getBytes();
        lh.append(data);
    }
    lh.close();
    long elapsed = System.currentTimeMillis() - start;
    LOG.info("Entries added to ledgerId " + lh.getId() + ". count=1000, elapsed=" + elapsed);
}
 

@Override
protected int runCmd() throws Exception {
    LedgerHandle lh = getBookKeeperClient().get().openLedgerNoRecovery(
            getLedgerID(), BookKeeper.DigestType.CRC32, dlConf.getBKDigestPW().getBytes(UTF_8));
    try {
        long lac = lh.readLastConfirmed();
        System.out.println("LastAddConfirmed: " + lac);
    } finally {
        lh.close();
    }
    return 0;
}
 

@Test(timeout = 60000)
public void testCreateLogStreamWithDifferentReplicationFactor() throws Exception {
    String name = runtime.getMethodName();
    DistributedLogConfiguration confLocal = new DistributedLogConfiguration();
    confLocal.addConfiguration(testConf);
    confLocal.setOutputBufferSize(0);
    confLocal.setImmediateFlushEnabled(false);
    confLocal.setPeriodicFlushFrequencyMilliSeconds(0);

    ConcurrentBaseConfiguration baseConf = new ConcurrentConstConfiguration(confLocal);
    DynamicDistributedLogConfiguration dynConf = new DynamicDistributedLogConfiguration(baseConf);
    dynConf.setProperty(DistributedLogConfiguration.BKDL_BOOKKEEPER_ENSEMBLE_SIZE,
            DistributedLogConfiguration.BKDL_BOOKKEEPER_ENSEMBLE_SIZE_DEFAULT - 1);

    URI uri = createDLMURI("/" + name);
    ensureURICreated(uri);
    Namespace namespace = NamespaceBuilder.newBuilder()
            .conf(confLocal).uri(uri).build();

    // use the pool
    DistributedLogManager dlm = namespace.openLog(name + "-pool");
    AsyncLogWriter writer = dlm.startAsyncLogSegmentNonPartitioned();
    Utils.ioResult(writer.write(DLMTestUtil.getLogRecordInstance(1L)));
    List<LogSegmentMetadata> segments = dlm.getLogSegments();
    assertEquals(1, segments.size());
    long ledgerId = segments.get(0).getLogSegmentId();
    LedgerHandle lh = ((BKNamespaceDriver) namespace.getNamespaceDriver()).getReaderBKC().get()
            .openLedgerNoRecovery(ledgerId, BookKeeper.DigestType.CRC32, confLocal.getBKDigestPW().getBytes(UTF_8));
    LedgerMetadata metadata = BookKeeperAccessor.getLedgerMetadata(lh);
    assertEquals(DistributedLogConfiguration.BKDL_BOOKKEEPER_ENSEMBLE_SIZE_DEFAULT, metadata.getEnsembleSize());
    lh.close();
    Utils.close(writer);
    dlm.close();

    // use customized configuration
    dlm = namespace.openLog(
            name + "-custom",
            java.util.Optional.empty(),
            java.util.Optional.of(dynConf),
            java.util.Optional.empty());
    writer = dlm.startAsyncLogSegmentNonPartitioned();
    Utils.ioResult(writer.write(DLMTestUtil.getLogRecordInstance(1L)));
    segments = dlm.getLogSegments();
    assertEquals(1, segments.size());
    ledgerId = segments.get(0).getLogSegmentId();
    lh = ((BKNamespaceDriver) namespace.getNamespaceDriver()).getReaderBKC().get()
            .openLedgerNoRecovery(ledgerId, BookKeeper.DigestType.CRC32, confLocal.getBKDigestPW().getBytes(UTF_8));
    metadata = BookKeeperAccessor.getLedgerMetadata(lh);
    assertEquals(DistributedLogConfiguration.BKDL_BOOKKEEPER_ENSEMBLE_SIZE_DEFAULT - 1, metadata.getEnsembleSize());
    lh.close();
    Utils.close(writer);
    dlm.close();
    namespace.close();
}
 

@Test(timeout = 60000)
public void testBadVersionOnTwoAllocators() throws Exception {
    String allocationPath = "/allocation-bad-version";
    zkc.get().create(allocationPath, new byte[0], ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
    Stat stat = new Stat();
    byte[] data = zkc.get().getData(allocationPath, false, stat);
    Versioned<byte[]> allocationData = new Versioned<byte[]>(data, new LongVersion(stat.getVersion()));

    SimpleLedgerAllocator allocator1 =
            new SimpleLedgerAllocator(allocationPath, allocationData, newQuorumConfigProvider(dlConf), zkc, bkc);
    SimpleLedgerAllocator allocator2 =
            new SimpleLedgerAllocator(allocationPath, allocationData, newQuorumConfigProvider(dlConf), zkc, bkc);
    allocator1.allocate();
    // wait until allocated
    ZKTransaction txn1 = newTxn();
    LedgerHandle lh = Utils.ioResult(allocator1.tryObtain(txn1, NULL_LISTENER));
    allocator2.allocate();
    ZKTransaction txn2 = newTxn();
    try {
        Utils.ioResult(allocator2.tryObtain(txn2, NULL_LISTENER));
        fail("Should fail allocating on second allocator as allocator1 is starting allocating something.");
    } catch (ZKException ke) {
        assertEquals(KeeperException.Code.BADVERSION, ke.getKeeperExceptionCode());
    }
    Utils.ioResult(txn1.execute());
    Utils.close(allocator1);
    Utils.close(allocator2);

    long eid = lh.addEntry("hello world".getBytes());
    lh.close();
    LedgerHandle readLh = bkc.get().openLedger(lh.getId(),
            BookKeeper.DigestType.CRC32, dlConf.getBKDigestPW().getBytes());
    Enumeration<LedgerEntry> entries = readLh.readEntries(eid, eid);
    int i = 0;
    while (entries.hasMoreElements()) {
        LedgerEntry entry = entries.nextElement();
        assertEquals("hello world", new String(entry.getEntry(), UTF_8));
        ++i;
    }
    assertEquals(1, i);
}
 

@Test(timeout = 60000)
public void testCreateLogStreamWithDifferentReplicationFactor() throws Exception {
    String name = runtime.getMethodName();
    DistributedLogConfiguration confLocal = new DistributedLogConfiguration();
    confLocal.addConfiguration(testConf);
    confLocal.setOutputBufferSize(0);
    confLocal.setImmediateFlushEnabled(false);
    confLocal.setPeriodicFlushFrequencyMilliSeconds(0);

    ConcurrentBaseConfiguration baseConf = new ConcurrentConstConfiguration(confLocal);
    DynamicDistributedLogConfiguration dynConf = new DynamicDistributedLogConfiguration(baseConf);
    dynConf.setProperty(DistributedLogConfiguration.BKDL_BOOKKEEPER_ENSEMBLE_SIZE,
            DistributedLogConfiguration.BKDL_BOOKKEEPER_ENSEMBLE_SIZE_DEFAULT - 1);

    URI uri = createDLMURI("/" + name);
    ensureURICreated(uri);
    DistributedLogNamespace namespace = DistributedLogNamespaceBuilder.newBuilder()
            .conf(confLocal).uri(uri).build();

    // use the pool
    DistributedLogManager dlm = namespace.openLog(name + "-pool");
    AsyncLogWriter writer = dlm.startAsyncLogSegmentNonPartitioned();
    FutureUtils.result(writer.write(DLMTestUtil.getLogRecordInstance(1L)));
    List<LogSegmentMetadata> segments = dlm.getLogSegments();
    assertEquals(1, segments.size());
    long ledgerId = segments.get(0).getLedgerId();
    LedgerHandle lh = ((BKDistributedLogNamespace) namespace).getReaderBKC()
            .get().openLedgerNoRecovery(ledgerId, BookKeeper.DigestType.CRC32, confLocal.getBKDigestPW().getBytes(UTF_8));
    LedgerMetadata metadata = BookKeeperAccessor.getLedgerMetadata(lh);
    assertEquals(DistributedLogConfiguration.BKDL_BOOKKEEPER_ENSEMBLE_SIZE_DEFAULT, metadata.getEnsembleSize());
    lh.close();
    Utils.close(writer);
    dlm.close();

    // use customized configuration
    dlm = namespace.openLog(
            name + "-custom",
            Optional.<DistributedLogConfiguration>absent(),
            Optional.of(dynConf));
    writer = dlm.startAsyncLogSegmentNonPartitioned();
    FutureUtils.result(writer.write(DLMTestUtil.getLogRecordInstance(1L)));
    segments = dlm.getLogSegments();
    assertEquals(1, segments.size());
    ledgerId = segments.get(0).getLedgerId();
    lh = ((BKDistributedLogNamespace) namespace).getReaderBKC()
            .get().openLedgerNoRecovery(ledgerId, BookKeeper.DigestType.CRC32, confLocal.getBKDigestPW().getBytes(UTF_8));
    metadata = BookKeeperAccessor.getLedgerMetadata(lh);
    assertEquals(DistributedLogConfiguration.BKDL_BOOKKEEPER_ENSEMBLE_SIZE_DEFAULT - 1, metadata.getEnsembleSize());
    lh.close();
    Utils.close(writer);
    dlm.close();
    namespace.close();
}
 

@Test
public void testCleanupOldCompactedTopicLedger() throws Exception {
    BookKeeper bk = pulsar.getBookKeeperClientFactory().create(
            this.conf, null, Optional.empty(), null);

    LedgerHandle oldCompactedLedger = bk.createLedger(1, 1,
            Compactor.COMPACTED_TOPIC_LEDGER_DIGEST_TYPE,
            Compactor.COMPACTED_TOPIC_LEDGER_PASSWORD);
    oldCompactedLedger.close();
    LedgerHandle newCompactedLedger = bk.createLedger(1, 1,
            Compactor.COMPACTED_TOPIC_LEDGER_DIGEST_TYPE,
            Compactor.COMPACTED_TOPIC_LEDGER_PASSWORD);
    newCompactedLedger.close();

    // set the compacted topic ledger
    CompactedTopicImpl compactedTopic = new CompactedTopicImpl(bk);
    compactedTopic.newCompactedLedger(new PositionImpl(1,2), oldCompactedLedger.getId()).get();

    // ensure both ledgers still exist, can be opened
    bk.openLedger(oldCompactedLedger.getId(),
                  Compactor.COMPACTED_TOPIC_LEDGER_DIGEST_TYPE,
                  Compactor.COMPACTED_TOPIC_LEDGER_PASSWORD).close();
    bk.openLedger(newCompactedLedger.getId(),
                  Compactor.COMPACTED_TOPIC_LEDGER_DIGEST_TYPE,
                  Compactor.COMPACTED_TOPIC_LEDGER_PASSWORD).close();

    // update the compacted topic ledger
    compactedTopic.newCompactedLedger(new PositionImpl(1,2), newCompactedLedger.getId()).get();

    // old ledger should be deleted, new still there
    try {
        bk.openLedger(oldCompactedLedger.getId(),
                      Compactor.COMPACTED_TOPIC_LEDGER_DIGEST_TYPE,
                      Compactor.COMPACTED_TOPIC_LEDGER_PASSWORD).close();
        Assert.fail("Should have failed to open old ledger");
    } catch (BKException.BKNoSuchLedgerExistsException
        | BKException.BKNoSuchLedgerExistsOnMetadataServerException e) {
        // correct, expected behaviour
    }
    bk.openLedger(newCompactedLedger.getId(),
                  Compactor.COMPACTED_TOPIC_LEDGER_DIGEST_TYPE,
                  Compactor.COMPACTED_TOPIC_LEDGER_PASSWORD).close();
}