org.apache.bookkeeper.client.LedgerHandle Java Examples

The following examples show how to use org.apache.bookkeeper.client.LedgerHandle. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: DistributedLogTool.java    From distributedlog with Apache License 2.0 6 votes vote down vote up
private void simpleReadEntries(LedgerHandle lh, long fromEntryId, long untilEntryId) throws Exception {
    Enumeration<LedgerEntry> entries = lh.readEntries(fromEntryId, untilEntryId);
    long i = fromEntryId;
    System.out.println("Entries:");
    while (entries.hasMoreElements()) {
        LedgerEntry entry = entries.nextElement();
        System.out.println("\t" + i  + "(eid=" + entry.getEntryId() + ")\t: ");
        Entry.Reader reader = Entry.newBuilder()
                .setLogSegmentInfo(0L, 0L)
                .setEntryId(entry.getEntryId())
                .setInputStream(entry.getEntryInputStream())
                .setEnvelopeEntry(LogSegmentMetadata.supportsEnvelopedEntries(metadataVersion))
                .buildReader();
        printEntry(reader);
        ++i;
    }
}
 
Example #2
Source File: BookKeeperEditLogOutputStream.java    From big-c with Apache License 2.0 6 votes vote down vote up
@Override
public void addComplete(int rc, LedgerHandle handle,
                        long entryId, Object ctx) {
  synchronized(this) {
    outstandingRequests.decrementAndGet();
    if (!transmitResult.compareAndSet(BKException.Code.OK, rc)) {
      LOG.warn("Tried to set transmit result to (" + rc + ") \""
          + BKException.getMessage(rc) + "\""
          + " but is already (" + transmitResult.get() + ") \""
          + BKException.getMessage(transmitResult.get()) + "\"");
    }
    CountDownLatch l = syncLatch;
    if (l != null) {
      l.countDown();
    }
  }
}
 
Example #3
Source File: TestRollLogSegments.java    From distributedlog with Apache License 2.0 6 votes vote down vote up
private void checkAndWaitWriterReaderPosition(BKLogSegmentWriter writer, long expectedWriterPosition,
                                              BKAsyncLogReaderDLSN reader, long expectedReaderPosition,
                                              LedgerHandle inspector, long expectedLac) throws Exception {
    while (getLedgerHandle(writer).getLastAddConfirmed() < expectedWriterPosition) {
        Thread.sleep(1000);
    }
    assertEquals(expectedWriterPosition, getLedgerHandle(writer).getLastAddConfirmed());
    assertEquals(expectedLac, inspector.readLastConfirmed());
    LedgerReadPosition readPosition = reader.bkLedgerManager.readAheadWorker.getNextReadAheadPosition();
    logger.info("ReadAhead moved read position {} : ", readPosition);
    while (readPosition.getEntryId() < expectedReaderPosition) {
        Thread.sleep(1000);
        readPosition = reader.bkLedgerManager.readAheadWorker.getNextReadAheadPosition();
        logger.info("ReadAhead moved read position {} : ", readPosition);
    }
    assertEquals(expectedReaderPosition, readPosition.getEntryId());
}
 
Example #4
Source File: BookKeeperEditLogInputStream.java    From big-c with Apache License 2.0 6 votes vote down vote up
/**
 * Construct BookKeeper edit log input stream. 
 * Starts reading from firstBookKeeperEntry. This allows the stream
 * to take a shortcut during recovery, as it doesn't have to read
 * every edit log transaction to find out what the last one is.
 */
BookKeeperEditLogInputStream(LedgerHandle lh, EditLogLedgerMetadata metadata,
                             long firstBookKeeperEntry) 
    throws IOException {
  this.lh = lh;
  this.firstTxId = metadata.getFirstTxId();
  this.lastTxId = metadata.getLastTxId();
  this.logVersion = metadata.getDataLayoutVersion();
  this.inProgress = metadata.isInProgress();

  if (firstBookKeeperEntry < 0
      || firstBookKeeperEntry > lh.getLastAddConfirmed()) {
    throw new IOException("Invalid first bk entry to read: "
        + firstBookKeeperEntry + ", LAC: " + lh.getLastAddConfirmed());
  }
  BufferedInputStream bin = new BufferedInputStream(
      new LedgerInputStream(lh, firstBookKeeperEntry));
  tracker = new FSEditLogLoader.PositionTrackingInputStream(bin);
  DataInputStream in = new DataInputStream(tracker);

  reader = new FSEditLogOp.Reader(in, tracker, logVersion);
}
 
Example #5
Source File: BKLogSegmentWriter.java    From distributedlog with Apache License 2.0 6 votes vote down vote up
private void closeLedgerOnClose(final boolean abort,
                                final AtomicReference<Throwable> throwExc,
                                final Promise<Void> closePromise) {
    // close the log segment if it isn't in error state, so all the outstanding addEntry(s) will callback.
    if (null == throwExc.get() && !isLogSegmentInError()) {
        // Synchronous closing the ledger handle, if we couldn't close a ledger handle successfully.
        // we should throw the exception to #closeToFinalize, so it would fail completing a log segment.
        entryWriter.asyncClose(new CloseCallback() {
            @Override
            public void closeComplete(int rc, LedgerHandle lh, Object ctx) {
                if (BKException.Code.OK != rc && BKException.Code.LedgerClosedException != rc) {
                    if (!abort) {
                        throwExc.set(new IOException("Failed to close ledger for " + fullyQualifiedLogSegment + " : " +
                                BKException.getMessage(rc)));
                    }
                }
                completeClosePromise(abort, throwExc, closePromise);
            }
        }, null);
    } else {
        completeClosePromise(abort, throwExc, closePromise);
    }
}
 
Example #6
Source File: SimpleLedgerAllocator.java    From distributedlog with Apache License 2.0 6 votes vote down vote up
@Override
public synchronized CompletableFuture<LedgerHandle> tryObtain(final Transaction<Object> txn,
                                                              final OpListener<LedgerHandle> listener) {
    if (Phase.ERROR == phase) {
        return FutureUtils.exception(new AllocationException(Phase.ERROR,
                "Error on allocating ledger under " + allocatePath));
    }
    if (Phase.HANDING_OVER == phase || Phase.HANDED_OVER == phase || null != tryObtainTxn) {
        return FutureUtils.exception(new ConcurrentObtainException(phase,
                "Ledger handle is handling over to another thread : " + phase));
    }
    tryObtainTxn = txn;
    tryObtainListener = listener;
    if (null != allocatedLh) {
        completeAllocation(allocatedLh);
    }
    return allocatePromise;
}
 
Example #7
Source File: SimpleLedgerAllocator.java    From distributedlog with Apache License 2.0 6 votes vote down vote up
private void markAsAllocated(final LedgerHandle lh) {
    byte[] data = DLUtils.logSegmentId2Bytes(lh.getId());
    Utils.zkSetData(zkc, allocatePath, data, getVersion())
        .whenComplete(new FutureEventListener<LongVersion>() {
            @Override
            public void onSuccess(LongVersion version) {
                // we only issue deleting ledger left from previous allocation when we could allocate first ledger
                // as zookeeper version could prevent us doing stupid things.
                deleteLedgerLeftFromPreviousAllocationIfNecessary();
                setVersion(version);
                setPhase(Phase.ALLOCATED);
                // complete the allocation after it is marked as allocated
                completeAllocation(lh);
            }

            @Override
            public void onFailure(Throwable cause) {
                setPhase(Phase.ERROR);
                deleteLedger(lh.getId());
                LOG.error("Fail mark ledger {} as allocated under {} : ",
                        new Object[] { lh.getId(), allocatePath, cause });
                // fail the allocation since failed to mark it as allocated
                failAllocation(cause);
            }
        });
}
 
Example #8
Source File: BKLogSegmentWriter.java    From distributedlog with Apache License 2.0 6 votes vote down vote up
private void closeLedgerOnClose(final boolean abort,
                                final AtomicReference<Throwable> throwExc,
                                final CompletableFuture<Void> closePromise) {
    // close the log segment if it isn't in error state, so all the outstanding addEntry(s) will callback.
    if (null == throwExc.get() && !isLogSegmentInError()) {
        // Synchronous closing the ledger handle, if we couldn't close a ledger handle successfully.
        // we should throw the exception to #closeToFinalize, so it would fail completing a log segment.
        entryWriter.asyncClose(new CloseCallback() {
            @Override
            public void closeComplete(int rc, LedgerHandle lh, Object ctx) {
                if (BKException.Code.OK != rc && BKException.Code.LedgerClosedException != rc) {
                    if (!abort) {
                        throwExc.set(new IOException("Failed to close ledger for "
                                + fullyQualifiedLogSegment + " : " + BKException.getMessage(rc)));
                    }
                }
                completeClosePromise(abort, throwExc, closePromise);
            }
        }, null);
    } else {
        completeClosePromise(abort, throwExc, closePromise);
    }
}
 
Example #9
Source File: LedgerStreamReader.java    From distributedlog with Apache License 2.0 6 votes vote down vote up
@Override
public void readComplete(int rc, LedgerHandle lh, Enumeration<LedgerEntry> enumeration, Object ctx) {
    this.rc = rc;
    if (BKException.Code.OK == rc && enumeration.hasMoreElements()) {
        entry = enumeration.nextElement();
    } else {
        entry = null;
    }
    isDone = true;
    // construct a new read request
    long nextEntry = nextReadEntry.getAndIncrement();
    if (nextEntry <= lac) {
        PendingReadRequest nextRead =
                new PendingReadRequest(nextEntry);
        pendingReads.add(nextRead);
        nextRead.read();
    }
    triggerCallbacks();
}
 
Example #10
Source File: DistributedLogTool.java    From distributedlog with Apache License 2.0 6 votes vote down vote up
@Override
protected int runCmd() throws Exception {
    LedgerHandle lh = getBookKeeperClient().get().openLedgerNoRecovery(
            getLedgerID(), BookKeeper.DigestType.CRC32, dlConf.getBKDigestPW().getBytes(UTF_8));
    final CountDownLatch doneLatch = new CountDownLatch(1);
    final AtomicInteger resultHolder = new AtomicInteger(-1234);
    BookkeeperInternalCallbacks.GenericCallback<Void> recoverCb =
            new BookkeeperInternalCallbacks.GenericCallback<Void>() {
        @Override
        public void operationComplete(int rc, Void result) {
            resultHolder.set(rc);
            doneLatch.countDown();
        }
    };
    try {
        BookKeeperAccessor.forceRecoverLedger(lh, recoverCb);
        doneLatch.await();
        if (BKException.Code.OK != resultHolder.get()) {
            throw BKException.create(resultHolder.get());
        }
    } finally {
        lh.close();
    }
    return 0;
}
 
Example #11
Source File: TestLedgerHandleCache.java    From distributedlog with Apache License 2.0 6 votes vote down vote up
@Test(timeout = 60000)
public void testOpenAndCloseLedger() throws Exception {
    LedgerHandle lh = bkc.get().createLedger(1, 1, 1,
            BookKeeper.DigestType.CRC32, conf.getBKDigestPW().getBytes(UTF_8));
    LedgerHandleCache cache =
            LedgerHandleCache.newBuilder().bkc(bkc).conf(conf).build();
    LogSegmentMetadata segment = new LogSegmentMetadata.LogSegmentMetadataBuilder(
            "/data", LogSegmentMetadata.LogSegmentMetadataVersion.VERSION_V5_SEQUENCE_ID, lh.getId(), 0L)
            .build();
    LedgerDescriptor desc1 = cache.openLedger(segment, false);
    assertTrue(cache.handlesMap.containsKey(desc1));
    LedgerHandleCache.RefCountedLedgerHandle refLh = cache.handlesMap.get(desc1);
    assertEquals(1, refLh.getRefCount());
    cache.openLedger(segment, false);
    assertTrue(cache.handlesMap.containsKey(desc1));
    assertEquals(2, refLh.getRefCount());
    // close the ledger
    cache.closeLedger(desc1);
    assertTrue(cache.handlesMap.containsKey(desc1));
    assertEquals(1, refLh.getRefCount());
    cache.closeLedger(desc1);
    assertFalse(cache.handlesMap.containsKey(desc1));
    assertEquals(0, refLh.getRefCount());
}
 
Example #12
Source File: TestBKLogSegmentWriter.java    From distributedlog with Apache License 2.0 6 votes vote down vote up
private BKLogSegmentWriter createLogSegmentWriter(DistributedLogConfiguration conf,
                                                  long logSegmentSequenceNumber,
                                                  long startTxId,
                                                  ZKDistributedLock lock) throws Exception {
    LedgerHandle lh = bkc.get().createLedger(3, 2, 2,
            BookKeeper.DigestType.CRC32, conf.getBKDigestPW().getBytes(UTF_8));
    return new BKLogSegmentWriter(
            runtime.getMethodName(),
            runtime.getMethodName(),
            conf,
            LogSegmentMetadata.LEDGER_METADATA_CURRENT_LAYOUT_VERSION,
            new BKLogSegmentEntryWriter(lh),
            lock,
            startTxId,
            logSegmentSequenceNumber,
            scheduler,
            NullStatsLogger.INSTANCE,
            NullStatsLogger.INSTANCE,
            new AlertStatsLogger(NullStatsLogger.INSTANCE, "test"),
            PermitLimiter.NULL_PERMIT_LIMITER,
            new SettableFeatureProvider("", 0),
            ConfUtils.getConstDynConf(conf));
}
 
Example #13
Source File: TestLedgerAllocator.java    From distributedlog with Apache License 2.0 6 votes vote down vote up
/**
 * {@link https://issues.apache.org/jira/browse/DL-26}.
 */
@DistributedLogAnnotations.FlakyTest
@Ignore
@Test(timeout = 60000)
public void testCloseAllocatorAfterConfirm() throws Exception {
    String allocationPath = "/allocation2";
    SimpleLedgerAllocator allocator = createAllocator(allocationPath);
    allocator.allocate();
    ZKTransaction txn = newTxn();
    // close during obtaining ledger.
    LedgerHandle lh = Utils.ioResult(allocator.tryObtain(txn, NULL_LISTENER));
    Utils.ioResult(txn.execute());
    Utils.close(allocator);
    byte[] data = zkc.get().getData(allocationPath, false, null);
    assertEquals(0, data.length);
    // the ledger is not deleted.
    bkc.get().openLedger(lh.getId(), BookKeeper.DigestType.CRC32,
            dlConf.getBKDigestPW().getBytes(UTF_8));
}
 
Example #14
Source File: TwoPhaseCompactor.java    From pulsar with Apache License 2.0 6 votes vote down vote up
private CompletableFuture<LedgerHandle> createLedger(BookKeeper bk, Map<String,byte[]> metadata) {
    CompletableFuture<LedgerHandle> bkf = new CompletableFuture<>();

    try {
        bk.asyncCreateLedger(conf.getManagedLedgerDefaultEnsembleSize(),
                conf.getManagedLedgerDefaultWriteQuorum(),
                conf.getManagedLedgerDefaultAckQuorum(),
                Compactor.COMPACTED_TOPIC_LEDGER_DIGEST_TYPE,
                Compactor.COMPACTED_TOPIC_LEDGER_PASSWORD,
                (rc, ledger, ctx) -> {
                    if (rc != BKException.Code.OK) {
                        bkf.completeExceptionally(BKException.create(rc));
                    } else {
                        bkf.complete(ledger);
                    }
                }, null, metadata);
    } catch (Throwable t) {
        log.error("Encountered unexpected error when creating compaction ledger", t);
        return FutureUtil.failedFuture(t);
    }
    return bkf;
}
 
Example #15
Source File: BkHelperLiveTest.java    From tutorials with MIT License 6 votes vote down vote up
@Test
void whenWriteAndReadEntries_thenSuccess() throws Exception {
    LedgerHandle lh = createLedger(bk, "myledger", ledgerPassword);
    long start = System.currentTimeMillis();
    for (int i = 0; i < 1000; i++) {
        byte[] data = new String("message-" + i).getBytes();
        lh.append(data);
    }
    lh.close();
    long elapsed = System.currentTimeMillis() - start;
    LOG.info("Entries added to ledgerId " + lh.getId() + ", elapsed=" + elapsed);

    Long ledgerId = findLedgerByName(bk, "myledger").orElse(null);
    assertNotNull(ledgerId);
    lh = bk.openLedger(ledgerId, BookKeeper.DigestType.MAC, ledgerPassword);
    long lastId = lh.readLastConfirmed();
    Enumeration<LedgerEntry> entries = lh.readEntries(0, lastId);
    while (entries.hasMoreElements()) {
        LedgerEntry entry = entries.nextElement();
        String msg = new String(entry.getEntry());
        LOG.info("Entry: id=" + entry.getEntryId() + ", data=" + msg);
    }
}
 
Example #16
Source File: BKLogSegmentEntryReader.java    From distributedlog with Apache License 2.0 6 votes vote down vote up
void processReadEntries(int rc,
                        LedgerHandle lh,
                        Enumeration<LedgerEntry> entries,
                        Object ctx) {
    if (isDone()) {
        return;
    }
    if (!checkReturnCodeAndHandleFailure(rc, false)) {
        return;
    }
    LedgerEntry entry = null;
    while (entries.hasMoreElements()) {
        // more entries are returned
        if (null != entry) {
            completeExceptionally(BKException.Code.UnexpectedConditionException);
            return;
        }
        entry = entries.nextElement();
    }
    if (null == entry || entry.getEntryId() != entryId) {
        completeExceptionally(BKException.Code.UnexpectedConditionException);
        return;
    }
    complete(entry);
}
 
Example #17
Source File: ManagedLedgerImpl.java    From pulsar with Apache License 2.0 6 votes vote down vote up
@Override
public void rollCurrentLedgerIfFull() {
    log.info("[{}] Start checking if current ledger is full", name);
    if (currentLedgerEntries > 0 && currentLedgerIsFull()) {
        STATE_UPDATER.set(this, State.ClosingLedger);
        currentLedger.asyncClose(new AsyncCallback.CloseCallback() {
            @Override
            public void closeComplete(int rc, LedgerHandle lh, Object o) {
                checkArgument(currentLedger.getId() == lh.getId(), "ledgerId %s doesn't match with acked ledgerId %s",
                        currentLedger.getId(),
                        lh.getId());

                if (rc == BKException.Code.OK) {
                    log.debug("Successfuly closed ledger {}", lh.getId());
                } else {
                    log.warn("Error when closing ledger {}. Status={}", lh.getId(), BKException.getMessage(rc));
                }

                ledgerClosed(lh);
                createLedgerAfterClosed();
            }
        }, System.nanoTime());
    }
}
 
Example #18
Source File: BookkeeperSchemaStorage.java    From pulsar with Apache License 2.0 6 votes vote down vote up
@NotNull
private CompletableFuture<LedgerHandle> openLedger(Long ledgerId) {
    final CompletableFuture<LedgerHandle> future = new CompletableFuture<>();
    bookKeeper.asyncOpenLedger(
        ledgerId,
        BookKeeper.DigestType.fromApiDigestType(config.getManagedLedgerDigestType()),
        LedgerPassword,
        (rc, handle, ctx) -> {
            if (rc != BKException.Code.OK) {
                future.completeExceptionally(bkException("Failed to open ledger", rc, ledgerId, -1));
            } else {
                future.complete(handle);
            }
        }, null
    );
    return future;
}
 
Example #19
Source File: LedgerStreamReader.java    From distributedlog with Apache License 2.0 6 votes vote down vote up
public LedgerStreamReader(LedgerHandle lh,
                          ReadEntryListener readEntryListener,
                          int concurrency) {
    this.lh = lh;
    this.lac = lh.getLastAddConfirmed();
    this.readEntryListener = readEntryListener;
    this.concurrency = concurrency;
    for (int i = 0; i < concurrency; i++) {
        long entryId = nextReadEntry.getAndIncrement();
        if (entryId > lac) {
            break;
        }
        PendingReadRequest request = new PendingReadRequest(entryId);
        pendingReads.add(request);
        request.read();
    }
    if (pendingReads.isEmpty()) {
        done.countDown();
    }
}
 
Example #20
Source File: TestLedgerAllocator.java    From distributedlog with Apache License 2.0 5 votes vote down vote up
@Test(timeout = 60000)
public void testCloseAllocatorDuringObtaining() throws Exception {
    String allocationPath = "/allocation2";
    SimpleLedgerAllocator allocator = createAllocator(allocationPath);
    allocator.allocate();
    ZKTransaction txn = newTxn();
    // close during obtaining ledger.
    LedgerHandle lh = FutureUtils.result(allocator.tryObtain(txn, NULL_LISTENER));
    Utils.close(allocator);
    byte[] data = zkc.get().getData(allocationPath, false, null);
    assertEquals((Long) lh.getId(), Long.valueOf(new String(data, UTF_8)));
    // the ledger is not deleted
    bkc.get().openLedger(lh.getId(), BookKeeper.DigestType.CRC32,
            dlConf.getBKDigestPW().getBytes(UTF_8));
}
 
Example #21
Source File: LedgerBatchReader.java    From distributedlog with Apache License 2.0 5 votes vote down vote up
public LedgerBatchReader(LedgerHandle lh,
                         ReadEntryListener readEntryListener,
                         int batchSize) {
    this.lh = lh;
    this.batchSize = batchSize;
    this.readEntryListener = readEntryListener;
}
 
Example #22
Source File: CompactedTopicImpl.java    From pulsar with Apache License 2.0 5 votes vote down vote up
private static CompletableFuture<List<Entry>> readEntries(LedgerHandle lh, long from, long to) {
    CompletableFuture<Enumeration<LedgerEntry>> promise = new CompletableFuture<>();

    lh.asyncReadEntries(from, to,
                        (rc, _lh, seq, ctx) -> {
                            if (rc != BKException.Code.OK) {
                                promise.completeExceptionally(BKException.create(rc));
                            } else {
                                promise.complete(seq);
                            }
                        }, null);
    return promise.thenApply(
            (seq) -> {
                List<Entry> entries = new ArrayList<Entry>();
                while (seq.hasMoreElements()) {
                    ByteBuf buf = seq.nextElement().getEntryBuffer();
                    try (RawMessage m = RawMessageImpl.deserializeFrom(buf)) {
                        entries.add(EntryImpl.create(m.getMessageIdData().getLedgerId(),
                                                     m.getMessageIdData().getEntryId(),
                                                     m.getHeadersAndPayload()));
                    } finally {
                        buf.release();
                    }
                }
                return entries;
            });
}
 
Example #23
Source File: TestLedgerAllocator.java    From distributedlog with Apache License 2.0 5 votes vote down vote up
/**
 * {@link https://issues.apache.org/jira/browse/DL-43}.
 */
@DistributedLogAnnotations.FlakyTest
@Ignore
@Test(timeout = 60000)
public void testAllocation() throws Exception {
    String allocationPath = "/allocation1";
    SimpleLedgerAllocator allocator = createAllocator(allocationPath);
    allocator.allocate();
    ZKTransaction txn = newTxn();
    LedgerHandle lh = Utils.ioResult(allocator.tryObtain(txn, NULL_LISTENER));
    logger.info("Try obtaining ledger handle {}", lh.getId());
    byte[] data = zkc.get().getData(allocationPath, false, null);
    assertEquals((Long) lh.getId(), Long.valueOf(new String(data, UTF_8)));
    txn.addOp(DefaultZKOp.of(Op.setData("/unexistedpath", "data".getBytes(UTF_8), -1), null));
    try {
        Utils.ioResult(txn.execute());
        fail("Should fail the transaction when setting unexisted path");
    } catch (ZKException ke) {
        // expected
        logger.info("Should fail on executing transaction when setting unexisted path", ke);
    }
    data = zkc.get().getData(allocationPath, false, null);
    assertEquals((Long) lh.getId(), Long.valueOf(new String(data, UTF_8)));

    // Create new transaction to obtain the ledger again.
    txn = newTxn();
    // we could obtain the ledger if it was obtained
    LedgerHandle newLh = Utils.ioResult(allocator.tryObtain(txn, NULL_LISTENER));
    assertEquals(lh.getId(), newLh.getId());
    Utils.ioResult(txn.execute());
    data = zkc.get().getData(allocationPath, false, null);
    assertEquals(0, data.length);
    Utils.close(allocator);
}
 
Example #24
Source File: BookkeeperSchemaStorage.java    From pulsar with Apache License 2.0 5 votes vote down vote up
@NotNull
private CompletableFuture<Void> closeLedger(LedgerHandle ledgerHandle) {
    CompletableFuture<Void> future = new CompletableFuture<>();
    ledgerHandle.asyncClose((rc, handle, ctx) -> {
        if (rc != BKException.Code.OK) {
            future.completeExceptionally(bkException("Failed to close ledger", rc, ledgerHandle.getId(), -1));
        } else {
            future.complete(null);
        }
    }, null);
    return future;
}
 
Example #25
Source File: TwoPhaseCompactor.java    From pulsar with Apache License 2.0 5 votes vote down vote up
private CompletableFuture<Void> closeLedger(LedgerHandle lh) {
    CompletableFuture<Void> bkf = new CompletableFuture<>();
    lh.asyncClose((rc, ledger, ctx) -> {
            if (rc != BKException.Code.OK) {
                bkf.completeExceptionally(BKException.create(rc));
            } else {
                bkf.complete(null);
            }
        }, null);
    return bkf;
}
 
Example #26
Source File: BKUtils.java    From distributedlog with Apache License 2.0 5 votes vote down vote up
/**
 * Close a list of ledgers <i>lhs</i>.
 *
 * @param lhs a list of ledgers
 * @return future represents close results.
 */
public static CompletableFuture<Void> closeLedgers(LedgerHandle ... lhs) {
    List<CompletableFuture<Void>> closeResults = Lists.newArrayListWithExpectedSize(lhs.length);
    for (LedgerHandle lh : lhs) {
        closeResults.add(closeLedger(lh));
    }
    return FutureUtils.collect(closeResults).thenApply(VoidFunctions.LIST_TO_VOID_FUNC);
}
 
Example #27
Source File: BKLogSegmentAllocator.java    From distributedlog with Apache License 2.0 5 votes vote down vote up
@Override
public CompletableFuture<LogSegmentEntryWriter> tryObtain(Transaction<Object> txn, final
Transaction.OpListener<LogSegmentEntryWriter> listener) {
    return allocator.tryObtain(txn, new Transaction.OpListener<LedgerHandle>() {
        @Override
        public void onCommit(LedgerHandle lh) {
            listener.onCommit(new BKLogSegmentEntryWriter(lh));
        }

        @Override
        public void onAbort(Throwable t) {
            listener.onAbort(t);
        }
    }).thenApply(NewLogSegmentEntryWriterFn.INSTANCE);
}
 
Example #28
Source File: TwoPhaseCompactor.java    From pulsar with Apache License 2.0 5 votes vote down vote up
private CompletableFuture<Long> phaseTwoSeekThenLoop(RawReader reader, MessageId from, MessageId to,
        MessageId lastReadId, Map<String, MessageId> latestForKey, BookKeeper bk, LedgerHandle ledger) {
    CompletableFuture<Long> promise = new CompletableFuture<>();

    reader.seekAsync(from).thenCompose((v) -> {
        Semaphore outstanding = new Semaphore(MAX_OUTSTANDING);
        CompletableFuture<Void> loopPromise = new CompletableFuture<Void>();
        phaseTwoLoop(reader, to, latestForKey, ledger, outstanding, loopPromise);
        return loopPromise;
    }).thenCompose((v) -> closeLedger(ledger))
            .thenCompose((v) -> reader.acknowledgeCumulativeAsync(lastReadId,
                    ImmutableMap.of(COMPACTED_TOPIC_LEDGER_PROPERTY, ledger.getId())))
            .whenComplete((res, exception) -> {
                if (exception != null) {
                    deleteLedger(bk, ledger).whenComplete((res2, exception2) -> {
                        if (exception2 != null) {
                            log.warn("Cleanup of ledger {} for failed", ledger, exception2);
                        }
                        // complete with original exception
                        promise.completeExceptionally(exception);
                    });
                } else {
                    promise.complete(ledger.getId());
                }
            });
    return promise;
}
 
Example #29
Source File: BkHelperLiveTest.java    From tutorials with MIT License 5 votes vote down vote up
@Test
void whenWriteEntries_thenSuccess() throws Exception {
    LedgerHandle lh = createLedger(bk, "myledger", ledgerPassword);
    long start = System.currentTimeMillis();
    for (int i = 0; i < 1000; i++) {
        byte[] data = new String("message-" + i).getBytes();
        lh.append(data);
    }
    lh.close();
    long elapsed = System.currentTimeMillis() - start;
    LOG.info("Entries added to ledgerId " + lh.getId() + ". count=1000, elapsed=" + elapsed);
}
 
Example #30
Source File: BkHelper.java    From tutorials with MIT License 5 votes vote down vote up
/**
 * Creates a Ledger with the given name added as custom metadata
 * @param bk
 * @param name
 * @param password
 * @return
 */
public static LedgerHandle createLedger(BookKeeper bk, String name, byte[] password) {
    try {
        return bk.createLedger(3, 2, 2, DigestType.MAC, password, Collections.singletonMap("name", name.getBytes()));
    } catch (Exception ex) {
        throw new RuntimeException(ex);
    }
}