org.apache.bookkeeper.client.api.ReadHandle Java Examples

@Test
public void testDeleteOffloaded() throws Exception {
    ReadHandle readHandle = buildReadHandle(DEFAULT_BLOCK_SIZE, 1);
    UUID uuid = UUID.randomUUID();
    LedgerOffloader offloader = new BlobStoreManagedLedgerOffloader(blobStore, BUCKET, scheduler,
                                                             DEFAULT_BLOCK_SIZE, DEFAULT_READ_BUFFER_SIZE);

    // verify object exist after offload
    offloader.offload(readHandle, uuid, new HashMap<>()).get();
    Assert.assertTrue(blobStore.blobExists(BUCKET, BlobStoreManagedLedgerOffloader.dataBlockOffloadKey(readHandle.getId(), uuid)));
    Assert.assertTrue(blobStore.blobExists(BUCKET, BlobStoreManagedLedgerOffloader.indexBlockOffloadKey(readHandle.getId(), uuid)));

    // verify object deleted after delete
    offloader.deleteOffloaded(readHandle.getId(), uuid, Collections.emptyMap()).get();
    Assert.assertFalse(blobStore.blobExists(BUCKET, BlobStoreManagedLedgerOffloader.dataBlockOffloadKey(readHandle.getId(), uuid)));
    Assert.assertFalse(blobStore.blobExists(BUCKET, BlobStoreManagedLedgerOffloader.indexBlockOffloadKey(readHandle.getId(), uuid)));
}
 

@Override
public CompletableFuture<ReadHandle> readOffloaded(long ledgerId, UUID uid,
                                                   Map<String, String> offloadDriverMetadata) {
    String readBucket = getReadBucket(offloadDriverMetadata);
    BlobStore readBlobstore = getReadBlobStore(offloadDriverMetadata);

    CompletableFuture<ReadHandle> promise = new CompletableFuture<>();
    String key = dataBlockOffloadKey(ledgerId, uid);
    String indexKey = indexBlockOffloadKey(ledgerId, uid);
    scheduler.chooseThread(ledgerId).submit(() -> {
            try {
                promise.complete(BlobStoreBackedReadHandleImpl.open(scheduler.chooseThread(ledgerId),
                                                             readBlobstore,
                                                             readBucket, key, indexKey,
                                                             VERSION_CHECK,
                                                             ledgerId, readBufferSize));
            } catch (Throwable t) {
                log.error("Failed readOffloaded: ", t);
                promise.completeExceptionally(t);
            }
        });
    return promise;
}
 

@Test
public void testOnlyNegativeOnEOF() throws Exception {
    int ledgerId = 1;
    int entrySize = 10000;
    int lac = 0;

    Random r = new Random(0);
    ReadHandle readHandle = new MockReadHandle(ledgerId, entrySize, lac, () -> (byte)r.nextInt());

    int blockSize = DataBlockHeaderImpl.getDataStartOffset() + entrySize * 2;
    BlockAwareSegmentInputStreamImpl inputStream = new BlockAwareSegmentInputStreamImpl(readHandle, 0, blockSize);

    int bytesRead = 0;
    for (int i = 0; i < blockSize*2; i++) {
        int ret = inputStream.read();
        if (ret < 0) { // should only be EOF
            assertEquals(bytesRead, blockSize);
            break;
        } else {
            bytesRead++;
        }
    }
}
 

private ReadHandle buildReadHandle() throws Exception {

        lh = bk.createLedger(1,1,1, BookKeeper.DigestType.CRC32, "foobar".getBytes());

        int i = 0;
        int blocksWritten = 1;
        while (blocksWritten <= numberOfEntries) {
            byte[] entry = ("foobar"+i).getBytes();
            blocksWritten++;
            lh.addEntry(entry);
            i++;
        }
        lh.close();

        return bk.newOpenLedgerOp().withLedgerId(lh.getId())
                .withPassword("foobar".getBytes()).withDigestType(DigestType.CRC32).execute().get();
    }
 

public static ReadHandle open(ScheduledExecutorService executor,
                              BlobStore blobStore, String bucket, String key, String indexKey,
                              VersionCheck versionCheck,
                              long ledgerId, int readBufferSize)
        throws IOException {
    Blob blob = blobStore.getBlob(bucket, indexKey);
    versionCheck.check(indexKey, blob);
    OffloadIndexBlockBuilder indexBuilder = OffloadIndexBlockBuilder.create();
    OffloadIndexBlock index = indexBuilder.fromStream(blob.getPayload().openStream());

    BackedInputStream inputStream = new BlobStoreBackedInputStreamImpl(blobStore, bucket, key,
        versionCheck,
        index.getDataObjectLength(),
        readBufferSize);
    return new BlobStoreBackedReadHandleImpl(ledgerId, index, inputStream, executor);
}
 

@Override
public CompletableFuture<ReadHandle> readOffloaded(long ledgerId, UUID uuid, Map<String, String> offloadDriverMetadata) {

    CompletableFuture<ReadHandle> promise = new CompletableFuture<>();
    String storagePath = getStoragePath(storageBasePath, offloadDriverMetadata.get(MANAGED_LEDGER_NAME));
    String dataFilePath = getDataFilePath(storagePath, ledgerId, uuid);
    scheduler.chooseThread(ledgerId).submit(() -> {
        try {
            MapFile.Reader reader = new MapFile.Reader(new Path(dataFilePath),
                    configuration);
            promise.complete(FileStoreBackedReadHandleImpl.open(scheduler.chooseThread(ledgerId), reader, ledgerId));
        } catch (Throwable t) {
            log.error("Failed to open FileStoreBackedReadHandleImpl: ManagerLedgerName: {}, " +
                    "LegerId: {}, UUID: {}", offloadDriverMetadata.get(MANAGED_LEDGER_NAME), ledgerId, uuid, t);
            promise.completeExceptionally(t);
        }
    });
    return promise;
}
 

@Test
public void testOffloadFailInitDataBlockUpload() throws Exception {
    ReadHandle readHandle = buildReadHandle();
    UUID uuid = UUID.randomUUID();
    String failureString = "fail InitDataBlockUpload";

    // mock throw exception when initiateMultipartUpload
    try {

        BlobStore spiedBlobStore = mock(BlobStore.class, delegatesTo(blobStore));
        Mockito
            .doThrow(new RuntimeException(failureString))
            .when(spiedBlobStore).initiateMultipartUpload(any(), any(), any());

        LedgerOffloader offloader = new BlobStoreManagedLedgerOffloader(spiedBlobStore, BUCKET, scheduler,
                                                                 DEFAULT_BLOCK_SIZE, DEFAULT_READ_BUFFER_SIZE);
        offloader.offload(readHandle, uuid, new HashMap<>()).get();
        Assert.fail("Should throw exception when initiateMultipartUpload");
    } catch (Exception e) {
        // excepted
        Assert.assertTrue(e.getCause() instanceof RuntimeException);
        Assert.assertTrue(e.getCause().getMessage().contains(failureString));
        Assert.assertFalse(blobStore.blobExists(BUCKET, BlobStoreManagedLedgerOffloader.dataBlockOffloadKey(readHandle.getId(), uuid)));
        Assert.assertFalse(blobStore.blobExists(BUCKET, BlobStoreManagedLedgerOffloader.indexBlockOffloadKey(readHandle.getId(), uuid)));
    }
}
 

@Test
public void testOffloadFailDataBlockPartUpload() throws Exception {
    ReadHandle readHandle = buildReadHandle();
    UUID uuid = UUID.randomUUID();
    String failureString = "fail DataBlockPartUpload";

    // mock throw exception when uploadPart
    try {
        BlobStore spiedBlobStore = mock(BlobStore.class, delegatesTo(blobStore));
        Mockito
            .doThrow(new RuntimeException(failureString))
            .when(spiedBlobStore).uploadMultipartPart(any(), anyInt(), any());

        LedgerOffloader offloader = new BlobStoreManagedLedgerOffloader(spiedBlobStore, BUCKET, scheduler,
            DEFAULT_BLOCK_SIZE, DEFAULT_READ_BUFFER_SIZE);
        offloader.offload(readHandle, uuid, new HashMap<>()).get();
        Assert.fail("Should throw exception for when uploadPart");
    } catch (Exception e) {
        // excepted
        Assert.assertTrue(e.getCause() instanceof RuntimeException);
        Assert.assertTrue(e.getCause().getMessage().contains(failureString));
        Assert.assertFalse(blobStore.blobExists(BUCKET, BlobStoreManagedLedgerOffloader.dataBlockOffloadKey(readHandle.getId(), uuid)));
        Assert.assertFalse(blobStore.blobExists(BUCKET, BlobStoreManagedLedgerOffloader.indexBlockOffloadKey(readHandle.getId(), uuid)));
    }
}
 

@Test
public void testOffloadEmpty() throws Exception {
    CompletableFuture<LedgerEntries> noEntries = new CompletableFuture<>();
    noEntries.completeExceptionally(new BKException.BKReadException());

    ReadHandle readHandle = Mockito.mock(ReadHandle.class);
    Mockito.doReturn(-1L).when(readHandle).getLastAddConfirmed();
    Mockito.doReturn(noEntries).when(readHandle).readAsync(anyLong(), anyLong());
    Mockito.doReturn(0L).when(readHandle).getLength();
    Mockito.doReturn(true).when(readHandle).isClosed();
    Mockito.doReturn(1234L).when(readHandle).getId();

    UUID uuid = UUID.randomUUID();
    LedgerOffloader offloader = new BlobStoreManagedLedgerOffloader(blobStore, BUCKET, scheduler,
                                                             DEFAULT_BLOCK_SIZE, DEFAULT_READ_BUFFER_SIZE);
    try {
        offloader.offload(readHandle, uuid, new HashMap<>()).get();
        Assert.fail("Shouldn't have been able to offload");
    } catch (ExecutionException e) {
        Assert.assertEquals(e.getCause().getClass(), IllegalArgumentException.class);
    }
}
 

@Test
public void testOffloadFailPutIndexBlock() throws Exception {
    ReadHandle readHandle = buildReadHandle();
    UUID uuid = UUID.randomUUID();
    String failureString = "fail putObject";

    // mock throw exception when putObject
    try {
        BlobStore spiedBlobStore = mock(BlobStore.class, delegatesTo(blobStore));
        Mockito
            .doThrow(new RuntimeException(failureString))
            .when(spiedBlobStore).putBlob(any(), any());

        LedgerOffloader offloader = new BlobStoreManagedLedgerOffloader(spiedBlobStore, BUCKET, scheduler,
            DEFAULT_BLOCK_SIZE, DEFAULT_READ_BUFFER_SIZE);
        offloader.offload(readHandle, uuid, new HashMap<>()).get();

        Assert.fail("Should throw exception for when putObject for index block");
    } catch (Exception e) {
        // excepted
        Assert.assertTrue(e.getCause() instanceof RuntimeException);
        Assert.assertTrue(e.getCause().getMessage().contains(failureString));
        Assert.assertFalse(blobStore.blobExists(BUCKET, BlobStoreManagedLedgerOffloader.dataBlockOffloadKey(readHandle.getId(), uuid)));
        Assert.assertFalse(blobStore.blobExists(BUCKET, BlobStoreManagedLedgerOffloader.indexBlockOffloadKey(readHandle.getId(), uuid)));
    }
}
 

static ReadHandle getLedgerHandle() {
    final ReadHandle lh = mock(ReadHandle.class);
    final LedgerEntry ledgerEntry = mock(LedgerEntry.class, Mockito.CALLS_REAL_METHODS);
    doReturn(Unpooled.wrappedBuffer(new byte[10])).when(ledgerEntry).getEntryBuffer();
    doReturn((long) 10).when(ledgerEntry).getLength();

    doAnswer((invocation) -> {
            Object[] args = invocation.getArguments();
            long firstEntry = (Long) args[0];
            long lastEntry = (Long) args[1];

            Vector<LedgerEntry> entries = new Vector<LedgerEntry>();
            for (int i = 0; i <= (lastEntry - firstEntry); i++) {
                entries.add(ledgerEntry);
            }
            LedgerEntries ledgerEntries = mock(LedgerEntries.class);
            doAnswer((invocation2) -> entries.iterator()).when(ledgerEntries).iterator();
            return CompletableFuture.completedFuture(ledgerEntries);
        }).when(lh).readAsync(anyLong(), anyLong());

    return lh;
}
 

void invalidateLedgerHandle(ReadHandle ledgerHandle, Throwable t) {
    long ledgerId = ledgerHandle.getId();
    if (currentLedger != null && ledgerId != currentLedger.getId()) {
        // remove handle from ledger cache since we got a (read) error
        ledgerCache.remove(ledgerId);
        if (log.isDebugEnabled()) {
            log.debug("[{}] Removed ledger {} from cache (after read error)", name, ledgerId, t);
        }
    } else {
        if (log.isDebugEnabled()) {
            log.debug("[{}] Ledger that encountered read error is current ledger", name, t);
        }
    }
}
 

protected void asyncReadEntry(ReadHandle ledger, long firstEntry, long lastEntry, boolean isSlowestReader,
        OpReadEntry opReadEntry, Object ctx) {
    if (config.getReadEntryTimeoutSeconds() > 0) {
        // set readOpCount to uniquely validate if ReadEntryCallbackWrapper is already recycled
        long readOpCount = READ_OP_COUNT_UPDATER.incrementAndGet(this);
        long createdTime = System.nanoTime();
        ReadEntryCallbackWrapper readCallback = ReadEntryCallbackWrapper.create(name, ledger.getId(), firstEntry,
                opReadEntry, readOpCount, createdTime, ctx);
        lastReadCallback = readCallback;
        entryCache.asyncReadEntry(ledger, firstEntry, lastEntry, isSlowestReader, readCallback, readOpCount);
    } else {
        entryCache.asyncReadEntry(ledger, firstEntry, lastEntry, isSlowestReader, opReadEntry, ctx);
    }
}
 

@Override
public CompletableFuture<Void> offload(ReadHandle ledger,
                                       UUID uid,
                                       Map<String, String> extraMetadata) {
    CompletableFuture<Void> promise = new CompletableFuture<>();
    promise.completeExceptionally(new UnsupportedOperationException());
    return promise;
}
 

@Override
public void asyncReadEntry(ReadHandle lh, PositionImpl position, final ReadEntryCallback callback,
        final Object ctx) {
    try {
        asyncReadEntry0(lh, position, callback, ctx);
    } catch (Throwable t) {
        log.warn("failed to read entries for {}-{}", lh.getId(), position, t);
        // invalidate all entries related to ledger from the cache (it might happen if entry gets corrupt
        // (entry.data is already deallocate due to any race-condition) so, invalidate cache and next time read from
        // the bookie)
        invalidateAllEntries(lh.getId());
        callback.readEntryFailed(createManagedLedgerException(t), ctx);
    }
}
 

MockOffloadReadHandle(ReadHandle toCopy) throws Exception {
    id = toCopy.getId();
    long lac = toCopy.getLastAddConfirmed();
    try (LedgerEntries entries = toCopy.read(0, lac)) {
        for (LedgerEntry e : entries) {
            this.entries.add(e.getEntryBuffer().retainedSlice());
        }
    }
    metadata = new MockMetadata(toCopy.getLedgerMetadata());
}
 

protected void asyncReadEntry(ReadHandle ledger, PositionImpl position, ReadEntryCallback callback, Object ctx) {
    if (config.getReadEntryTimeoutSeconds() > 0) {
        // set readOpCount to uniquely validate if ReadEntryCallbackWrapper is already recycled
        long readOpCount = READ_OP_COUNT_UPDATER.incrementAndGet(this);
        long createdTime = System.nanoTime();
        ReadEntryCallbackWrapper readCallback = ReadEntryCallbackWrapper.create(name, position.getLedgerId(),
                position.getEntryId(), callback, readOpCount, createdTime, ctx);
        lastReadCallback = readCallback;
        entryCache.asyncReadEntry(ledger, position, readCallback, readOpCount);
    } else {
        entryCache.asyncReadEntry(ledger, position, callback, ctx);
    }
}
 

@Override
public CompletableFuture<ReadHandle> readOffloaded(long ledgerId, UUID uid,
                                                   Map<String, String> offloadDriverMetadata) {
    CompletableFuture<ReadHandle> promise = new CompletableFuture<>();
    promise.completeExceptionally(new UnsupportedOperationException());
    return promise;
}
 

@Override
public void asyncReadEntry(ReadHandle lh, PositionImpl position, AsyncCallbacks.ReadEntryCallback callback,
        Object ctx) {
    lh.readAsync(position.getEntryId(), position.getEntryId()).whenCompleteAsync(
            (ledgerEntries, exception) -> {
                if (exception != null) {
                    ml.invalidateLedgerHandle(lh, exception);
                    callback.readEntryFailed(createManagedLedgerException(exception), ctx);
                    return;
                }

                try {
                    Iterator<LedgerEntry> iterator = ledgerEntries.iterator();
                    if (iterator.hasNext()) {
                        LedgerEntry ledgerEntry = iterator.next();
                        EntryImpl returnEntry = EntryImpl.create(ledgerEntry);

                        mlFactoryMBean.recordCacheMiss(1, returnEntry.getLength());
                        ml.getMBean().addReadEntriesSample(1, returnEntry.getLength());
                        callback.readEntryComplete(returnEntry, ctx);
                    } else {
                        callback.readEntryFailed(new ManagedLedgerException("Could not read given position"), ctx);
                    }
                } finally {
                    ledgerEntries.close();
                }
            }, ml.getExecutor().chooseThread(ml.getName()));
}
 

@Override
public CompletableFuture<Void> offload(ReadHandle ledger,
                                       UUID uuid,
                                       Map<String, String> extraMetadata) {
    CompletableFuture<Void> promise = new CompletableFuture<>();
    if (offloads.putIfAbsent(ledger.getId(), uuid) == null) {
        promise.complete(null);
    } else {
        promise.completeExceptionally(new Exception("Already exists exception"));
    }
    return promise;
}
 

@Override
public CompletableFuture<ReadHandle> readOffloaded(long ledgerId, UUID uuid,
                                                   Map<String, String> offloadDriverMetadata) {
    CompletableFuture<ReadHandle> promise = new CompletableFuture<>();
    promise.completeExceptionally(new UnsupportedOperationException());
    return promise;
}
 

@Override
public CompletableFuture<Void> offload(ReadHandle ledger,
                                       UUID uuid,
                                       Map<String, String> extraMetadata) {
    return errorLedgers.thenCompose(
                    (errors) -> {
                        if (errors.contains(ledger.getId())) {
                            CompletableFuture<Void> future = new CompletableFuture<>();
                            future.completeExceptionally(new Exception("Some kind of error"));
                            return future;
                        } else {
                            return super.offload(ledger, uuid, extraMetadata);
                        }
                    });
}
 

@Override
public CompletableFuture<Void> offload(ReadHandle ledger,
                                       UUID uuid,
                                       Map<String, String> extraMetadata) {
    CompletableFuture<Void> promise = new CompletableFuture<>();
    try {
        offloads.put(uuid, new MockOffloadReadHandle(ledger));
        promise.complete(null);
    } catch (Exception e) {
        promise.completeExceptionally(e);
    }
    return promise;
}
 

public static void scanRawLedger(long ledgerId, long fromId, long toId, herddb.client.ClientConfiguration clientConfiguration,
        ZookeeperMetadataStorageManager metadataStorageManager, Consumer<LogEntryWithSequenceNumber> consumer) throws Exception {
    ClientConfiguration config = new ClientConfiguration();
    config.setZkServers(metadataStorageManager.getZkAddress());
    config.setZkTimeout(metadataStorageManager.getZkSessionTimeout());
    config.setZkLedgersRootPath(clientConfiguration.getString(ServerConfiguration.PROPERTY_BOOKKEEPER_LEDGERS_PATH,
            ServerConfiguration.PROPERTY_BOOKKEEPER_LEDGERS_PATH_DEFAULT));
    config.setEnableParallelRecoveryRead(true);
    config.setEnableDigestTypeAutodetection(true);

    try (org.apache.bookkeeper.client.api.BookKeeper bookKeeper = org.apache.bookkeeper.client.api.BookKeeper.newBuilder(config).build();) {
        try (ReadHandle lh = bookKeeper
                .newOpenLedgerOp()
                .withRecovery(false)
                .withLedgerId(ledgerId)
                .withPassword(BookkeeperCommitLog.SHARED_SECRET.getBytes(StandardCharsets.UTF_8))
                .execute()
                .get()) {
            long lastAddConfirmed = lh.readLastAddConfirmed();
            if (toId < 0) {
                toId = lastAddConfirmed;
            }
            LOG.log(Level.INFO, "Scanning Ledger {0} from {1} to {2} LAC {3}", new Object[]{ledgerId, fromId, toId, lastAddConfirmed});
            for (long id = fromId; id <= toId; id++) {
                try (LedgerEntries entries = lh.readUnconfirmed(id, id);) {
                    LedgerEntry entry = entries.getEntry(id);
                    LogEntry lEntry = LogEntry.deserialize(entry.getEntryBytes());
                    LogEntryWithSequenceNumber e = new LogEntryWithSequenceNumber(
                            new LogSequenceNumber(ledgerId, id),
                            lEntry);
                    consumer.accept(e);
                }
            }
        }
    }

}
 

@Override
public void asyncReadEntry(ReadHandle lh, long firstEntry, long lastEntry, boolean isSlowestReader,
        final ReadEntriesCallback callback, Object ctx) {
    lh.readAsync(firstEntry, lastEntry).whenComplete(
            (ledgerEntries, exception) -> {
                if (exception != null) {
                    callback.readEntriesFailed(createManagedLedgerException(exception), ctx);
                    return;
                }
                List<Entry> entries = Lists.newArrayList();
                long totalSize = 0;
                try {
                    for (LedgerEntry e : ledgerEntries) {
                        // Insert the entries at the end of the list (they will be unsorted for now)
                        EntryImpl entry = EntryImpl.create(e);
                        entries.add(entry);
                        totalSize += entry.getLength();
                    }
                } finally {
                    ledgerEntries.close();
                }
                mlFactoryMBean.recordCacheMiss(entries.size(), totalSize);
                ml.mbean.addReadEntriesSample(entries.size(), totalSize);

                callback.readEntriesComplete(entries, ctx);
            }).exceptionally(exception -> {
            	callback.readEntriesFailed(createManagedLedgerException(exception), ctx);
            	return null;
            });
}
 

@Override
public OpenBuilder newOpenLedgerOp() {
    return new OpenBuilderBase() {
        @Override
        public CompletableFuture<ReadHandle> execute() {
            return getProgrammedFailure().thenCompose(
                    (res) -> {
                        int rc = validate();
                        if (rc != BKException.Code.OK) {
                            return FutureUtils.exception(BKException.create(rc));
                        }

                        PulsarMockLedgerHandle lh = ledgers.get(ledgerId);
                        if (lh == null) {
                            return FutureUtils.exception(new BKException.BKNoSuchLedgerExistsException());
                        } else if (lh.digest != DigestType.fromApiDigestType(digestType)) {
                            return FutureUtils.exception(new BKException.BKDigestMatchException());
                        } else if (!Arrays.equals(lh.passwd, password)) {
                            return FutureUtils.exception(new BKException.BKUnauthorizedAccessException());
                        } else {
                            return FutureUtils.value(new PulsarMockReadHandle(PulsarMockBookKeeper.this, ledgerId,
                                                                              lh.getLedgerMetadata(), lh.entries));
                        }
                    });
        }
    };
}
 

@Override
public CompletableFuture<ReadHandle> readOffloaded(long ledgerId, UUID uuid,
                                                   Map<String, String> offloadDriverMetadata) {
    CompletableFuture<ReadHandle> promise = new CompletableFuture<>();
    promise.completeExceptionally(new UnsupportedOperationException());
    return promise;
}
 

@Override
public CompletableFuture<Void> offload(ReadHandle ledger,
                                       UUID uuid,
                                       Map<String, String> extraMetadata) {
    CompletableFuture<Void> promise = new CompletableFuture<>();
    if (offloads.putIfAbsent(ledger.getId(), uuid) == null) {
        promise.complete(null);
    } else {
        promise.completeExceptionally(new Exception("Already exists exception"));
    }
    return promise;
}
 

private LedgerReader(ReadHandle readHandle, UUID uuid, Map<String, String> extraMetadata, CompletableFuture<Void> promise,
                     String storageBasePath, Configuration configuration, OrderedScheduler assignmentScheduler, int managedLedgerOffloadPrefetchRounds) {
    this.readHandle = readHandle;
    this.uuid = uuid;
    this.extraMetadata = extraMetadata;
    this.promise = promise;
    this.storageBasePath = storageBasePath;
    this.configuration = configuration;
    this.assignmentScheduler = assignmentScheduler;
    this.managedLedgerOffloadPrefetchRounds = managedLedgerOffloadPrefetchRounds;
}
 

@Test
public void testDeleteOffloadedFail() throws Exception {
    String failureString = "fail deleteOffloaded";
    ReadHandle readHandle = buildReadHandle(DEFAULT_BLOCK_SIZE, 1);
    UUID uuid = UUID.randomUUID();
    BlobStore spiedBlobStore = mock(BlobStore.class, delegatesTo(blobStore));

    Mockito
        .doThrow(new RuntimeException(failureString))
        .when(spiedBlobStore).removeBlobs(any(), any());

    LedgerOffloader offloader = new BlobStoreManagedLedgerOffloader(spiedBlobStore, BUCKET, scheduler,
        DEFAULT_BLOCK_SIZE, DEFAULT_READ_BUFFER_SIZE);

    try {
        // verify object exist after offload
        offloader.offload(readHandle, uuid, new HashMap<>()).get();
        Assert.assertTrue(blobStore.blobExists(BUCKET, BlobStoreManagedLedgerOffloader.dataBlockOffloadKey(readHandle.getId(), uuid)));
        Assert.assertTrue(blobStore.blobExists(BUCKET, BlobStoreManagedLedgerOffloader.indexBlockOffloadKey(readHandle.getId(), uuid)));

        offloader.deleteOffloaded(readHandle.getId(), uuid, Collections.emptyMap()).get();
    } catch (Exception e) {
        // expected
        Assert.assertTrue(e.getCause().getMessage().contains(failureString));
        // verify object still there.
        Assert.assertTrue(blobStore.blobExists(BUCKET, BlobStoreManagedLedgerOffloader.dataBlockOffloadKey(readHandle.getId(), uuid)));
        Assert.assertTrue(blobStore.blobExists(BUCKET, BlobStoreManagedLedgerOffloader.indexBlockOffloadKey(readHandle.getId(), uuid)));
    }
}