com.alipay.sofa.jraft.Status Java Examples

void onCaughtUp(final long version, final PeerId peer, final boolean success) {
    if (version != this.version) {
        LOG.warn("Ignore onCaughtUp message, mismatch configuration context version, expect {}, but is {}.",
            this.version, version);
        return;
    }
    Requires.requireTrue(this.stage == Stage.STAGE_CATCHING_UP, "Stage is not in STAGE_CATCHING_UP");
    if (success) {
        this.addingPeers.remove(peer);
        if (this.addingPeers.isEmpty()) {
            nextStage();
            return;
        }
        return;
    }
    LOG.warn("Node {} fail to catch up peer {} when trying to change peers from {} to {}.",
        this.node.getNodeId(), peer, this.oldPeers, this.newPeers);
    reset(new Status(RaftError.ECATCHUP, "Peer %s failed to catch up.", peer));
}
 

/**
 * Test method: {@link MemoryRawKVStore#getAndPut(byte[], byte[], KVStoreClosure)}
 */
@Test
public void getAndPutTest() {
    final byte[] key = makeKey("put_test");
    TestClosure closure = new TestClosure();
    this.kvStore.get(key, closure);
    byte[] value = (byte[]) closure.getData();
    assertNull(value);

    value = makeValue("put_test_value");
    KVStoreClosure kvStoreClosure = new BaseKVStoreClosure() {
        @Override
        public void run(Status status) {
            assertEquals(status, Status.OK());
        }
    };
    this.kvStore.getAndPut(key, value, kvStoreClosure);
    assertNull(kvStoreClosure.getData());

    byte[] newVal = makeValue("put_test_value_new");
    this.kvStore.getAndPut(key, newVal, kvStoreClosure);
    assertArrayEquals(value, (byte[]) kvStoreClosure.getData());
}
 

private Replicator testRpcReturnedError() {
    final Replicator r = getReplicator();
    assertNull(r.getBlockTimer());
    final RpcRequests.AppendEntriesRequest request = createEmptyEntriesRequest();
    final RpcRequests.AppendEntriesResponse response = RpcRequests.AppendEntriesResponse.newBuilder() //
        .setSuccess(false) //
        .setLastLogIndex(12) //
        .setTerm(2) //
        .build();
    this.id.unlock();

    Replicator.onRpcReturned(this.id, Replicator.RequestType.AppendEntries, new Status(-1, "test error"), request,
        response, 0, 0, Utils.monotonicMs());
    assertEquals(r.statInfo.runningState, Replicator.RunningState.BLOCKING);
    assertNotNull(r.getBlockTimer());
    return r;
}
 

@Override
public void onApply(final Iterator iter) {
    while (iter.hasNext()) {
        this.lock.lock();
        try {
            if (iter.getIndex() <= this.lastAppliedIndex.get()) {
                //prevent duplication
                continue;
            }
            this.lastAppliedIndex.set(iter.getIndex());
            this.logs.add(iter.getData().slice());
            if (iter.done() != null) {
                iter.done().run(Status.OK());
            }
        } finally {
            this.lock.unlock();
        }
        this.appliedIndex = iter.getIndex();
        iter.next();
    }
}
 

@Override
public boolean transferLeader(final long regionId, final Peer peer, final boolean refreshConf) {
    Requires.requireNonNull(peer, "peer");
    Requires.requireNonNull(peer.getEndpoint(), "peer.endpoint");
    final String raftGroupId = JRaftHelper.getJRaftGroupId(this.clusterName, regionId);
    final Configuration conf = RouteTable.getInstance().getConfiguration(raftGroupId);
    final Status status = this.cliService.transferLeader(raftGroupId, conf, JRaftHelper.toJRaftPeerId(peer));
    if (status.isOk()) {
        if (refreshConf) {
            refreshRouteConfiguration(regionId);
        }
        return true;
    }
    LOG.error("Fail to [transferLeader], [regionId: {}, peer: {}], status: {}.", regionId, peer, status);
    return false;
}
 

@Override
public Status resetPeer(final String groupId, final PeerId peerId, final Configuration newPeers) {
    Requires.requireTrue(!StringUtils.isBlank(groupId), "Blank group id");
    Requires.requireNonNull(peerId, "Null peerId");
    Requires.requireNonNull(newPeers, "Null new peers");

    if (!this.cliClientService.connect(peerId.getEndpoint())) {
        return new Status(-1, "Fail to init channel to %s", peerId);
    }

    final ResetPeerRequest.Builder rb = ResetPeerRequest.newBuilder() //
        .setGroupId(groupId) //
        .setPeerId(peerId.toString());
    for (final PeerId peer : newPeers) {
        rb.addNewPeers(peer.toString());
    }

    try {
        final Message result = this.cliClientService.resetPeer(peerId.getEndpoint(), rb.build(), null).get();
        return statusFromResponse(result);
    } catch (final Exception e) {
        return new Status(-1, e.getMessage());
    }
}
 

private void checkDeadNodes(final Configuration conf, final long monotonicNowMs) {
    // Check learner replicators at first.
    for (PeerId peer : conf.getLearners()) {
        checkReplicator(peer);
    }
    // Ensure quorum nodes alive.
    final List<PeerId> peers = conf.listPeers();
    final Configuration deadNodes = new Configuration();
    if (checkDeadNodes0(peers, monotonicNowMs, true, deadNodes)) {
        return;
    }
    LOG.warn("Node {} steps down when alive nodes don't satisfy quorum, term={}, deadNodes={}, conf={}.",
        getNodeId(), this.currTerm, deadNodes, conf);
    final Status status = new Status();
    status.setError(RaftError.ERAFTTIMEDOUT, "Majority of the group dies: %d/%d", deadNodes.size(), peers.size());
    stepDown(this.currTerm, false, status);
}
 

@Override
public Status snapshot(final String groupId, final PeerId peer) {
    Requires.requireTrue(!StringUtils.isBlank(groupId), "Blank group id");
    Requires.requireNonNull(peer, "Null peer");

    if (!this.cliClientService.connect(peer.getEndpoint())) {
        return new Status(-1, "Fail to init channel to %s", peer);
    }

    final SnapshotRequest.Builder rb = SnapshotRequest.newBuilder() //
        .setGroupId(groupId) //
        .setPeerId(peer.toString());

    try {
        final Message result = this.cliClientService.snapshot(peer.getEndpoint(), rb.build(), null).get();
        return statusFromResponse(result);
    } catch (final Exception e) {
        return new Status(-1, e.getMessage());
    }
}
 

@Test
public void testOnSnapshotSaveEmptyConf() throws Exception {
    final CountDownLatch latch = new CountDownLatch(1);
    this.fsmCaller.onSnapshotSave(new SaveSnapshotClosure() {

        @Override
        public void run(final Status status) {
            assertFalse(status.isOk());
            assertEquals("Empty conf entry for lastAppliedIndex=10", status.getErrorMsg());
            latch.countDown();
        }

        @Override
        public SnapshotWriter start(final SnapshotMeta meta) {
            // TODO Auto-generated method stub
            return null;
        }
    });
    latch.await();
}
 

protected void compressSnapshot(final SnapshotWriter writer, final LocalFileMeta.Builder metaBuilder,
                                final Closure done) {
    final String writerPath = writer.getPath();
    final String outputFile = Paths.get(writerPath, SNAPSHOT_ARCHIVE).toString();
    try {
        final Checksum checksum = new CRC64();
        ZipUtil.compress(writerPath, SNAPSHOT_DIR, outputFile, checksum);
        metaBuilder.setChecksum(Long.toHexString(checksum.getValue()));
        if (writer.addFile(SNAPSHOT_ARCHIVE, metaBuilder.build())) {
            done.run(Status.OK());
        } else {
            done.run(new Status(RaftError.EIO, "Fail to add snapshot file: %s", writerPath));
        }
    } catch (final Throwable t) {
        LOG.error("Fail to compress snapshot, path={}, file list={}, {}.", writerPath, writer.listFiles(),
            StackTraceUtil.stackTrace(t));
        done.run(new Status(RaftError.EIO, "Fail to compress snapshot at %s, error is %s", writerPath, t
            .getMessage()));
    }
}
 

@Test
public void testDoSnapshotWithIntervalDist() throws Exception {
    final NodeOptions nodeOptions = new NodeOptions();
    nodeOptions.setSnapshotLogIndexMargin(5);
    Mockito.when(this.node.getOptions()).thenReturn(nodeOptions);
    Mockito.when(this.fSMCaller.getLastAppliedIndex()).thenReturn(6L);

    final ArgumentCaptor<SaveSnapshotClosure> saveSnapshotClosureArg = ArgumentCaptor
        .forClass(SaveSnapshotClosure.class);
    Mockito.when(this.fSMCaller.onSnapshotSave(saveSnapshotClosureArg.capture())).thenReturn(true);
    final SynchronizedClosure done = new SynchronizedClosure();
    this.executor.doSnapshot(done);
    final SaveSnapshotClosure closure = saveSnapshotClosureArg.getValue();
    assertNotNull(closure);
    closure.start(RaftOutter.SnapshotMeta.newBuilder().setLastIncludedIndex(6).setLastIncludedTerm(1).build());
    closure.run(Status.OK());
    done.await();
    this.executor.join();

    assertEquals(1, this.executor.getLastSnapshotTerm());
    assertEquals(6, this.executor.getLastSnapshotIndex());

}
 

@Override
public void onLeaderStop(final Status status) {
    super.onLeaderStop(status);
    final long oldTerm = this.leaderTerm.get();
    this.leaderTerm.set(-1L);
    // Because of the raft state machine must be a sequential commit, in order to prevent the user
    // doing something (needs to go through the raft state machine) in the listeners, we asynchronously
    // triggers the listeners.
    final List<StateListener> listeners = this.storeEngine.getStateListenerContainer() //
        .getStateListenerGroup(getRegionId());
    if (listeners.isEmpty()) {
        return;
    }
    this.storeEngine.getRaftStateTrigger().execute(() -> {
        for (final StateListener listener : listeners) { // iterator the snapshot
            listener.onLeaderStop(oldTerm);
        }
    });
}
 

public static void waitForCaughtUp(final ThreadId id, final long maxMargin, final long dueTime,
                                   final CatchUpClosure done) {
    final Replicator r = (Replicator) id.lock();

    if (r == null) {
        RpcUtils.runClosureInThread(done, new Status(RaftError.EINVAL, "No such replicator"));
        return;
    }
    try {
        if (r.catchUpClosure != null) {
            LOG.error("Previous wait_for_caught_up is not over");
            Utils.runClosureInThread(done, new Status(RaftError.EINVAL, "Duplicated call"));
            return;
        }
        done.setMaxMargin(maxMargin);
        if (dueTime > 0) {
            done.setTimer(r.timerManager.schedule(() -> onCatchUpTimedOut(id), dueTime - Utils.nowMs(),
                TimeUnit.MILLISECONDS));
        }
        r.catchUpClosure = done;
    } finally {
        id.unlock();
    }
}
 

@Test
public void testOnApplied() throws Exception {
    final ArrayList<ReadIndexState> states = new ArrayList<>();
    final byte[] reqContext = TestUtils.getRandomBytes();
    final CountDownLatch latch = new CountDownLatch(1);
    final ReadIndexState state = new ReadIndexState(new Bytes(reqContext), new ReadIndexClosure() {

        @Override
        public void run(final Status status, final long index, final byte[] reqCtx) {
            assertTrue(status.isOk());
            assertEquals(index, 1);
            assertArrayEquals(reqCtx, reqContext);
            latch.countDown();
        }
    }, Utils.monotonicMs());
    state.setIndex(1);
    states.add(state);
    final ReadIndexStatus readIndexStatus = new ReadIndexStatus(states, null, 1);
    this.readOnlyServiceImpl.getPendingNotifyStatus().put(1L, Arrays.asList(readIndexStatus));

    this.readOnlyServiceImpl.onApplied(2);
    latch.await();
    assertTrue(this.readOnlyServiceImpl.getPendingNotifyStatus().isEmpty());
}
 

protected Node getNode(String groupId, PeerId peerId, Status st) {
    Node node = null;

    if (peerId != null) {
        node = NodeManager.getInstance().get(groupId, peerId);
        if (node == null) {
            st.setError(RaftError.ENOENT, "Fail to find node %s in group %s", peerId, groupId);
        }
    } else {
        List<Node> nodes = NodeManager.getInstance().getNodesByGroupId(groupId);
        if (nodes == null || nodes.isEmpty()) {
            st.setError(RaftError.ENOENT, "Empty nodes in group %s", groupId);
        } else if (nodes.size() > 1) {
            st.setError(RaftError.EINVAL, "Peer must be specified since there're %d nodes in group %s",
                nodes.size(), groupId);
        } else {
            node = nodes.get(0);
        }

    }
    if (node != null && node.getOptions().isDisableCli()) {
        st.setError(RaftError.EACCES, "Cli service is not allowed to access node %s", node.getNodeId());
    }
    return node;
}
 

@Override
public void containsKey(final byte[] key, final KVStoreClosure closure) {
    this.node.readIndex(BytesUtil.EMPTY_BYTES, new ReadIndexClosure() {

        @Override
        public void run(final Status status, final long index, final byte[] reqCtx) {
            if (status.isOk()) {
                RaftRawKVStore.this.kvStore.containsKey(key, closure);
                return;
            }
            RaftRawKVStore.this.readIndexExecutor.execute(() -> {
                if (isLeader()) {
                    LOG.warn("Fail to [containsKey] with 'ReadIndex': {}, try to applying to the state machine.", status);
                    // If 'read index' read fails, try to applying to the state machine at the leader node
                    applyOperation(KVOperation.createContainsKey(key), closure);
                } else {
                    LOG.warn("Fail to [containsKey] with 'ReadIndex': {}.", status);
                    // Client will retry to leader node
                    new KVClosureAdapter(closure, null).run(status);
                }
            });
        }
    });
}
 

@Test
@SuppressWarnings({ "rawtypes", "unchecked" })
public void testCancelByRemote() throws Exception {
    final FutureImpl<Message> future = new FutureImpl<>();
    final RpcRequests.GetFileRequest.Builder rb = RpcRequests.GetFileRequest.newBuilder().setReaderId(99)
        .setFilename(Snapshot.JRAFT_SNAPSHOT_META_FILE).setCount(Integer.MAX_VALUE).setOffset(0)
        .setReadPartly(true);

    //mock get metadata
    final ArgumentCaptor<RpcResponseClosure> argument = ArgumentCaptor.forClass(RpcResponseClosure.class);
    Mockito.when(
        this.raftClientService.getFile(eq(new Endpoint("localhost", 8081)), eq(rb.build()),
            eq(this.copyOpts.getTimeoutMs()), argument.capture())).thenReturn(future);
    this.copier.start();
    Thread.sleep(500);
    final RpcResponseClosure<RpcRequests.GetFileResponse> closure = argument.getValue();

    closure.run(new Status(RaftError.ECANCELED, "test cancel"));

    this.copier.join();
    //start timer
    final SnapshotReader reader = this.copier.getReader();
    assertNull(reader);
    Assert.assertEquals(RaftError.ECANCELED.getNumber(), this.copier.getCode());
    Assert.assertEquals("test cancel", this.copier.getErrorMsg());
}
 

@Override
public void execute(final NodeExecutor nodeExecutor, final boolean isLeader, final KVStoreClosure closure) {
    final Timer.Context timeCtx = getTimeContext("EXECUTE");
    try {
        if (nodeExecutor != null) {
            nodeExecutor.execute(Status.OK(), isLeader);
        }
        setSuccess(closure, Boolean.TRUE);
    } catch (final Exception e) {
        final Logger LOG = LoggerFactory.getLogger(getClass());
        LOG.error("Fail to [EXECUTE], {}.", StackTraceUtil.stackTrace(e));
        if (nodeExecutor != null) {
            nodeExecutor.execute(new Status(RaftError.EIO, "Fail to [EXECUTE]"), isLeader);
        }
        setCriticalError(closure, "Fail to [EXECUTE]", e);
    } finally {
        timeCtx.stop();
    }
}
 

@Test
public void testOnRpcReturnedTermMismatch() {
    final Replicator r = getReplicator();
    final RpcRequests.AppendEntriesRequest request = createEmptyEntriesRequest();
    final RpcRequests.AppendEntriesResponse response = RpcRequests.AppendEntriesResponse.newBuilder() //
        .setSuccess(false) //
        .setLastLogIndex(12) //
        .setTerm(2) //
        .build();
    this.id.unlock();

    Replicator.onRpcReturned(this.id, Replicator.RequestType.AppendEntries, Status.OK(), request, response, 0, 0,
        Utils.monotonicMs());
    Mockito.verify(this.node).increaseTermTo(
        2,
        new Status(RaftError.EHIGHERTERMRESPONSE, "Leader receives higher term heartbeat_response from peer:%s",
            this.peerId));
    assertNull(r.id);
}
 

/**
 * Notify replicator event(such as created, error, destroyed) to replicatorStateListener which is implemented by users.
 *
 * @param replicator replicator object
 * @param event      replicator's state listener event type
 * @param status     replicator's error detailed status
 */
private static void notifyReplicatorStatusListener(final Replicator replicator, final ReplicatorEvent event,
                                                   final Status status) {
    final ReplicatorOptions replicatorOpts = Requires.requireNonNull(replicator.getOpts(), "replicatorOptions");
    final Node node = Requires.requireNonNull(replicatorOpts.getNode(), "node");
    final PeerId peer = Requires.requireNonNull(replicatorOpts.getPeerId(), "peer");

    final List<ReplicatorStateListener> listenerList = node.getReplicatorStatueListeners();
    for (int i = 0; i < listenerList.size(); i++) {
        final ReplicatorStateListener listener = listenerList.get(i);
        if (listener != null) {
            try {
                switch (event) {
                    case CREATED:
                        RpcUtils.runInThread(() -> listener.onCreated(peer));
                        break;
                    case ERROR:
                        RpcUtils.runInThread(() -> listener.onError(peer, status));
                        break;
                    case DESTROYED:
                        RpcUtils.runInThread(() -> listener.onDestroyed(peer));
                        break;
                    default:
                        break;
                }
            } catch (final Exception e) {
                LOG.error("Fail to notify ReplicatorStatusListener, listener={}, event={}.", listener, event);
            }
        }
    }
}
 

/**
 * Run closure with OK status in thread pool.
 */
public static Future<?> runClosureInThread(final Closure done) {
    if (done == null) {
        return null;
    }
    return runClosureInThread(done, Status.OK());
}
 

private void handleElectionTimeout() {
    boolean doUnlock = true;
    this.writeLock.lock();
    try {
        if (this.state != State.STATE_FOLLOWER) {
            return;
        }
        if (isCurrentLeaderValid()) {
            return;
        }
        resetLeaderId(PeerId.emptyPeer(), new Status(RaftError.ERAFTTIMEDOUT, "Lost connection from leader %s.",
            this.leaderId));

        // Judge whether to launch a election.
        if (!allowLaunchElection()) {
            return;
        }

        doUnlock = false;
        preVote();

    } finally {
        if (doUnlock) {
            this.writeLock.unlock();
        }
    }
}
 

@Override
public void multiGet(final List<byte[]> keys, final boolean readOnlySafe, final KVStoreClosure closure) {
    if (!readOnlySafe) {
        this.kvStore.multiGet(keys, false, closure);
        return;
    }
    this.node.readIndex(BytesUtil.EMPTY_BYTES, new ReadIndexClosure() {

        @Override
        public void run(final Status status, final long index, final byte[] reqCtx) {
            if (status.isOk()) {
                RaftRawKVStore.this.kvStore.multiGet(keys, true, closure);
                return;
            }
            RaftRawKVStore.this.readIndexExecutor.execute(() -> {
                if (isLeader()) {
                    LOG.warn("Fail to [multiGet] with 'ReadIndex': {}, try to applying to the state machine.", status);
                    // If 'read index' read fails, try to applying to the state machine at the leader node
                    applyOperation(KVOperation.createMultiGet(keys), closure);
                } else {
                    LOG.warn("Fail to [multiGet] with 'ReadIndex': {}.", status);
                    // Client will retry to leader node
                    new KVClosureAdapter(closure, null).run(status);
                }
            });
        }
    });
}
 

@Test
public void testConnectFailure() throws Exception {
    Mockito.when(
        this.rpcClient.invokeSync(eq(this.endpoint), Mockito.any(),
            eq((long) this.rpcOptions.getRpcConnectTimeoutMs()))) //
        .thenReturn(this.rpcResponseFactory.newResponse(null, new Status(-1, "test")));
    assertFalse(this.clientService.connect(this.endpoint));
}
 

@Override
public void onSnapshotSave(final SnapshotWriter writer, final Closure done) {
    this.saveSnapshotTimes++;
    final String path = writer.getPath() + File.separator + "data";
    final File file = new File(path);
    try (FileOutputStream fout = new FileOutputStream(file);
            BufferedOutputStream out = new BufferedOutputStream(fout)) {
        this.lock.lock();
        try {
            for (final ByteBuffer buf : this.logs) {
                final byte[] bs = new byte[4];
                Bits.putInt(bs, 0, buf.remaining());
                out.write(bs);
                out.write(buf.array());
            }
            this.snapshotIndex = this.appliedIndex;
        } finally {
            this.lock.unlock();
        }
        System.out.println("Node<" + this.address + "> saved snapshot into " + file);
        writer.addFile("data");
        done.run(Status.OK());
    } catch (final IOException e) {
        e.printStackTrace();
        done.run(new Status(RaftError.EIO, "Fail to save snapshot"));
    }
}
 

@Override
public void reverseScan(final byte[] startKey, final byte[] endKey, final int limit, final boolean readOnlySafe,
                        final boolean returnValue, final KVStoreClosure closure) {
    if (!readOnlySafe) {
        this.kvStore.reverseScan(startKey, endKey, limit, false, returnValue, closure);
        return;
    }
    this.node.readIndex(BytesUtil.EMPTY_BYTES, new ReadIndexClosure() {

        @Override
        public void run(final Status status, final long index, final byte[] reqCtx) {
            if (status.isOk()) {
                RaftRawKVStore.this.kvStore.reverseScan(startKey, endKey, limit, true, returnValue, closure);
                return;
            }
            RaftRawKVStore.this.readIndexExecutor.execute(() -> {
                if (isLeader()) {
                    LOG.warn("Fail to [reverseScan] with 'ReadIndex': {}, try to applying to the state machine.", status);
                    // If 'read index' read fails, try to applying to the state machine at the leader node
                    applyOperation(KVOperation.createReverseScan(startKey, endKey, limit, returnValue), closure);
                } else {
                    LOG.warn("Fail to [reverseScan] with 'ReadIndex': {}.", status);
                    // Client will retry to leader node
                    new KVClosureAdapter(closure, null).run(status);
                }
            });
        }
    });
}
 

@Override
public void onSnapshotSave(final SnapshotWriter writer, final Closure done) {
    this.saveSnapshotTimes++;
    final String path = writer.getPath() + File.separator + "data";
    final File file = new File(path);
    try (FileOutputStream fout = new FileOutputStream(file);
            BufferedOutputStream out = new BufferedOutputStream(fout)) {
        this.lock.lock();
        try {
            for (final ByteBuffer buf : this.logs) {
                final byte[] bs = new byte[4];
                Bits.putInt(bs, 0, buf.remaining());
                out.write(bs);
                out.write(buf.array());
            }
            this.snapshotIndex = this.appliedIndex;
        } finally {
            this.lock.unlock();
        }
        System.out.println("Node<" + this.address + "> saved snapshot into " + file);
        writer.addFile("data");
        done.run(Status.OK());
    } catch (final IOException e) {
        e.printStackTrace();
        done.run(new Status(RaftError.EIO, "Fail to save snapshot"));
    }
}
 

@Override
public void verify(final String interest, final Node node, final ArgumentCaptor<Closure> doneArg) {
    assertEquals(interest, ResetLearnersRequest.class.getName());
    Mockito.verify(node).resetLearners(
        eq(Arrays.asList(new PeerId("learner", 8082), new PeerId("test", 8182), new PeerId("test", 8183))),
        doneArg.capture());
    Closure done = doneArg.getValue();
    assertNotNull(done);
    done.run(Status.OK());
    assertNotNull(this.asyncContext.getResponseObject());
    assertEquals("[learner:8081, learner:8082, learner:8083]", this.asyncContext.as(LearnersOpResponse.class)
        .getOldLearnersList().toString());
    assertEquals("[learner:8082, test:8182, test:8183]", this.asyncContext.as(LearnersOpResponse.class)
        .getNewLearnersList().toString());
}
 

private boolean passByStatus(final Closure done) {
    final Status status = this.error.getStatus();
    if (!status.isOk()) {
        if (done != null) {
            done.run(new Status(RaftError.EINVAL, "FSMCaller is in bad status=`%s`", status));
            return false;
        }
    }
    return true;
}
 

@Test
public void testInvokeWithDoneOK() throws Exception {
    ArgumentCaptor<InvokeCallback> callbackArg = ArgumentCaptor.forClass(InvokeCallback.class);
    PingRequest request = TestUtils.createPingRequest();

    MockRpcResponseClosure<ErrorResponse> done = new MockRpcResponseClosure<>();
    Future<Message> future = this.clientService.invokeWithDone(this.endpoint, request, done, -1);
    Mockito.verify(this.rpcClient).invokeAsync(eq(this.endpoint), eq(request), Mockito.any(),
        callbackArg.capture(), eq((long) this.rpcOptions.getRpcDefaultTimeout()));
    InvokeCallback cb = callbackArg.getValue();
    assertNotNull(cb);
    assertNotNull(future);

    assertNull(done.getResponse());
    assertNull(done.status);
    assertFalse(future.isDone());

    ErrorResponse response = (ErrorResponse) this.rpcResponseFactory.newResponse(null, Status.OK());
    cb.complete(response, null);

    Message msg = future.get();
    assertNotNull(msg);
    assertTrue(msg instanceof ErrorResponse);
    assertSame(msg, response);

    done.latch.await();
    assertNotNull(done.status);
    assertEquals(0, done.status.getCode());
}