com.alipay.sofa.jraft.error.RaftError Java Examples

@Override
public void cancel() {
    this.lock.lock();
    try {
        if (this.finished) {
            return;
        }
        if (this.timer != null) {
            this.timer.cancel(true);
        }
        if (this.rpcCall != null) {
            this.rpcCall.cancel(true);
        }
        if (this.st.isOk()) {
            this.st.setError(RaftError.ECANCELED, RaftError.ECANCELED.name());
        }
        onFinished();
    } finally {
        this.lock.unlock();
    }
}
 

private void unsafeApplyConfiguration(final Configuration newConf, final Configuration oldConf,
                                      final boolean leaderStart) {
    Requires.requireTrue(this.confCtx.isBusy(), "ConfigurationContext is not busy");
    final LogEntry entry = new LogEntry(EnumOutter.EntryType.ENTRY_TYPE_CONFIGURATION);
    entry.setId(new LogId(0, this.currTerm));
    entry.setPeers(newConf.listPeers());
    entry.setLearners(newConf.listLearners());
    if (oldConf != null) {
        entry.setOldPeers(oldConf.listPeers());
        entry.setOldLearners(oldConf.listLearners());
    }
    final ConfigurationChangeDone configurationChangeDone = new ConfigurationChangeDone(this.currTerm, leaderStart);
    // Use the new_conf to deal the quorum of this very log
    if (!this.ballotBox.appendPendingTask(newConf, oldConf, configurationChangeDone)) {
        Utils.runClosureInThread(configurationChangeDone, new Status(RaftError.EINTERNAL, "Fail to append task."));
        return;
    }
    final List<LogEntry> entries = new ArrayList<>();
    entries.add(entry);
    this.logManager.appendEntries(entries, new LeaderStableClosure(entries));
    checkAndSetConfiguration(false);
}
 

private void checkDeadNodes(final Configuration conf, final long monotonicNowMs) {
    // Check learner replicators at first.
    for (PeerId peer : conf.getLearners()) {
        checkReplicator(peer);
    }
    // Ensure quorum nodes alive.
    final List<PeerId> peers = conf.listPeers();
    final Configuration deadNodes = new Configuration();
    if (checkDeadNodes0(peers, monotonicNowMs, true, deadNodes)) {
        return;
    }
    LOG.warn("Node {} steps down when alive nodes don't satisfy quorum, term={}, deadNodes={}, conf={}.",
        getNodeId(), this.currTerm, deadNodes, conf);
    final Status status = new Status();
    status.setError(RaftError.ERAFTTIMEDOUT, "Majority of the group dies: %d/%d", deadNodes.size(), peers.size());
    stepDown(this.currTerm, false, status);
}
 

protected void compressSnapshot(final SnapshotWriter writer, final LocalFileMeta.Builder metaBuilder,
                                final Closure done) {
    final String writerPath = writer.getPath();
    final String outputFile = Paths.get(writerPath, SNAPSHOT_ARCHIVE).toString();
    try {
        final Checksum checksum = new CRC64();
        ZipUtil.compress(writerPath, SNAPSHOT_DIR, outputFile, checksum);
        metaBuilder.setChecksum(Long.toHexString(checksum.getValue()));
        if (writer.addFile(SNAPSHOT_ARCHIVE, metaBuilder.build())) {
            done.run(Status.OK());
        } else {
            done.run(new Status(RaftError.EIO, "Fail to add snapshot file: %s", writerPath));
        }
    } catch (final Throwable t) {
        LOG.error("Fail to compress snapshot, path={}, file list={}, {}.", writerPath, writer.listFiles(),
            StackTraceUtil.stackTrace(t));
        done.run(new Status(RaftError.EIO, "Fail to compress snapshot at %s, error is %s", writerPath, t
            .getMessage()));
    }
}
 

@Override
public void save(final SnapshotWriter writer, final Region region, final Closure done,
                 final ExecutorService executor) {
    final String writerPath = writer.getPath();
    final String snapshotPath = Paths.get(writerPath, SNAPSHOT_DIR).toString();
    try {
        doSnapshotSave(snapshotPath, region, executor).whenComplete((metaBuilder, throwable) -> {
            if (throwable == null) {
                executor.execute(() -> compressSnapshot(writer, metaBuilder, done));
            } else {
                LOG.error("Fail to save snapshot, path={}, file list={}, {}.", writerPath, writer.listFiles(),
                        StackTraceUtil.stackTrace(throwable));
                done.run(new Status(RaftError.EIO, "Fail to save snapshot at %s, error is %s", writerPath,
                        throwable.getMessage()));
            }
        });
    } catch (final Throwable t) {
        LOG.error("Fail to save snapshot, path={}, file list={}, {}.", writerPath, writer.listFiles(),
                StackTraceUtil.stackTrace(t));
        done.run(new Status(RaftError.EIO, "Fail to save snapshot at %s, error is %s", writerPath,
                t.getMessage()));
    }
}
 

private void handleVoteTimeout() {
    this.writeLock.lock();
    if (this.state != State.STATE_CANDIDATE) {
        this.writeLock.unlock();
        return;
    }

    if (this.raftOptions.isStepDownWhenVoteTimedout()) {
        LOG.warn(
            "Candidate node {} term {} steps down when election reaching vote timeout: fail to get quorum vote-granted.",
            this.nodeId, this.currTerm);
        stepDown(this.currTerm, false, new Status(RaftError.ETIMEDOUT,
            "Vote timeout: fail to get quorum vote-granted."));
        // unlock in preVote
        preVote();
    } else {
        LOG.debug("Node {} term {} retry to vote self.", getNodeId(), this.currTerm);
        // unlock in electSelf
        electSelf();
    }
}
 

public void onError(final RaftException error) {
    LOG.warn("Node {} got error: {}.", getNodeId(), error);
    if (this.fsmCaller != null) {
        // onError of fsmCaller is guaranteed to be executed once.
        this.fsmCaller.onError(error);
    }
    if (this.readOnlyService != null) {
        this.readOnlyService.setError(error);
    }
    this.writeLock.lock();
    try {
        // If it is leader, need to wake up a new one;
        // If it is follower, also step down to call on_stop_following.
        if (this.state.compareTo(State.STATE_FOLLOWER) <= 0) {
            stepDown(this.currTerm, this.state == State.STATE_LEADER, new Status(RaftError.EBADNODE,
                "Raft node(leader or candidate) is in error."));
        }
        if (this.state.compareTo(State.STATE_ERROR) < 0) {
            this.state = State.STATE_ERROR;
        }
    } finally {
        this.writeLock.unlock();
    }
}
 

@Test
public void testOnRpcReturnedRetry() throws Exception {
    assertNull(this.session.getTimer());
    assertNull(this.session.getRpcCall());
    final ByteBufferCollector bufRef = ByteBufferCollector.allocate(0);
    this.session.setDestBuf(bufRef);

    final FutureImpl<Message> future = new FutureImpl<>();
    final RpcRequests.GetFileRequest.Builder rb = RpcRequests.GetFileRequest.newBuilder().setReaderId(99)
        .setFilename("data").setCount(Integer.MAX_VALUE).setOffset(0).setReadPartly(true);
    Mockito
        .when(this.rpcService.getFile(this.address, rb.build(), this.copyOpts.getTimeoutMs(), session.getDone()))
        .thenReturn(future);

    this.session.onRpcReturned(new Status(RaftError.EINTR, "test"), null);
    assertNotNull(this.session.getTimer());
    Thread.sleep(this.copyOpts.getRetryIntervalMs() + 100);
    assertNotNull(this.session.getRpcCall());
    assertSame(future, this.session.getRpcCall());
    assertNull(this.session.getTimer());
}
 

@Override
protected Message processRequest0(final CliRequestContext ctx, final ResetPeerRequest request,
                                  final RpcRequestClosure done) {
    final Configuration newConf = new Configuration();
    for (final String peerIdStr : request.getNewPeersList()) {
        final PeerId peer = new PeerId();
        if (peer.parse(peerIdStr)) {
            newConf.addPeer(peer);
        } else {
            return RpcFactoryHelper //
                .responseFactory() //
                .newResponse(defaultResp(), RaftError.EINVAL, "Fail to parse peer id %s", peerIdStr);
        }
    }
    LOG.info("Receive ResetPeerRequest to {} from {}, new conf is {}", ctx.node.getNodeId(), done.getRpcCtx()
        .getRemoteAddress(), newConf);
    final Status st = ctx.node.resetPeers(newConf);
    return RpcFactoryHelper //
        .responseFactory() //
        .newResponse(defaultResp(), st);
}
 

/**
 * Handle read index request.
 */
@Override
public void handleReadIndexRequest(final ReadIndexRequest request, final RpcResponseClosure<ReadIndexResponse> done) {
    final long startMs = Utils.monotonicMs();
    this.readLock.lock();
    try {
        switch (this.state) {
            case STATE_LEADER:
                readLeader(request, ReadIndexResponse.newBuilder(), done);
                break;
            case STATE_FOLLOWER:
                readFollower(request, done);
                break;
            case STATE_TRANSFERRING:
                done.run(new Status(RaftError.EBUSY, "Is transferring leadership."));
                break;
            default:
                done.run(new Status(RaftError.EPERM, "Invalid state for readIndex: %s.", this.state));
                break;
        }
    } finally {
        this.readLock.unlock();
        this.metrics.recordLatency("handle-read-index", Utils.monotonicMs() - startMs);
        this.metrics.recordSize("handle-read-index-entries", request.getEntriesCount());
    }
}
 

private void applyOperation(final CounterOperation op, final CounterClosure closure) {
    if (!isLeader()) {
        handlerNotLeaderError(closure);
        return;
    }

    try {
        closure.setCounterOperation(op);
        final Task task = new Task();
        task.setData(ByteBuffer.wrap(SerializerManager.getSerializer(SerializerManager.Hessian2).serialize(op)));
        task.setDone(closure);
        this.counterServer.getNode().apply(task);
    } catch (CodecException e) {
        String errorMsg = "Fail to encode CounterOperation";
        LOG.error(errorMsg, e);
        closure.failure(errorMsg, StringUtils.EMPTY);
        closure.run(new Status(RaftError.EINTERNAL, errorMsg));
    }
}
 

protected Node getNode(String groupId, PeerId peerId, Status st) {
    Node node = null;

    if (peerId != null) {
        node = NodeManager.getInstance().get(groupId, peerId);
        if (node == null) {
            st.setError(RaftError.ENOENT, "Fail to find node %s in group %s", peerId, groupId);
        }
    } else {
        List<Node> nodes = NodeManager.getInstance().getNodesByGroupId(groupId);
        if (nodes == null || nodes.isEmpty()) {
            st.setError(RaftError.ENOENT, "Empty nodes in group %s", groupId);
        } else if (nodes.size() > 1) {
            st.setError(RaftError.EINVAL, "Peer must be specified since there're %d nodes in group %s",
                nodes.size(), groupId);
        } else {
            node = nodes.get(0);
        }

    }
    if (node != null && node.getOptions().isDisableCli()) {
        st.setError(RaftError.EACCES, "Cli service is not allowed to access node %s", node.getNodeId());
    }
    return node;
}
 

void reset(final Status st) {
    if (st != null && st.isOk()) {
        this.node.stopReplicator(this.newPeers, this.oldPeers);
        this.node.stopReplicator(this.newLearners, this.oldLearners);
    } else {
        this.node.stopReplicator(this.oldPeers, this.newPeers);
        this.node.stopReplicator(this.oldLearners, this.newLearners);
    }
    clearPeers();
    clearLearners();

    this.version++;
    this.stage = Stage.STAGE_NONE;
    this.nchanges = 0;
    if (this.done != null) {
        Utils.runClosureInThread(this.done, st != null ? st : new Status(RaftError.EPERM,
            "Leader stepped down."));
        this.done = null;
    }
}
 

@Override
public Message processRequest(final T request, final RpcRequestClosure done) {
    final PeerId peer = new PeerId();
    final String peerIdStr = getPeerId(request);
    if (peer.parse(peerIdStr)) {
        final String groupId = getGroupId(request);
        final Node node = NodeManager.getInstance().get(groupId, peer);
        if (node != null) {
            return processRequest0((RaftServerService) node, request, done);
        } else {
            return RpcFactoryHelper //
                .responseFactory() //
                .newResponse(defaultResp(), RaftError.ENOENT, "Peer id not found: %s, group: %s", peerIdStr,
                    groupId);
        }
    } else {
        return RpcFactoryHelper //
            .responseFactory() //
            .newResponse(defaultResp(), RaftError.EINVAL, "Fail to parse peerId: %s", peerIdStr);
    }
}
 

@Override
public void onSnapshotSave(final SnapshotWriter writer, final Closure done) {
  final long currVal = this.value.get();
  Utils.runInThread(() -> {
    final CounterSnapshotFile snapshot = new CounterSnapshotFile(writer.getPath() + File.separator + "data");
    if (snapshot.save(currVal)) {
      if (writer.addFile("data")) {
        done.run(Status.OK());
      } else {
        done.run(new Status(RaftError.EIO, "Fail to add file to writer"));
      }
    } else {
      done.run(new Status(RaftError.EIO, "Fail to save counter snapshot %s", snapshot.getPath()));
    }
  });
}
 

@Test
public void testOnTimeoutNowReturnedTermMismatch() {
    final Replicator r = getReplicator();
    this.id.unlock();
    final RpcRequests.TimeoutNowRequest request = createTimeoutnowRequest();
    final RpcRequests.TimeoutNowResponse response = RpcRequests.TimeoutNowResponse.newBuilder() //
        .setSuccess(false) //
        .setTerm(12) //
        .build();
    this.id.unlock();

    Replicator.onTimeoutNowReturned(this.id, Status.OK(), request, response, false);
    Mockito.verify(this.node).increaseTermTo(
        12,
        new Status(RaftError.EHIGHERTERMRESPONSE, "Leader receives higher term timeout_now_response from peer:%s",
            this.peerId));
    assertNull(r.id);
}
 

/**
 * Move to next
 */
public void next() {
    this.currEntry = null; //release current entry
    //get next entry
    if (this.currentIndex <= this.committedIndex) {
        ++this.currentIndex;
        if (this.currentIndex <= this.committedIndex) {
            try {
                this.currEntry = this.logManager.getEntry(this.currentIndex);
                if (this.currEntry == null) {
                    getOrCreateError().setType(EnumOutter.ErrorType.ERROR_TYPE_LOG);
                    getOrCreateError().getStatus().setError(-1,
                        "Fail to get entry at index=%d while committed_index=%d", this.currentIndex,
                        this.committedIndex);
                }
            } catch (final LogEntryCorruptedException e) {
                getOrCreateError().setType(EnumOutter.ErrorType.ERROR_TYPE_LOG);
                getOrCreateError().getStatus().setError(RaftError.EINVAL, e.getMessage());
            }
            this.applyingIndex.set(this.currentIndex);
        }
    }
}
 

@Test
public void testOnRpcReturnedTermMismatch() {
    final Replicator r = getReplicator();
    final RpcRequests.AppendEntriesRequest request = createEmptyEntriesRequest();
    final RpcRequests.AppendEntriesResponse response = RpcRequests.AppendEntriesResponse.newBuilder() //
        .setSuccess(false) //
        .setLastLogIndex(12) //
        .setTerm(2) //
        .build();
    this.id.unlock();

    Replicator.onRpcReturned(this.id, Replicator.RequestType.AppendEntries, Status.OK(), request, response, 0, 0,
        Utils.monotonicMs());
    Mockito.verify(this.node).increaseTermTo(
        2,
        new Status(RaftError.EHIGHERTERMRESPONSE, "Leader receives higher term heartbeat_response from peer:%s",
            this.peerId));
    assertNull(r.id);
}
 

@Override
public Status checkConsistency() {
    this.readLock.lock();
    try {
        Requires.requireTrue(this.firstLogIndex > 0);
        Requires.requireTrue(this.lastLogIndex >= 0);
        if (this.lastSnapshotId.equals(new LogId(0, 0))) {
            if (this.firstLogIndex == 1) {
                return Status.OK();
            }
            return new Status(RaftError.EIO, "Missing logs in (0, %d)", this.firstLogIndex);
        } else {
            if (this.lastSnapshotId.getIndex() >= this.firstLogIndex - 1
                && this.lastSnapshotId.getIndex() <= this.lastLogIndex) {
                return Status.OK();
            }
            return new Status(RaftError.EIO, "There's a gap between snapshot={%d, %d} and log=[%d, %d] ",
                this.lastSnapshotId.toString(), this.lastSnapshotId.getTerm(), this.firstLogIndex,
                this.lastLogIndex);
        }
    } finally {
        this.readLock.unlock();
    }
}
 

@Override
public void clear() {
    List<Closure> savedQueue;
    this.lock.lock();
    try {
        this.firstIndex = 0;
        savedQueue = this.queue;
        this.queue = new LinkedList<>();
    } finally {
        this.lock.unlock();
    }

    final Status status = new Status(RaftError.EPERM, "Leader stepped down");
    Utils.runInThread(() -> {
        for (final Closure done : savedQueue) {
            if (done != null) {
                done.run(status);
            }
        }
    });
}
 

@Test
public void testManyNodes() {
    Node node1 = Mockito.mock(Node.class);
    Mockito.when(node1.getGroupId()).thenReturn("test");
    Mockito.when(node1.getNodeId()).thenReturn(new NodeId("test", new PeerId("localhost", 8081)));
    NodeOptions opts = new NodeOptions();
    Mockito.when(node1.getOptions()).thenReturn(opts);
    NodeManager.getInstance().addAddress(new Endpoint("localhost", 8081));
    NodeManager.getInstance().add(node1);

    Node node2 = Mockito.mock(Node.class);
    Mockito.when(node2.getGroupId()).thenReturn("test");
    Mockito.when(node2.getNodeId()).thenReturn(new NodeId("test", new PeerId("localhost", 8082)));
    Mockito.when(node2.getOptions()).thenReturn(opts);
    NodeManager.getInstance().addAddress(new Endpoint("localhost", 8082));
    NodeManager.getInstance().add(node2);

    this.processor = new MockCliRequestProcessor(null, "test");
    this.processor.handleRequest(asyncContext, TestUtils.createPingRequest());
    ErrorResponse resp = (ErrorResponse) asyncContext.getResponseObject();
    assertNotNull(resp);
    assertEquals(RaftError.EINVAL.getNumber(), resp.getErrorCode());
    assertEquals("Peer must be specified since there're 2 nodes in group test", resp.getErrorMsg());
}
 

@Override
public void cancel() {
    this.lock.lock();
    try {
        if (this.cancelled) {
            return;
        }
        if (isOk()) {
            setError(RaftError.ECANCELED, "Cancel the copier manually.");
        }
        this.cancelled = true;
        if (this.curSession != null) {
            this.curSession.cancel();
        }
        if (this.future != null) {
            this.future.cancel(true);
        }
    } finally {
        this.lock.unlock();
    }

}
 

public void handlePreVoteResponse(final PeerId peerId, final long term, final RequestVoteResponse response) {
    boolean doUnlock = true;
    this.writeLock.lock();
    try {
        if (this.state != State.STATE_FOLLOWER) {
            LOG.warn("Node {} received invalid PreVoteResponse from {}, state not in STATE_FOLLOWER but {}.",
                getNodeId(), peerId, this.state);
            return;
        }
        if (term != this.currTerm) {
            LOG.warn("Node {} received invalid PreVoteResponse from {}, term={}, currTerm={}.", getNodeId(),
                peerId, term, this.currTerm);
            return;
        }
        if (response.getTerm() > this.currTerm) {
            LOG.warn("Node {} received invalid PreVoteResponse from {}, term {}, expect={}.", getNodeId(), peerId,
                response.getTerm(), this.currTerm);
            stepDown(response.getTerm(), false, new Status(RaftError.EHIGHERTERMRESPONSE,
                "Raft node receives higher term pre_vote_response."));
            return;
        }
        LOG.info("Node {} received PreVoteResponse from {}, term={}, granted={}.", getNodeId(), peerId,
            response.getTerm(), response.getGranted());
        // check granted quorum?
        if (response.getGranted()) {
            this.prevVoteCtx.grant(peerId);
            if (this.prevVoteCtx.isGranted()) {
                doUnlock = false;
                electSelf();
            }
        }
    } finally {
        if (doUnlock) {
            this.writeLock.unlock();
        }
    }
}
 

private void readFollower(final ReadIndexRequest request, final RpcResponseClosure<ReadIndexResponse> closure) {
    if (this.leaderId == null || this.leaderId.isEmpty()) {
        closure.run(new Status(RaftError.EPERM, "No leader at term %d.", this.currTerm));
        return;
    }
    // send request to leader.
    final ReadIndexRequest newRequest = ReadIndexRequest.newBuilder() //
        .mergeFrom(request) //
        .setPeerId(this.leaderId.toString()) //
        .build();
    this.rpcService.readIndex(this.leaderId.getEndpoint(), newRequest, -1, closure);
}
 

@Test
public void testInvalidPeerId() {
    this.processor = new MockCliRequestProcessor("localhost", "test");
    this.processor.handleRequest(asyncContext, TestUtils.createPingRequest());
    ErrorResponse resp = (ErrorResponse) asyncContext.getResponseObject();
    assertNotNull(resp);
    assertEquals(RaftError.EINVAL.getNumber(), resp.getErrorCode());
    assertEquals("Fail to parse peer: localhost", resp.getErrorMsg());
}
 

public static void sendHeartbeat(final ThreadId id, final RpcResponseClosure<AppendEntriesResponse> closure) {
    final Replicator r = (Replicator) id.lock();
    if (r == null) {
        RpcUtils.runClosureInThread(closure, new Status(RaftError.EHOSTDOWN, "Peer %s is not connected", id));
        return;
    }
    //id unlock in send empty entries.
    r.sendEmptyEntries(true, closure);
}
 

private LogId appendToStorage(final List<LogEntry> toAppend) {
    LogId lastId = null;
    if (!this.hasError) {
        final long startMs = Utils.monotonicMs();
        final int entriesCount = toAppend.size();
        this.nodeMetrics.recordSize("append-logs-count", entriesCount);
        try {
            int writtenSize = 0;
            for (int i = 0; i < entriesCount; i++) {
                final LogEntry entry = toAppend.get(i);
                writtenSize += entry.getData() != null ? entry.getData().remaining() : 0;
            }
            this.nodeMetrics.recordSize("append-logs-bytes", writtenSize);
            final int nAppent = this.logStorage.appendEntries(toAppend);
            if (nAppent != entriesCount) {
                LOG.error("**Critical error**, fail to appendEntries, nAppent={}, toAppend={}", nAppent,
                    toAppend.size());
                reportError(RaftError.EIO.getNumber(), "Fail to append log entries");
            }
            if (nAppent > 0) {
                lastId = toAppend.get(nAppent - 1).getId();
            }
            toAppend.clear();
        } finally {
            this.nodeMetrics.recordLatency("append-logs", Utils.monotonicMs() - startMs);
        }
    }
    return lastId;
}
 

private boolean tryOfferEvent(final StableClosure done, final EventTranslator<StableClosureEvent> translator) {
    if (this.stopped) {
        Utils.runClosureInThread(done, new Status(RaftError.ESTOP, "Log manager is stopped."));
        return true;
    }
    return this.diskQueue.tryPublishEvent(translator);
}
 

private void offerEvent(final StableClosure done, final EventType type) {
    if (this.stopped) {
        Utils.runClosureInThread(done, new Status(RaftError.ESTOP, "Log manager is stopped."));
        return;
    }
    if (!this.diskQueue.tryPublishEvent((event, sequence) -> {
        event.reset();
        event.type = type;
        event.done = done;
    })) {
        reportError(RaftError.EBUSY.getNumber(), "Log manager is overload.");
        Utils.runClosureInThread(done, new Status(RaftError.EBUSY, "Log manager is overload."));
    }
}
 

@Test
public void testInvokeWithDoneOnException() throws Exception {
    InvokeContext invokeCtx = new InvokeContext();
    invokeCtx.put(InvokeContext.CRC_SWITCH, false);
    ArgumentCaptor<InvokeCallback> callbackArg = ArgumentCaptor.forClass(InvokeCallback.class);
    PingRequest request = TestUtils.createPingRequest();

    MockRpcResponseClosure<ErrorResponse> done = new MockRpcResponseClosure<>();
    Future<Message> future = this.clientService.invokeWithDone(this.endpoint, request, invokeCtx, done, -1);
    Mockito.verify(this.rpcClient).invokeAsync(eq(this.endpoint), eq(request), eq(invokeCtx),
        callbackArg.capture(), eq((long) this.rpcOptions.getRpcDefaultTimeout()));
    InvokeCallback cb = callbackArg.getValue();
    assertNotNull(cb);
    assertNotNull(future);

    assertNull(done.getResponse());
    assertNull(done.status);
    assertFalse(future.isDone());

    cb.complete(null, new InvokeTimeoutException());

    try {
        future.get();
        fail();
    } catch (ExecutionException e) {
        assertTrue(e.getCause() instanceof InvokeTimeoutException);
    }

    done.latch.await();
    assertNotNull(done.status);
    assertEquals(RaftError.ETIMEDOUT.getNumber(), done.status.getCode());
}