io.atomix.protocols.raft.RaftServer Java Examples

/**
 * Creates a Raft server.
 */
private RaftServer createServer(RaftMember member) {
  RaftServerProtocol protocol;
  if (USE_NETTY) {
    Address address = Address.from(++port);
    MessagingService messagingManager = new NettyMessagingService("test", address, new MessagingConfig()).start().join();
    messagingServices.add(messagingManager);
    addressMap.put(member.memberId(), address);
    protocol = new RaftServerMessagingProtocol(messagingManager, PROTOCOL_SERIALIZER, addressMap::get);
  } else {
    protocol = protocolFactory.newServerProtocol(member.memberId());
  }

  RaftServer.Builder builder = RaftServer.builder(member.memberId())
      .withProtocol(protocol)
      .withStorage(RaftStorage.builder()
          .withStorageLevel(StorageLevel.DISK)
          .withDirectory(new File(String.format("target/fuzz-logs/%s", member.memberId())))
          .withNamespace(STORAGE_NAMESPACE)
          .withMaxSegmentSize(1024 * 1024)
          .build());

  RaftServer server = builder.build();
  servers.add(server);
  return server;
}
 

@Override
public CompletableFuture<AppendResponse> onAppend(final AppendRequest request) {
  raft.checkThread();
  logRequest(request);

  // If the request indicates a term that is greater than the current term then
  // assign that term and leader to the current context and transition to follower.
  boolean transition = updateTermAndLeader(request.term(), request.leader());

  // Handle the append request.
  CompletableFuture<AppendResponse> future = handleAppend(request);

  // If a transition is required then transition back to the follower state.
  // If the node is already a follower then the transition will be ignored.
  if (transition) {
    raft.transition(RaftServer.Role.FOLLOWER);
  }
  return future;
}
 

@Override
protected void failAttempt(RaftMemberContext member, RaftRequest request, Throwable error) {
  super.failAttempt(member, request, error);

  // Fail heartbeat futures.
  failHeartbeat();

  // Verify that the leader has contacted a majority of the cluster within the last two election timeouts.
  // If the leader is not able to contact a majority of the cluster within two election timeouts, assume
  // that a partition occurred and transition back to the FOLLOWER state.
  if (member.getFailureCount() >= MIN_STEP_DOWN_FAILURE_COUNT && System.currentTimeMillis() - Math.max(computeResponseTime(), leaderTime) > electionTimeout * 2) {
    log.warn("Suspected network partition. Stepping down");
    raft.setLeader(null);
    raft.transition(RaftServer.Role.FOLLOWER);
  }
}
 

/**
 * Commits a no-op entry to the log, ensuring any entries from a previous term are committed.
 */
private CompletableFuture<Void> commitInitialEntries() {
  // The Raft protocol dictates that leaders cannot commit entries from previous terms until
  // at least one entry from their current term has been stored on a majority of servers. Thus,
  // we force entries to be appended up to the leader's no-op entry. The LeaderAppender will ensure
  // that the commitIndex is not increased until the no-op entry (appender.index()) is committed.
  CompletableFuture<Void> future = new CompletableFuture<>();
  appender.appendEntries(appender.getIndex()).whenComplete((resultIndex, error) -> {
    raft.checkThread();
    if (isRunning()) {
      if (error == null) {
        raft.getServiceManager().apply(resultIndex);
        future.complete(null);
      } else {
        raft.setLeader(null);
        raft.transition(RaftServer.Role.FOLLOWER);
      }
    }
  });
  return future;
}
 

@Override
public CompletableFuture<AppendResponse> onAppend(final AppendRequest request) {
  raft.checkThread();
  if (updateTermAndLeader(request.term(), request.leader())) {
    CompletableFuture<AppendResponse> future = super.onAppend(request);
    raft.transition(RaftServer.Role.FOLLOWER);
    return future;
  } else if (request.term() < raft.getTerm()) {
    logRequest(request);
    return CompletableFuture.completedFuture(logResponse(AppendResponse.builder()
        .withStatus(RaftResponse.Status.OK)
        .withTerm(raft.getTerm())
        .withSucceeded(false)
        .withLastLogIndex(raft.getLogWriter().getLastIndex())
        .withLastSnapshotIndex(raft.getSnapshotStore().getCurrentSnapshotIndex())
        .build()));
  } else {
    raft.setLeader(request.leader());
    raft.transition(RaftServer.Role.FOLLOWER);
    return super.onAppend(request);
  }
}
 

/**
 * Creates an internal state for the given state type.
 */
private RaftRole createRole(RaftServer.Role role) {
  switch (role) {
    case INACTIVE:
      return new InactiveRole(this);
    case PASSIVE:
      return new PassiveRole(this);
    case PROMOTABLE:
      return new PromotableRole(this);
    case FOLLOWER:
      return new FollowerRole(this);
    case CANDIDATE:
      return new CandidateRole(this);
    case LEADER:
      return new LeaderRole(this);
    default:
      throw new AssertionError();
  }
}
 

/**
 * Transitions the server to the base state for the given member type.
 */
public void transition(RaftMember.Type type) {
  switch (type) {
    case ACTIVE:
      if (!(role instanceof ActiveRole)) {
        transition(RaftServer.Role.FOLLOWER);
      }
      break;
    case PROMOTABLE:
      if (this.role.role() != RaftServer.Role.PROMOTABLE) {
        transition(RaftServer.Role.PROMOTABLE);
      }
      break;
    case PASSIVE:
      if (this.role.role() != RaftServer.Role.PASSIVE) {
        transition(RaftServer.Role.PASSIVE);
      }
      break;
    default:
      if (this.role.role() != RaftServer.Role.INACTIVE) {
        transition(RaftServer.Role.INACTIVE);
      }
      break;
  }
}
 

/**
 * Creates a set of Raft servers.
 */
private List<RaftServer> createServers(int nodes) throws Exception {
  List<RaftServer> servers = new ArrayList<>();

  for (int i = 0; i < nodes; i++) {
    members.add(nextNode());
  }

  CountDownLatch latch = new CountDownLatch(nodes);
  for (int i = 0; i < nodes; i++) {
    RaftServer server = createServer(members.get(i), Lists.newArrayList(members));
    server.bootstrap(members.stream().map(Member::id).collect(Collectors.toList())).thenRun(latch::countDown);
    servers.add(server);
  }

  latch.await(30000, TimeUnit.MILLISECONDS);

  return servers;
}
 

/**
 * Creates a set of Raft servers.
 */
private List<RaftServer> createServers(int nodes) throws Exception {
  List<RaftServer> servers = new ArrayList<>();

  for (int i = 0; i < nodes; i++) {
    members.add(nextMember(RaftMember.Type.ACTIVE));
  }

  CountDownLatch latch = new CountDownLatch(nodes);
  for (int i = 0; i < nodes; i++) {
    RaftServer server = createServer(members.get(i));
    server.bootstrap(members.stream().map(RaftMember::memberId).collect(Collectors.toList())).thenRun(latch::countDown);
    servers.add(server);
  }

  latch.await(30000, TimeUnit.MILLISECONDS);

  return servers;
}
 

@Override
public synchronized CompletableFuture<RaftRole> start() {
  if (raft.getCluster().getActiveMemberStates().isEmpty()) {
    log.debug("Single member cluster. Transitioning directly to candidate.");
    raft.transition(RaftServer.Role.CANDIDATE);
    return CompletableFuture.completedFuture(this);
  }
  raft.getMembershipService().addListener(clusterListener);
  return super.start().thenRun(this::resetHeartbeatTimeout).thenApply(v -> this);
}
 

@Before
public void setupContext() throws IOException {
  deleteStorage();

  RaftStorage storage = RaftStorage.builder()
      .withPrefix("test")
      .withDirectory(PATH.toFile())
      .withNamespace(NAMESPACE)
      .build();
  PrimitiveTypeRegistry registry = new PrimitiveTypeRegistry() {
    @Override
    public Collection<PrimitiveType> getPrimitiveTypes() {
      return Collections.singleton(new TestType());
    }

    @Override
    public PrimitiveType getPrimitiveType(String typeName) {
      return new TestType();
    }
  };
  raft = new RaftContext(
      "test",
      MemberId.from("test-1"),
      mock(ClusterMembershipService.class),
      mock(RaftServerProtocol.class),
      storage,
      registry,
      ThreadModel.SHARED_THREAD_POOL.factory("raft-server-test-%d", 1, LoggerFactory.getLogger(RaftServer.class)),
      true);

  snapshotTaken = new AtomicBoolean();
  snapshotInstalled = new AtomicBoolean();
}
 

@Override
public CompletableFuture<VoteResponse> onVote(final VoteRequest request) {
  if (updateTermAndLeader(request.term(), null)) {
    log.debug("Received greater term");
    raft.transition(RaftServer.Role.FOLLOWER);
    return super.onVote(request);
  } else {
    logRequest(request);
    return CompletableFuture.completedFuture(logResponse(VoteResponse.builder()
        .withStatus(RaftResponse.Status.OK)
        .withTerm(raft.getTerm())
        .withVoted(false)
        .build()));
  }
}
 

/**
 * Schedules the given server to be shutdown for a period of time and then restarted.
 */
private void scheduleRestart(boolean remove, int serverIndex, ThreadContext context) {
  shutdownTimers.put(serverIndex, context.schedule(Duration.ofSeconds(randomNumber(120) + 10), () -> {
    shutdownTimers.remove(serverIndex);
    RaftServer server = servers.get(serverIndex);
    CompletableFuture<Void> leaveFuture;
    if (remove) {
      System.out.println("Removing server: " + server.cluster().getMember().memberId());
      leaveFuture = server.leave();
    } else {
      System.out.println("Shutting down server: " + server.cluster().getMember().memberId());
      leaveFuture = server.shutdown();
    }
    leaveFuture.whenComplete((result, error) -> {
      restartTimers.put(serverIndex, context.schedule(Duration.ofSeconds(randomNumber(120) + 10), () -> {
        restartTimers.remove(serverIndex);
        RaftServer newServer = createServer(server.cluster().getMember());
        servers.set(serverIndex, newServer);
        CompletableFuture<RaftServer> joinFuture;
        if (remove) {
          System.out.println("Adding server: " + newServer.cluster().getMember().memberId());
          joinFuture = newServer.join(members.get(members.size() - 1).memberId());
        } else {
          System.out.println("Bootstrapping server: " + newServer.cluster().getMember().memberId());
          joinFuture = newServer.bootstrap(members.stream().map(RaftMember::memberId).collect(Collectors.toList()));
        }
        joinFuture.whenComplete((result2, error2) -> {
          scheduleRestarts(context);
        });
      }));
    });
  }));
}
 

@Override
public CompletableFuture<VoteResponse> onVote(VoteRequest request) {
  raft.checkThread();
  logRequest(request);

  // If the request indicates a term that is greater than the current term then
  // assign that term and leader to the current context and step down as a candidate.
  if (updateTermAndLeader(request.term(), null)) {
    CompletableFuture<VoteResponse> future = super.onVote(request);
    raft.transition(RaftServer.Role.FOLLOWER);
    return future;
  }

  // If the vote request is not for this candidate then reject the vote.
  if (request.candidate() == raft.getCluster().getMember().memberId()) {
    return CompletableFuture.completedFuture(logResponse(VoteResponse.builder()
        .withStatus(RaftResponse.Status.OK)
        .withTerm(raft.getTerm())
        .withVoted(true)
        .build()));
  } else {
    return CompletableFuture.completedFuture(logResponse(VoteResponse.builder()
        .withStatus(RaftResponse.Status.OK)
        .withTerm(raft.getTerm())
        .withVoted(false)
        .build()));
  }
}
 

@Override
public CompletableFuture<AppendResponse> onAppend(AppendRequest request) {
  raft.checkThread();

  // If the request indicates a term that is greater than the current term then
  // assign that term and leader to the current context and step down as a candidate.
  if (request.term() >= raft.getTerm()) {
    raft.setTerm(request.term());
    raft.transition(RaftServer.Role.FOLLOWER);
  }
  return super.onAppend(request);
}
 

@Override
public synchronized CompletableFuture<RaftRole> start() {
  if (raft.getCluster().getActiveMemberStates().isEmpty()) {
    log.debug("Single member cluster. Transitioning directly to leader.");
    raft.setTerm(raft.getTerm() + 1);
    raft.setLastVotedFor(raft.getCluster().getMember().memberId());
    raft.transition(RaftServer.Role.LEADER);
    return CompletableFuture.completedFuture(this);
  }
  return super.start().thenRun(this::startElection).thenApply(v -> this);
}
 

/**
 * Truncates uncommitted entries from the log.
 */
private void truncateUncommittedEntries() {
  if (role() == RaftServer.Role.PASSIVE) {
    final RaftLogWriter writer = raft.getLogWriter();
    writer.truncate(raft.getCommitIndex());
  }
}
 

/**
 * Handles a {@link io.atomix.protocols.raft.protocol.RaftResponse.Status#ERROR} response.
 */
protected void handleAppendResponseError(RaftMemberContext member, AppendRequest request, AppendResponse response) {
  // If we've received a greater term, update the term and transition back to follower.
  if (response.term() > raft.getTerm()) {
    log.debug("Received higher term from {}", member.getMember().memberId());
    raft.setTerm(response.term());
    raft.setLeader(null);
    raft.transition(RaftServer.Role.FOLLOWER);
  } else {
    super.handleAppendResponseError(member, request, response);
  }
}
 

@Override
protected void handleAppendResponseOk(RaftMemberContext member, AppendRequest request, AppendResponse response) {
  // Reset the member failure count and update the member's availability status if necessary.
  succeedAttempt(member);

  // If replication succeeded then trigger commit futures.
  if (response.succeeded()) {
    member.appendSucceeded();
    updateMatchIndex(member, response);

    // If entries were committed to the replica then check commit indexes.
    if (!request.entries().isEmpty()) {
      commitEntries();
    }

    // If there are more entries to send then attempt to send another commit.
    if (hasMoreEntries(member)) {
      appendEntries(member);
    }
  }
  // If we've received a greater term, update the term and transition back to follower.
  else if (response.term() > raft.getTerm()) {
    raft.setTerm(response.term());
    raft.setLeader(null);
    raft.transition(RaftServer.Role.FOLLOWER);
  }
  // If the response failed, the follower should have provided the correct last index in their log. This helps
  // us converge on the matchIndex faster than by simply decrementing nextIndex one index at a time.
  else {
    member.appendFailed();
    resetMatchIndex(member, response);
    resetNextIndex(member, response);
    resetSnapshotIndex(member, response);

    // If there are more entries to send then attempt to send another commit.
    if (hasMoreEntries(member)) {
      appendEntries(member);
    }
  }
}
 

protected AbstractRole(RaftContext raft) {
  this.raft = raft;
  this.log = ContextualLoggerFactory.getLogger(getClass(), LoggerContext.builder(RaftServer.class)
      .addValue(raft.getName())
      .add("role", role())
      .build());
}
 

private RaftServer buildServer() {
  return RaftServer.builder(localMemberId)
      .withName(partition.name())
      .withMembershipService(membershipService)
      .withProtocol(new RaftServerCommunicator(
          partition.name(),
          Serializer.using(RaftNamespaces.RAFT_PROTOCOL),
          clusterCommunicator))
      .withPrimitiveTypes(primitiveTypes)
      .withElectionTimeout(config.getElectionTimeout())
      .withHeartbeatInterval(config.getHeartbeatInterval())
      .withSessionTimeout(config.getDefaultSessionTimeout())
      .withStorage(RaftStorage.builder()
          .withPrefix(partition.name())
          .withDirectory(partition.dataDirectory())
          .withStorageLevel(config.getStorageConfig().getLevel())
          .withMaxSegmentSize((int) config.getStorageConfig().getSegmentSize().bytes())
          .withMaxEntrySize((int) config.getStorageConfig().getMaxEntrySize().bytes())
          .withFlushOnCommit(config.getStorageConfig().isFlushOnCommit())
          .withDynamicCompaction(config.getCompactionConfig().isDynamic())
          .withFreeDiskBuffer(config.getCompactionConfig().getFreeDiskBuffer())
          .withFreeMemoryBuffer(config.getCompactionConfig().getFreeMemoryBuffer())
          .withNamespace(RaftNamespaces.RAFT_STORAGE)
          .build())
      .withThreadContextFactory(threadContextFactory)
      .build();
}
 

@Override
public CompletableFuture<RaftPartitionServer> start() {
  log.info("Starting server for partition {}", partition.id());
  CompletableFuture<RaftServer> serverOpenFuture;
  if (partition.members().contains(localMemberId)) {
    if (server != null && server.isRunning()) {
      return CompletableFuture.completedFuture(null);
    }
    synchronized (this) {
      try {
        initServer();
      } catch (StorageException e) {
        return Futures.exceptionalFuture(e);
      }
    }
    serverOpenFuture = server.bootstrap(partition.members());
  } else {
    serverOpenFuture = CompletableFuture.completedFuture(null);
  }
  return serverOpenFuture.whenComplete((r, e) -> {
    if (e == null) {
      log.debug("Successfully started server for partition {}", partition.id());
    } else {
      log.warn("Failed to start server for partition {}", partition.id(), e);
    }
  }).thenApply(v -> this);
}
 

/**
 * Leaves the cluster.
 */
@Override
public synchronized CompletableFuture<Void> leave() {
  if (leaveFuture != null) {
    return leaveFuture;
  }

  leaveFuture = new CompletableFuture<>();

  raft.getThreadContext().execute(() -> {
    // If a join attempt is still underway, cancel the join and complete the join future exceptionally.
    // The join future will be set to null once completed.
    cancelJoinTimer();
    if (joinFuture != null) {
      joinFuture.completeExceptionally(new IllegalStateException("failed to join cluster"));
    }

    // If there are no remote members to leave, simply transition the server to INACTIVE.
    if (getActiveMemberStates().isEmpty() && configuration.index() <= raft.getCommitIndex()) {
      log.trace("Single member cluster. Transitioning directly to inactive.");
      raft.transition(RaftServer.Role.INACTIVE);
      leaveFuture.complete(null);
    } else {
      leave(leaveFuture);
    }
  });

  return leaveFuture.whenComplete((result, error) -> leaveFuture = null);
}
 

/**
 * Handles a {@link RaftResponse.Status#OK} response.
 */
protected void handleAppendResponseOk(RaftMemberContext member, AppendRequest request, AppendResponse response) {
  // Reset the member failure count and update the member's availability status if necessary.
  succeedAttempt(member);

  // If replication succeeded then trigger commit futures.
  if (response.succeeded()) {
    updateMatchIndex(member, response);

    // If there are more entries to send then attempt to send another commit.
    if (request.prevLogIndex() != response.lastLogIndex() && hasMoreEntries(member)) {
      appendEntries(member);
    }
  }
  // If we've received a greater term, update the term and transition back to follower.
  else if (response.term() > raft.getTerm()) {
    raft.setTerm(response.term());
    raft.setLeader(null);
    raft.transition(RaftServer.Role.FOLLOWER);
  }
  // If the response failed, the follower should have provided the correct last index in their log. This helps
  // us converge on the matchIndex faster than by simply decrementing nextIndex one index at a time.
  else {
    resetMatchIndex(member, response);
    resetNextIndex(member, response);
    resetSnapshotIndex(member, response);

    // If there are more entries to send then attempt to send another commit.
    if (response.lastLogIndex() != request.prevLogIndex() && hasMoreEntries(member)) {
      appendEntries(member);
    }
  }
}
 

public RaftServiceManager(RaftContext raft, ThreadContext stateContext, ThreadContextFactory threadContextFactory) {
  this.raft = checkNotNull(raft, "state cannot be null");
  this.log = raft.getLog();
  this.reader = log.openReader(1, RaftLogReader.Mode.COMMITS);
  this.stateContext = stateContext;
  this.threadContextFactory = threadContextFactory;
  this.logger = ContextualLoggerFactory.getLogger(getClass(), LoggerContext.builder(RaftServer.class)
      .addValue(raft.getName())
      .build());
  this.lastEnqueued = reader.getFirstIndex() - 1;
  scheduleSnapshots();
}
 

/**
 * Starts the server.
 */
private CompletableFuture<RaftServer> start(Supplier<CompletableFuture<Void>> joiner) {
  if (started) {
    return CompletableFuture.completedFuture(this);
  }

  if (openFuture == null) {
    synchronized (this) {
      if (openFuture == null) {
        CompletableFuture<RaftServer> future = new AtomixFuture<>();
        openFuture = future;
        joiner.get().whenComplete((result, error) -> {
          if (error == null) {
            context.awaitState(RaftContext.State.READY, state -> {
              started = true;
              future.complete(null);
            });
          } else {
            future.completeExceptionally(error);
          }
        });
      }
    }
  }

  return openFuture.whenComplete((result, error) -> {
    if (error == null) {
      log.debug("Server started successfully!");
    } else {
      log.warn("Failed to start server!");
    }
  });
}
 

@Override
public CompletableFuture<VoteResponse> onVote(VoteRequest request) {
  raft.checkThread();
  logRequest(request);

  // If the request indicates a term that is greater than the current term then
  // assign that term and leader to the current context.
  boolean transition = updateTermAndLeader(request.term(), null);

  CompletableFuture<VoteResponse> future = CompletableFuture.completedFuture(logResponse(handleVote(request)));
  if (transition) {
    raft.transition(RaftServer.Role.FOLLOWER);
  }
  return future;
}
 

@Override
public CompletableFuture<RaftServer> bootstrap(Collection<MemberId> cluster) {
  return start(() -> cluster().bootstrap(cluster));
}
 

@SuppressWarnings("unchecked")
public RaftContext(
    String name,
    MemberId localMemberId,
    ClusterMembershipService membershipService,
    RaftServerProtocol protocol,
    RaftStorage storage,
    PrimitiveTypeRegistry primitiveTypes,
    ThreadContextFactory threadContextFactory,
    boolean closeOnStop) {
  this.name = checkNotNull(name, "name cannot be null");
  this.membershipService = checkNotNull(membershipService, "membershipService cannot be null");
  this.protocol = checkNotNull(protocol, "protocol cannot be null");
  this.storage = checkNotNull(storage, "storage cannot be null");
  this.primitiveTypes = checkNotNull(primitiveTypes, "registry cannot be null");
  this.log = ContextualLoggerFactory.getLogger(getClass(), LoggerContext.builder(RaftServer.class)
      .addValue(name)
      .build());

  // Lock the storage directory.
  if (!storage.lock(localMemberId.id())) {
    throw new StorageException("Failed to acquire storage lock; ensure each Raft server is configured with a distinct storage directory");
  }

  String baseThreadName = String.format("raft-server-%s", name);
  this.threadContext = new SingleThreadContext(namedThreads(baseThreadName, log));
  this.loadContext = new SingleThreadContext(namedThreads(baseThreadName + "-load", log));
  this.stateContext = new SingleThreadContext(namedThreads(baseThreadName + "-state", log));

  this.threadContextFactory = checkNotNull(threadContextFactory, "threadContextFactory cannot be null");
  this.closeOnStop = closeOnStop;

  this.loadMonitor = new LoadMonitor(LOAD_WINDOW_SIZE, HIGH_LOAD_THRESHOLD, loadContext);

  // Open the metadata store.
  this.meta = storage.openMetaStore();

  // Load the current term and last vote from disk.
  this.term = meta.loadTerm();
  this.lastVotedFor = meta.loadVote();

  // Construct the core log, reader, writer, and compactor.
  this.raftLog = storage.openLog();
  this.logWriter = raftLog.writer();
  this.logReader = raftLog.openReader(1, RaftLogReader.Mode.ALL);

  // Open the snapshot store.
  this.snapshotStore = storage.openSnapshotStore();

  // Create a new internal server state machine.
  this.stateMachine = new RaftServiceManager(this, stateContext, threadContextFactory);

  this.cluster = new RaftClusterContext(localMemberId, this);

  // Register protocol listeners.
  registerHandlers(protocol);
}
 

/**
 * Creates a Raft server.
 */
private RaftServer createServer(Member member, List<Node> members) {
  RaftServerProtocol protocol;
  ManagedMessagingService messagingService;
  if (USE_NETTY) {
    messagingService = (ManagedMessagingService) new NettyMessagingService("test", member.address(), new MessagingConfig())
        .start()
        .join();
    messagingServices.add(messagingService);
    protocol = new RaftServerMessagingProtocol(messagingService, PROTOCOL_SERIALIZER, addressMap::get);
  } else {
    protocol = protocolFactory.newServerProtocol(member.id());
  }

  BootstrapService bootstrapService = new BootstrapService() {
    @Override
    public MessagingService getMessagingService() {
      return messagingService;
    }

    @Override
    public UnicastService getUnicastService() {
      return new UnicastServiceAdapter();
    }

    @Override
    public BroadcastService getBroadcastService() {
      return new BroadcastServiceAdapter();
    }
  };

  RaftServer.Builder builder = RaftServer.builder(member.id())
      .withProtocol(protocol)
      .withThreadModel(ThreadModel.SHARED_THREAD_POOL)
      .withMembershipService(new DefaultClusterMembershipService(
          member,
          Version.from("1.0.0"),
          new DefaultNodeDiscoveryService(bootstrapService, member, new BootstrapDiscoveryProvider(members)),
          bootstrapService,
          new HeartbeatMembershipProtocol(new HeartbeatMembershipProtocolConfig())))
      .withStorage(RaftStorage.builder()
          .withStorageLevel(StorageLevel.DISK)
          .withDirectory(new File(String.format("target/perf-logs/%s", member.id())))
          .withNamespace(STORAGE_NAMESPACE)
          .withMaxSegmentSize(1024 * 1024 * 64)
          .withDynamicCompaction()
          .withFlushOnCommit(false)
          .build());

  RaftServer server = builder.build();
  servers.add(server);
  return server;
}