org.apache.ratis.util.TimeDuration Java Examples

@Test
public void testExponentialBackoffRetry() {
  TimeDuration baseSleep = TimeDuration.valueOf(1, TimeUnit.SECONDS);
  TimeDuration maxSleep = TimeDuration.valueOf(40, TimeUnit.SECONDS);

  // Test maxAttempts
  ExponentialBackoffRetry retryPolicy = createPolicy(baseSleep, null, 1);
  Assert.assertFalse(retryPolicy.handleAttemptFailure(() -> 1).shouldRetry());

  try {
    // baseSleep should not be null
    createPolicy(null, null, 1);
    Assert.fail("Policy creation should have failed");
  } catch (Throwable t) {
  }

  // test policy without max sleep
  retryPolicy = createPolicy(baseSleep, null,100);
  assertSleep(retryPolicy, baseSleep, null);

  // test policy with max sleep
  retryPolicy = createPolicy(baseSleep, maxSleep,100);
  assertSleep(retryPolicy, baseSleep, maxSleep);
}
 

public static RaftServerProxy newRaftServer(
    RaftPeerId id, StateMachine.Registry stateMachineRegistry, RaftProperties properties, Parameters parameters)
    throws IOException {
  final TimeDuration sleepTime = TimeDuration.valueOf(500, TimeUnit.MILLISECONDS);
  final RaftServerProxy proxy;
  try {
    // attempt multiple times to avoid temporary bind exception
    proxy = JavaUtils.attemptRepeatedly(
        () -> new RaftServerProxy(id, stateMachineRegistry, properties, parameters),
        5, sleepTime, "new RaftServerProxy", RaftServerProxy.LOG);
  } catch (InterruptedException e) {
    throw IOUtils.toInterruptedIOException(
        "Interrupted when creating RaftServer " + id, e);
  }
  return proxy;
}
 

/**
 * Build {@link ExceptionDependentRetry} object.
 * @param exceptionPolicyMap
 * @param defaultPolicy
 * @return ExceptionDependentRetry
 */
private ExceptionDependentRetry buildExceptionBasedRetry(Map<Class<?
    extends Throwable>, Pair> exceptionPolicyMap, Pair defaultPolicy) {
  final ExceptionDependentRetry.Builder policy =
      ExceptionDependentRetry.newBuilder();
  exceptionPolicyMap.forEach((k, v) -> {
    policy.setExceptionToPolicy(k,
        RetryPolicies.retryUpToMaximumCountWithFixedSleep(v.retries,
            TimeDuration.valueOf(v.sleepTime, TimeUnit.SECONDS)));
  });

  policy.setDefaultPolicy(RetryPolicies.retryUpToMaximumCountWithFixedSleep(defaultPolicy.retries,
      TimeDuration.valueOf(defaultPolicy.sleepTime, TimeUnit.SECONDS)));

  return policy.build();
}
 

@Test
public void testRetryMultipleTimesWithFixedSleep() {
  final int n = 4;
  final TimeDuration sleepTime = HUNDRED_MILLIS;
  final RetryPolicy policy = RetryPolicies.retryUpToMaximumCountWithFixedSleep(n, sleepTime);
  for(int i = 1; i < 2*n; i++) {
    final int attempt = i;
    final RetryPolicy.Event event = () -> attempt;
    final RetryPolicy.Action action = policy.handleAttemptFailure(event);

    final boolean expected = i < n;
    Assert.assertEquals(expected, action.shouldRetry());
    if (expected) {
      Assert.assertEquals(sleepTime, action.getSleepTime());
    } else {
      Assert.assertEquals(0L, action.getSleepTime().getDuration());
    }
  }
}
 

@BeforeClass
public static void setUp() {
  RaftServerImpl raftServer = mock(RaftServerImpl.class);

  RaftGroupId raftGroupId = RaftGroupId.randomId();
  RaftPeerId raftPeerId = RaftPeerId.valueOf("TestId");
  RaftGroupMemberId raftGroupMemberId = RaftGroupMemberId
      .valueOf(raftPeerId, raftGroupId);
  when(raftServer.getMemberId()).thenReturn(raftGroupMemberId);

  retryCache = new RetryCache(TimeDuration.valueOf(60, TimeUnit.SECONDS));
  when(raftServer.getRetryCache()).thenReturn(retryCache);

  RaftServerMetrics raftServerMetrics = RaftServerMetrics
      .getRaftServerMetrics(raftServer);
  ratisMetricRegistry = raftServerMetrics.getRegistry();
}
 

static <SERVER extends RaftServer> void blockQueueAndSetDelay(
    Collection<SERVER> servers,
    DelayLocalExecutionInjection injection, String leaderId, int delayMs,
    TimeDuration maxTimeout) throws InterruptedException {
  // block reqeusts sent to leader if delayMs > 0
  final boolean block = delayMs > 0;
  LOG.debug("{} requests sent to leader {} and set {}ms delay for the others",
      block? "Block": "Unblock", leaderId, delayMs);
  if (block) {
    BlockRequestHandlingInjection.getInstance().blockReplier(leaderId);
  } else {
    BlockRequestHandlingInjection.getInstance().unblockReplier(leaderId);
  }

  // delay RaftServerRequest for other servers
  servers.stream().filter(s -> !s.getId().toString().equals(leaderId))
      .forEach(s -> {
        if (block) {
          injection.setDelayMs(s.getId().toString(), delayMs);
        } else {
          injection.removeDelay(s.getId().toString());
        }
      });

  Thread.sleep(3 * maxTimeout.toLong(TimeUnit.MILLISECONDS));
}
 

@Test
public void testWatchRequestClientTimeout() throws Exception {
  final RaftProperties p = getProperties();
  RaftServerConfigKeys.Watch.setTimeout(p, TimeDuration.valueOf(100,
      TimeUnit.SECONDS));
  RaftClientConfigKeys.Rpc.setWatchRequestTimeout(p,
      TimeDuration.valueOf(15, TimeUnit.SECONDS));
  try {
    runWithNewCluster(NUM_SERVERS,
        cluster -> runSingleTest(WatchRequestTests::runTestWatchRequestClientTimeout, cluster, LOG));
  } finally {
    RaftServerConfigKeys.Watch.setTimeout(p, RaftServerConfigKeys.Watch.TIMEOUT_DEFAULT);
    RaftClientConfigKeys.Rpc.setWatchRequestTimeout(p,
        RaftClientConfigKeys.Rpc.WATCH_REQUEST_TIMEOUT_DEFAULT);
  }
}
 

@Test
public void testLeaderElectionDetection() throws Exception {
  RaftTestUtil.waitForLeader(cluster);
  final TimeDuration noLeaderTimeout = RaftServerConfigKeys.Notification.noLeaderTimeout(cluster.getProperties());

  RaftServerImpl healthyFollower = cluster.getFollowers().get(1);
  RaftServerImpl failedFollower = cluster.getFollowers().get(0);
  // fail the leader and one of the followers to that quorum is not present
  // for next leader election to succeed.
  cluster.killServer(failedFollower.getId());
  cluster.killServer(cluster.getLeader().getId());

  // Wait to ensure that leader election is triggered and also state machine callback is triggered
  noLeaderTimeout.sleep();
  noLeaderTimeout.sleep();

  RaftProtos.RoleInfoProto roleInfoProto =
      SimpleStateMachine4Testing.get(healthyFollower).getLeaderElectionTimeoutInfo();
  Assert.assertNotNull(roleInfoProto);

  Assert.assertEquals(roleInfoProto.getRole(), RaftProtos.RaftPeerRole.CANDIDATE);
  final long noLeaderTimeoutMs = noLeaderTimeout.toLong(TimeUnit.MILLISECONDS);
  Assert.assertTrue(roleInfoProto.getCandidateInfo().getLastLeaderElapsedTimeMs() > noLeaderTimeoutMs);
}
 

public static void testRequestTimeout(boolean async, MiniRaftCluster cluster, Logger LOG) throws Exception {
  waitForLeader(cluster);
  final Timestamp startTime = Timestamp.currentTime();
  try (final RaftClient client = cluster.createClient()) {
    // Get the next callId to be used by the client
    long callId = RaftClientTestUtil.getCallId(client);
    // Create an entry corresponding to the callId and clientId
    // in each server's retry cache.
    cluster.getServerAliveStream().forEach(
        raftServer -> RetryCacheTestUtil.getOrCreateEntry(raftServer.getRetryCache(), client.getId(), callId));
    // Client request for the callId now waits
    // as there is already a cache entry in the server for the request.
    // Ideally the client request should timeout and the client should retry.
    // The retry is successful when the retry cache entry for the corresponding callId and clientId expires.
    if (async) {
      CompletableFuture<RaftClientReply> replyFuture = client.sendAsync(new SimpleMessage("abc"));
      replyFuture.get();
    } else {
      client.send(new SimpleMessage("abc"));
    }
    // Eventually the request would be accepted by the server
    // when the retry cache entry is invalidated.
    // The duration for which the client waits should be more than the retryCacheExpiryDuration.
    final TimeDuration duration = startTime.elapsedTime();
    TimeDuration retryCacheExpiryDuration = RaftServerConfigKeys.RetryCache.expiryTime(cluster.getProperties());
    Assert.assertTrue(duration.compareTo(retryCacheExpiryDuration) >= 0);
  }
}
 

Pair(int numRetries, TimeDuration sleepTime) {
  if (numRetries < 0) {
    throw new IllegalArgumentException("numRetries = " + numRetries+" < 0");
  }
  if (sleepTime.isNegative()) {
    throw new IllegalArgumentException("sleepTime = " + sleepTime + " < 0");
  }

  this.numRetries = numRetries;
  this.sleepTime = sleepTime;
}
 

@Test
public void testWatchRequestTimeout() throws Exception {
  final RaftProperties p = getProperties();
  RaftServerConfigKeys.setWatchTimeout(p, TimeDuration.valueOf(500, TimeUnit.MILLISECONDS));
  RaftServerConfigKeys.setWatchTimeoutDenomination(p, TimeDuration.valueOf(100, TimeUnit.MILLISECONDS));
  try {
    runWithNewCluster(NUM_SERVERS,
        cluster -> runTest(WatchRequestTests::runTestWatchRequestTimeout, cluster, LOG));
  } finally {
    RaftServerConfigKeys.setWatchTimeout(p, RaftServerConfigKeys.WATCH_TIMEOUT_DEFAULT);
    RaftServerConfigKeys.setWatchTimeoutDenomination(p, RaftServerConfigKeys.WATCH_TIMEOUT_DENOMINATION_DEFAULT);
  }
}
 

void runTestAppendEntriesTimeout(CLUSTER cluster) throws Exception {
  LOG.info("Running testAppendEntriesTimeout");
  final TimeDuration oldExpiryTime = RaftServerConfigKeys.RetryCache.expiryTime(getProperties());
  RaftServerConfigKeys.RetryCache.setExpiryTime(getProperties(), TimeDuration.valueOf(20, TimeUnit.SECONDS));
  waitForLeader(cluster);
  long time = System.currentTimeMillis();
  long waitTime = 5000;
  try (final RaftClient client = cluster.createClient()) {
    // block append requests
    cluster.getServerAliveStream()
        .filter(impl -> !impl.isLeader())
        .map(SimpleStateMachine4Testing::get)
        .forEach(SimpleStateMachine4Testing::blockWriteStateMachineData);

    CompletableFuture<RaftClientReply> replyFuture = client.sendAsync(new SimpleMessage("abc"));
    Thread.sleep(waitTime);
    // replyFuture should not be completed until append request is unblocked.
    Assert.assertTrue(!replyFuture.isDone());
    // unblock append request.
    cluster.getServerAliveStream()
        .filter(impl -> !impl.isLeader())
        .map(SimpleStateMachine4Testing::get)
        .forEach(SimpleStateMachine4Testing::unblockWriteStateMachineData);

    replyFuture.get();
    Assert.assertTrue(System.currentTimeMillis() - time > waitTime);
  }

  //reset for the other tests
  RaftServerConfigKeys.RetryCache.setExpiryTime(getProperties(), oldExpiryTime);
}
 

RetryForeverWithSleep(TimeDuration sleepTime) {
  if (sleepTime.isNegative()) {
    throw new IllegalArgumentException(
        "sleepTime = " + sleepTime.getDuration() + " < 0");
  }
  this.sleepTime = sleepTime;
}
 

void runTestToRaftClientRequestProto(int n, SizeInBytes messageSize)
    throws Exception {
  final ClientId clientId = ClientId.randomId();
  final RaftPeerId leaderId = RaftPeerId.valueOf("s0");
  final RaftGroupId groupId = RaftGroupId.randomId();


  TimeDuration toProto = TimeDuration.ZERO;
  TimeDuration toRequest = TimeDuration.ZERO;

  for(int i = 0; i < n; i++) {
    final ByteString bytes = newByteString(messageSize.getSizeInt(), i);
    final RaftClientRequest request = new RaftClientRequest(clientId, leaderId, groupId,
        1, () -> bytes, RaftClientRequest.writeRequestType(), null);

    final Timestamp startTime = Timestamp.currentTime();
    final RaftClientRequestProto proto = ClientProtoUtils.toRaftClientRequestProto(request);
    final TimeDuration p = startTime.elapsedTime();
    final RaftClientRequest computed = ClientProtoUtils.toRaftClientRequest(proto);
    final TimeDuration r = startTime.elapsedTime().subtract(p);

    Assert.assertEquals(request.getMessage().getContent(), computed.getMessage().getContent());
    toProto = toProto.add(p);
    toRequest = toRequest.add(r);

  }

  System.out.printf("%nmessageSize=%s, n=%d%n", messageSize, n);
  print("toProto  ", toProto, n);
  print("toRequest", toRequest, n);
}
 

@Test
public void testLateServerStart() throws Exception {
  final int numServer = 3;
  LOG.info("Running testLateServerStart");
  final MiniRaftCluster cluster = newCluster(numServer);
  cluster.initServers();

  // start all except one servers
  final Iterator<RaftServerProxy> i = cluster.getServers().iterator();
  for(int j = 1; j < numServer; j++) {
    i.next().start();
  }

  final RaftServerImpl leader = waitForLeader(cluster);
  final TimeDuration sleepTime = TimeDuration.valueOf(3, TimeUnit.SECONDS);
  LOG.info("sleep " + sleepTime);
  sleepTime.sleep();

  // start the last server
  final RaftServerProxy lastServer = i.next();
  lastServer.start();
  final RaftPeerId lastServerLeaderId = JavaUtils.attempt(
      () -> getLeader(lastServer.getImpls().iterator().next().getState()),
      10, 1000, "getLeaderId", LOG);
  LOG.info(cluster.printServers());
  Assert.assertEquals(leader.getId(), lastServerLeaderId);
}
 

public static void waitAndCheckNewConf(MiniRaftCluster cluster,
    RaftPeer[] peers, int numOfRemovedPeers, Collection<RaftPeerId> deadPeers)
    throws Exception {
  final TimeDuration sleepTime = cluster.getTimeoutMax().apply(n -> n * (numOfRemovedPeers + 2));
  JavaUtils.attempt(() -> waitAndCheckNewConf(cluster, Arrays.asList(peers), deadPeers),
      10, sleepTime, "waitAndCheckNewConf", LOG);
}
 

void runTestAppendEntriesTimeout(CLUSTER cluster) throws Exception {
  LOG.info("Running testAppendEntriesTimeout");
  final TimeDuration oldExpiryTime = RaftServerConfigKeys.RetryCache.expiryTime(getProperties());
  RaftServerConfigKeys.RetryCache.setExpiryTime(getProperties(), TimeDuration.valueOf(20, TimeUnit.SECONDS));
  waitForLeader(cluster);
  long time = System.currentTimeMillis();
  long waitTime = 5000;
  try (final RaftClient client = cluster.createClient()) {
    // block append requests
    cluster.getServerAliveStream()
        .filter(impl -> !impl.isLeader())
        .map(SimpleStateMachine4Testing::get)
        .forEach(SimpleStateMachine4Testing::blockWriteStateMachineData);

    CompletableFuture<RaftClientReply> replyFuture = client.sendAsync(new SimpleMessage("abc"));
    Thread.sleep(waitTime);
    // replyFuture should not be completed until append request is unblocked.
    Assert.assertFalse(replyFuture.isDone());
    // unblock append request.
    cluster.getServerAliveStream()
        .filter(impl -> !impl.isLeader())
        .map(SimpleStateMachine4Testing::get)
        .forEach(SimpleStateMachine4Testing::unblockWriteStateMachineData);

    Assert.assertTrue(replyFuture.get().isSuccess());
    Assert.assertTrue(System.currentTimeMillis() - time > waitTime);
  }

  //reset for the other tests
  RaftServerConfigKeys.RetryCache.setExpiryTime(getProperties(), oldExpiryTime);
}
 

/**
 * @param capacity the capacity of the cache
 * @param expirationTime time for an entry to expire in milliseconds
 */
RetryCache(int capacity, TimeDuration expirationTime) {
  capacity = Math.max(capacity, MIN_CAPACITY);
  cache = CacheBuilder.newBuilder().maximumSize(capacity)
      .expireAfterWrite(expirationTime.toLong(TimeUnit.MILLISECONDS),
          TimeUnit.MILLISECONDS).build();
}
 

RetryLimited(int maxAttempts, TimeDuration sleepTime) {
  if (maxAttempts < 0) {
    throw new IllegalArgumentException("maxAttempts = " + maxAttempts+" < 0");
  }
  if (sleepTime.isNegative()) {
    throw new IllegalArgumentException(
        "sleepTime = " + sleepTime.getDuration() + " < 0");
  }

  this.maxAttempts = maxAttempts;
  this.sleepTime = sleepTime;
}
 

private void setTimeoutForRetryCache(RaftProperties properties) {
  TimeUnit timeUnit;
  long duration;
  timeUnit =
      OzoneConfigKeys.DFS_RATIS_SERVER_RETRY_CACHE_TIMEOUT_DURATION_DEFAULT
          .getUnit();
  duration = conf.getTimeDuration(
      OzoneConfigKeys.DFS_RATIS_SERVER_RETRY_CACHE_TIMEOUT_DURATION_KEY,
      OzoneConfigKeys.DFS_RATIS_SERVER_RETRY_CACHE_TIMEOUT_DURATION_DEFAULT
          .getDuration(), timeUnit);
  final TimeDuration retryCacheTimeout =
      TimeDuration.valueOf(duration, timeUnit);
  RaftServerConfigKeys.RetryCache
      .setExpiryTime(properties, retryCacheTimeout);
}
 

static BiConsumer<String, TimeDuration> requireNonNegativeTimeDuration() {
  return (key, value) -> {
    if (value.isNegative()) {
      throw new IllegalArgumentException(
          key + " = " + value + " is negative.");
    }
  };
}
 

static BiConsumer<String, TimeDuration> requireNonNegativeTimeDuration() {
  return (key, value) -> {
    if (value.isNegative()) {
      throw new IllegalArgumentException(
          key + " = " + value + " is negative.");
    }
  };
}
 

@SafeVarargs
static TimeDuration getTimeDuration(
    BiFunction<String, TimeDuration, TimeDuration> getter,
    String key, TimeDuration defaultValue, Consumer<String> logger, BiConsumer<String, TimeDuration>... assertions) {
  final TimeDuration value = get(getter, key, defaultValue, logger, assertions);
  requireNonNegativeTimeDuration().accept(key, value);
  return value;
}
 

@BeforeClass
public static void beforeClass() {
    cluster = new LogServiceCluster(3);
    cluster.createWorkers(3);
    workers = cluster.getWorkers();
    assert(workers.size() == 3);

    RaftProperties properties = new RaftProperties();
    RaftClientConfigKeys.Rpc.setRequestTimeout(properties, TimeDuration.valueOf(15, TimeUnit.SECONDS));

    cluster.getMasters().parallelStream().forEach(master ->
        ((MetaStateMachine)master.getMetaStateMachine()).setProperties(properties));

    client = new LogServiceClient(cluster.getMetaIdentity(), properties) {
      @Override
      public LogStream createLog(LogName logName) throws IOException {
        createCount.incrementAndGet();
        return super.createLog(logName);
      }

      @Override
      public void deleteLog(LogName logName) throws IOException {
        deleteCount.incrementAndGet();
        super.deleteLog(logName);
      }

      @Override
      public List<LogInfo> listLogs() throws IOException {
        listCount.incrementAndGet();
        return super.listLogs();
      }
    };
}
 

public static Consumer<RatisMetricRegistry> consoleReporter(TimeDuration rate) {
  return ratisMetricRegistry -> {
    ConsoleReporter reporter = ConsoleReporter.forRegistry(ratisMetricRegistry.getDropWizardMetricRegistry())
        .convertRatesTo(TimeUnit.SECONDS).convertDurationsTo(TimeUnit.MILLISECONDS).build();
    reporter.start(rate.getDuration(), rate.getUnit());
    ratisMetricRegistry.setConsoleReporter(reporter);
  };
}
 

public GrpcServerProtocolClient(RaftPeer target, int flowControlWindow,
    TimeDuration requestTimeoutDuration, GrpcTlsConfig tlsConfig) {
  raftPeerId = target.getId();
  NettyChannelBuilder channelBuilder =
      NettyChannelBuilder.forTarget(target.getAddress());

  if (tlsConfig!= null) {
    SslContextBuilder sslContextBuilder = GrpcSslContexts.forClient();
    if (tlsConfig.isFileBasedConfig()) {
      sslContextBuilder.trustManager(tlsConfig.getTrustStoreFile());
    } else {
      sslContextBuilder.trustManager(tlsConfig.getTrustStore());
    }
    if (tlsConfig.getMtlsEnabled()) {
      if (tlsConfig.isFileBasedConfig()) {
        sslContextBuilder.keyManager(tlsConfig.getCertChainFile(),
            tlsConfig.getPrivateKeyFile());
      } else {
        sslContextBuilder.keyManager(tlsConfig.getPrivateKey(),
            tlsConfig.getCertChain());
      }
    }
    try {
      channelBuilder.useTransportSecurity().sslContext(sslContextBuilder.build());
    } catch (Exception ex) {
      throw new IllegalArgumentException("Failed to build SslContext, peerId=" + raftPeerId
          + ", tlsConfig=" + tlsConfig, ex);
    }
  } else {
    channelBuilder.negotiationType(NegotiationType.PLAINTEXT);
  }
  channel = channelBuilder.flowControlWindow(flowControlWindow).build();
  blockingStub = RaftServerProtocolServiceGrpc.newBlockingStub(channel);
  asyncStub = RaftServerProtocolServiceGrpc.newStub(channel);
  this.requestTimeoutDuration = requestTimeoutDuration;
}
 

@Test
public void testRequestTimeout() throws Exception {
  final TimeDuration oldExpiryTime = RaftServerConfigKeys.RetryCache.expiryTime(getProperties());
  RaftServerConfigKeys.RetryCache.setExpiryTime(getProperties(), FIVE_SECONDS);
  RaftClientConfigKeys.Async.Experimental.setSendDummyRequest(getProperties(), false);
  runWithNewCluster(NUM_SERVERS, cluster -> RaftBasicTests.testRequestTimeout(true, cluster, LOG));

  //reset for the other tests
  RaftServerConfigKeys.RetryCache.setExpiryTime(getProperties(), oldExpiryTime);
  RaftClientConfigKeys.Async.Experimental.setSendDummyRequest(getProperties(), true);
}
 

private void scheduleWithTimeout(PendingOrderedRequest pending,
    RaftClientRequest request, RetryPolicy retryPolicy, Throwable e) {
  final int attempt = pending.getAttemptCount();
  final ClientRetryEvent event = new ClientRetryEvent(request, e, pending);
  final TimeDuration sleepTime = client.getEffectiveSleepTime(e,
      retryPolicy.handleAttemptFailure(event).getSleepTime());
  LOG.debug("schedule* attempt #{} with sleep {} and policy {} for {}", attempt, sleepTime, retryPolicy, request);
  scheduleWithTimeout(pending, sleepTime, getSlidingWindow(request));
}
 

private RequestTypeDependentRetryPolicy(
    EnumMap<RaftProtos.RaftClientRequestProto.TypeCase, RetryPolicy> map,
    EnumMap<RaftProtos.RaftClientRequestProto.TypeCase, TimeDuration> timeoutMap) {
  this.retryPolicyMap = Collections.unmodifiableMap(map);
  this.timeoutMap = timeoutMap;
  this.myString = () -> {
    final StringBuilder b = new StringBuilder(getClass().getSimpleName()).append("{");
    map.forEach((key, value) -> b.append(key).append("->").append(value).append(", "));
    b.setLength(b.length() - 2);
    return b.append("}").toString();
  };
}
 

static RaftServerImpl waitForLeader(
    MiniRaftCluster cluster, RaftGroupId groupId, boolean expectLeader)
    throws InterruptedException {
  final String name = "waitForLeader-" + groupId + "-(expectLeader? " + expectLeader + ")";
  final int numAttempts = expectLeader? 100: 10;
  final TimeDuration sleepTime = cluster.getTimeoutMax().apply(d -> (d * 3) >> 1);
  LOG.info(cluster.printServers(groupId));

  final AtomicReference<IllegalStateException> exception = new AtomicReference<>();
  final Runnable handleNoLeaders = () -> {
    throw cluster.newIllegalStateExceptionForNoLeaders(groupId);
  };
  final Consumer<List<RaftServerImpl>> handleMultipleLeaders = leaders -> {
    final IllegalStateException ise = cluster.newIllegalStateExceptionForMultipleLeaders(groupId, leaders);
    exception.set(ise);
  };

  final RaftServerImpl leader = JavaUtils.attempt(
      () -> cluster.getLeader(groupId, handleNoLeaders, handleMultipleLeaders),
      numAttempts, sleepTime, name, LOG);

  LOG.info(cluster.printServers(groupId));
  if (expectLeader) {
    return Optional.ofNullable(leader).orElseThrow(exception::get);
  } else {
    if (leader == null) {
      return null;
    } else {
      throw new IllegalStateException("expectLeader = " + expectLeader + " but leader = " + leader);
    }
  }
}