org.apache.ratis.server.RaftServerConfigKeys Java Examples

LeaderState(RaftServerImpl server, RaftProperties properties) {
  this.server = server;

  stagingCatchupGap = RaftServerConfigKeys.stagingCatchupGap(properties);
  syncInterval = RaftServerConfigKeys.Rpc.sleepTime(properties);

  final ServerState state = server.getState();
  this.raftLog = state.getLog();
  this.currentTerm = state.getCurrentTerm();
  processor = new EventProcessor();
  this.pendingRequests = new PendingRequests(server.getId());
  this.watchRequests = new WatchRequests(server.getId(), properties);

  final RaftConfiguration conf = server.getRaftConf();
  Collection<RaftPeer> others = conf.getOtherPeers(state.getSelfId());
  final Timestamp t = Timestamp.currentTime().addTimeMs(-server.getMaxTimeoutMs());
  placeHolderIndex = raftLog.getNextIndex();

  senders = new SenderList(others.stream().map(
      p -> server.newLogAppender(this, p, t, placeHolderIndex, true))
      .toArray(LogAppender[]::new));

  voterLists = divideFollowers(conf);
}
 

@Test
public void testLeaderElectionDetection() throws Exception {
  RaftTestUtil.waitForLeader(cluster);
  final TimeDuration noLeaderTimeout = RaftServerConfigKeys.Notification.noLeaderTimeout(cluster.getProperties());

  RaftServerImpl healthyFollower = cluster.getFollowers().get(1);
  RaftServerImpl failedFollower = cluster.getFollowers().get(0);
  // fail the leader and one of the followers to that quorum is not present
  // for next leader election to succeed.
  cluster.killServer(failedFollower.getId());
  cluster.killServer(cluster.getLeader().getId());

  // Wait to ensure that leader election is triggered and also state machine callback is triggered
  noLeaderTimeout.sleep();
  noLeaderTimeout.sleep();

  RaftProtos.RoleInfoProto roleInfoProto =
      SimpleStateMachine4Testing.get(healthyFollower).getLeaderElectionTimeoutInfo();
  Assert.assertNotNull(roleInfoProto);

  Assert.assertEquals(roleInfoProto.getRole(), RaftProtos.RaftPeerRole.CANDIDATE);
  final long noLeaderTimeoutMs = noLeaderTimeout.toLong(TimeUnit.MILLISECONDS);
  Assert.assertTrue(roleInfoProto.getCandidateInfo().getLastLeaderElapsedTimeMs() > noLeaderTimeoutMs);
}
 

/**
 * note we do not apply log entries to the state machine here since we do not
 * know whether they have been committed.
 */
private RaftLog initLog(RaftPeerId id, RaftProperties prop,
    long lastIndexInSnapshot, Consumer<LogEntryProto> logConsumer)
    throws IOException {
  final RaftLog log;
  if (RaftServerConfigKeys.Log.useMemory(prop)) {
    final int maxBufferSize =
        RaftServerConfigKeys.Log.Appender.bufferByteLimit(prop).getSizeInt();
    log = new MemoryRaftLog(id, lastIndexInSnapshot, maxBufferSize);
  } else {
    log = new SegmentedRaftLog(id, server, this.storage,
        lastIndexInSnapshot, prop);
  }
  log.open(lastIndexInSnapshot, logConsumer);
  return log;
}
 

@Test
public void testWatchRequestClientTimeout() throws Exception {
  final RaftProperties p = getProperties();
  RaftServerConfigKeys.Watch.setTimeout(p, TimeDuration.valueOf(100,
      TimeUnit.SECONDS));
  RaftClientConfigKeys.Rpc.setWatchRequestTimeout(p,
      TimeDuration.valueOf(15, TimeUnit.SECONDS));
  try {
    runWithNewCluster(NUM_SERVERS,
        cluster -> runSingleTest(WatchRequestTests::runTestWatchRequestClientTimeout, cluster, LOG));
  } finally {
    RaftServerConfigKeys.Watch.setTimeout(p, RaftServerConfigKeys.Watch.TIMEOUT_DEFAULT);
    RaftClientConfigKeys.Rpc.setWatchRequestTimeout(p,
        RaftClientConfigKeys.Rpc.WATCH_REQUEST_TIMEOUT_DEFAULT);
  }
}
 

/**
 * Simulate the scenario that the peer is shutdown without truncating
 * log segment file padding. Make sure the reader can correctly handle this.
 */
@Test
public void testReadWithPadding() throws IOException {
  final RaftStorage storage = new RaftStorage(storageDir, StartupOption.REGULAR);
  File openSegment = storage.getStorageDir().getOpenLogFile(0);
  long size = SegmentedRaftLogFormat.getHeaderLength();

  LogEntryProto[] entries = new LogEntryProto[100];
  final SegmentedRaftLogOutputStream out = new SegmentedRaftLogOutputStream(openSegment, false,
      segmentMaxSize, preallocatedSize, ByteBuffer.allocateDirect(bufferSize));
  size += writeMessages(entries, out);
  out.flush();

  // make sure the file contains padding
  Assert.assertEquals(
      RaftServerConfigKeys.Log.PREALLOCATED_SIZE_DEFAULT.getSize(),
      openSegment.length());

  // check if the reader can correctly read the log file
  LogEntryProto[] readEntries = readLog(openSegment, 0,
      RaftServerConstants.INVALID_LOG_INDEX, true);
  Assert.assertArrayEquals(entries, readEntries);

  out.close();
  Assert.assertEquals(size, openSegment.length());
}
 

/**
 * Sets common Ratis server properties for both the log and metadata state machines.
 */
void setRaftProperties(RaftProperties properties) {
  // Set the ports for the server
  GrpcConfigKeys.Server.setPort(properties, opts.getPort());
  NettyConfigKeys.Server.setPort(properties, opts.getPort());

  // Ozone sets the leader election timeout (min) to 1second.
  long leaderElectionTimeoutMinVal = getLeaderElectionTimeoutMin();
  TimeDuration leaderElectionTimeoutMin = TimeDuration.valueOf(leaderElectionTimeoutMinVal,
    TimeUnit.MILLISECONDS);
  RaftServerConfigKeys.Rpc.setTimeoutMin(properties, leaderElectionTimeoutMin);
  long leaderElectionTimeoutMaxVal = getLeaderElectionTimeoutMax();

  TimeDuration leaderElectionMaxTimeout = TimeDuration.valueOf(
    leaderElectionTimeoutMaxVal,
      TimeUnit.MILLISECONDS);
  RaftServerConfigKeys.Rpc.setTimeoutMax(properties, leaderElectionMaxTimeout);
}
 

/** Check the storage dir and add groups*/
void initGroups(RaftGroup group) {

  final Optional<RaftGroup> raftGroup = Optional.ofNullable(group);
  final Optional<RaftGroupId> raftGroupId = raftGroup.map(RaftGroup::getGroupId);

  RaftServerConfigKeys.storageDir(properties).parallelStream()
      .forEach((dir) -> Optional.ofNullable(dir.listFiles())
          .map(Arrays::stream).orElse(Stream.empty())
          .filter(File::isDirectory)
          .forEach(sub -> {
            try {
              LOG.info("{}: found a subdirectory {}", getId(), sub);
              final RaftGroupId groupId = RaftGroupId.valueOf(UUID.fromString(sub.getName()));
              if (!raftGroupId.filter(groupId::equals).isPresent()) {
                addGroup(RaftGroup.valueOf(groupId));
              }
            } catch (Throwable t) {
              LOG.warn(getId() + ": Failed to initialize the group directory "
                  + sub.getAbsolutePath() + ".  Ignoring it", t);
            }
          }));
  raftGroup.ifPresent(this::addGroup);
}
 

RaftServerImpl(RaftGroup group, StateMachine stateMachine, RaftServerProxy proxy) throws IOException {
  final RaftPeerId id = proxy.getId();
  LOG.info("{}: new RaftServerImpl for {} with {}", id, group, stateMachine);
  this.groupId = group.getGroupId();
  this.lifeCycle = new LifeCycle(id);
  this.stateMachine = stateMachine;
  this.role = new RoleInfo(id);

  final RaftProperties properties = proxy.getProperties();
  minTimeoutMs = RaftServerConfigKeys.Rpc.timeoutMin(properties).toIntExact(TimeUnit.MILLISECONDS);
  maxTimeoutMs = RaftServerConfigKeys.Rpc.timeoutMax(properties).toIntExact(TimeUnit.MILLISECONDS);
  rpcSlownessTimeoutMs = RaftServerConfigKeys.Rpc.slownessTimeout(properties).toIntExact(TimeUnit.MILLISECONDS);
  Preconditions.assertTrue(maxTimeoutMs > minTimeoutMs,
      "max timeout: %s, min timeout: %s", maxTimeoutMs, minTimeoutMs);
  this.proxy = proxy;

  this.state = new ServerState(id, group, properties, this, stateMachine);
  this.retryCache = initRetryCache(properties);

  this.jmxAdapter = new RaftServerJmxAdapter();
}
 

public void dumpSegmentFile() throws IOException {
  RaftStorageDirectory.LogPathAndIndex pi = RaftStorageDirectory.processOnePath(file.toPath());
  if (pi == null) {
    System.out.println("Invalid segment file");
    return;
  }

  System.out.println("Processing Raft Log file: " + file.getAbsolutePath() + " size:" + file.length());
  final int entryCount = LogSegment.readSegmentFile(file, pi.getStartIndex(), pi.getEndIndex(), pi.isOpen(),
      RaftServerConfigKeys.Log.CorruptionPolicy.EXCEPTION, null, this::processLogEntry);
  System.out.println("Num Total Entries: " + entryCount);
  System.out.println("Num Conf Entries: " + numConfEntries);
  System.out.println("Num Metadata Entries: " + numMetadataEntries);
  System.out.println("Num StateMachineEntries Entries: " + numStateMachineEntries);
  System.out.println("Num Invalid Entries: " + numInvalidEntries);
}
 

@Test
public void testLeaderElectionDetection() throws Exception {
  RaftTestUtil.waitForLeader(cluster);
  long leaderElectionTimeout = RaftServerConfigKeys.
      leaderElectionTimeout(cluster.getProperties()).toIntExact(TimeUnit.MILLISECONDS);

  RaftServerImpl healthyFollower = cluster.getFollowers().get(1);
  RaftServerImpl failedFollower = cluster.getFollowers().get(0);
  // fail the leader and one of the followers to that quorum is not present
  // for next leader election to succeed.
  cluster.killServer(failedFollower.getId());
  cluster.killServer(cluster.getLeader().getId());

  // Wait to ensure that leader election is triggered and also state machine callback is triggered
  Thread.sleep( leaderElectionTimeout * 2);

  RaftProtos.RoleInfoProto roleInfoProto =
      SimpleStateMachine4Testing.get(healthyFollower).getLeaderElectionTimeoutInfo();
  Assert.assertNotNull(roleInfoProto);

  Assert.assertEquals(roleInfoProto.getRole(), RaftProtos.RaftPeerRole.CANDIDATE);
  Assert.assertTrue(roleInfoProto.getCandidateInfo().getLastLeaderElapsedTimeMs() > leaderElectionTimeout);
}
 

@Override
public void run() throws Exception {
  RaftPeerId peerId = RaftPeerId.valueOf(id);
  RaftProperties properties = new RaftProperties();

  RaftPeer[] peers = getPeers();
  final int port = NetUtils.createSocketAddr(getPeer(peerId).getAddress()).getPort();
  GrpcConfigKeys.Server.setPort(properties, port);
  properties.setInt(GrpcConfigKeys.OutputStream.RETRY_TIMES_KEY, Integer.MAX_VALUE);
  RaftServerConfigKeys.setStorageDirs(properties, Collections.singletonList(storageDir));
  ConfUtils.setFile(properties::setFile, FileStoreCommon.STATEMACHINE_DIR_KEY,
      storageDir);
  StateMachine stateMachine = new FileStoreStateMachine(properties);

  final RaftGroup raftGroup = RaftGroup.valueOf(RaftGroupId.valueOf(ByteString.copyFromUtf8(raftGroupId)), peers);
  RaftServer raftServer = RaftServer.newBuilder()
      .setServerId(RaftPeerId.valueOf(id))
      .setStateMachine(stateMachine).setProperties(properties)
      .setGroup(raftGroup)
      .build();
  raftServer.start();

  for(; raftServer.getLifeCycleState() != LifeCycle.State.CLOSED;) {
    TimeUnit.SECONDS.sleep(1);
  }
}
 

private void setRatisLeaderElectionTimeout(RaftProperties properties) {
  long duration;
  TimeUnit leaderElectionMinTimeoutUnit =
      OzoneConfigKeys.
          DFS_RATIS_LEADER_ELECTION_MINIMUM_TIMEOUT_DURATION_DEFAULT
          .getUnit();
  duration = conf.getTimeDuration(
      OzoneConfigKeys.DFS_RATIS_LEADER_ELECTION_MINIMUM_TIMEOUT_DURATION_KEY,
      OzoneConfigKeys.
          DFS_RATIS_LEADER_ELECTION_MINIMUM_TIMEOUT_DURATION_DEFAULT
          .getDuration(), leaderElectionMinTimeoutUnit);
  final TimeDuration leaderElectionMinTimeout =
      TimeDuration.valueOf(duration, leaderElectionMinTimeoutUnit);
  RaftServerConfigKeys.Rpc
      .setTimeoutMin(properties, leaderElectionMinTimeout);
  long leaderElectionMaxTimeout =
      leaderElectionMinTimeout.toLong(TimeUnit.MILLISECONDS) + 200;
  RaftServerConfigKeys.Rpc.setTimeoutMax(properties,
      TimeDuration.valueOf(leaderElectionMaxTimeout, TimeUnit.MILLISECONDS));
}
 

/**
 * Try to enforce the leader of the cluster.
 * @param leaderId ID of the targeted leader server.
 * @return true if server has been successfully enforced to the leader, false
 *         otherwise.
 */
public boolean tryEnforceLeader(String leaderId) throws InterruptedException {
  // do nothing and see if the given id is already a leader.
  if (isLeader(leaderId)) {
    return true;
  }

  // Blocking all other server's RPC read process to make sure a read takes at
  // least ELECTION_TIMEOUT_MIN. In this way when the target leader request a
  // vote, all non-leader servers can grant the vote.
  // Disable the target leader server RPC so that it can request a vote.
  blockQueueAndSetDelay(leaderId,
      RaftServerConfigKeys.Rpc.TIMEOUT_MIN_DEFAULT.toIntExact(TimeUnit.MILLISECONDS));

  // Reopen queues so that the vote can make progress.
  blockQueueAndSetDelay(leaderId, 0);

  return isLeader(leaderId);
}
 

@Override
public void run() throws Exception {
  RaftPeerId peerId = RaftPeerId.valueOf(id);
  RaftProperties properties = new RaftProperties();

  RaftPeer[] peers = getPeers();
  final int port = NetUtils.createSocketAddr(getPeer(peerId).getAddress()).getPort();
  GrpcConfigKeys.Server.setPort(properties, port);
  properties.setInt(GrpcConfigKeys.OutputStream.RETRY_TIMES_KEY, Integer.MAX_VALUE);
  RaftServerConfigKeys.setStorageDirs(properties, Collections.singletonList(storageDir));
  StateMachine stateMachine = new ArithmeticStateMachine();

  final RaftGroup raftGroup = RaftGroup.valueOf(RaftGroupId.valueOf(ByteString.copyFromUtf8(raftGroupId)), peers);
  RaftServer raftServer = RaftServer.newBuilder()
      .setServerId(RaftPeerId.valueOf(id))
      .setStateMachine(stateMachine).setProperties(properties)
      .setGroup(raftGroup)
      .build();
  raftServer.start();

  for(; raftServer.getLifeCycleState() != LifeCycle.State.CLOSED;) {
    TimeUnit.SECONDS.sleep(1);
  }
}
 

/**
 * Try to enforce the leader of the cluster.
 * @param leaderId ID of the targeted leader server.
 * @return true if server has been successfully enforced to the leader, false
 *         otherwise.
 */
public boolean tryEnforceLeader(String leaderId) throws InterruptedException {
  // do nothing and see if the given id is already a leader.
  if (isLeader(leaderId)) {
    return true;
  }

  // Blocking all other server's RPC read process to make sure a read takes at
  // least ELECTION_TIMEOUT_MIN. In this way when the target leader request a
  // vote, all non-leader servers can grant the vote.
  // Disable the target leader server RPC so that it can request a vote.
  blockQueueAndSetDelay(leaderId,
      RaftServerConfigKeys.Rpc.TIMEOUT_MIN_DEFAULT.toIntExact(TimeUnit.MILLISECONDS));

  // Reopen queues so that the vote can make progress.
  blockQueueAndSetDelay(leaderId, 0);

  return isLeader(leaderId);
}
 

private GrpcService(RaftServer server, GrpcTlsConfig tlsConfig) {
  this(server, server::getId,
      GrpcConfigKeys.Server.port(server.getProperties()),
      GrpcConfigKeys.messageSizeMax(server.getProperties(), LOG::info),
      RaftServerConfigKeys.Log.Appender.bufferByteLimit(server.getProperties()),
      GrpcConfigKeys.flowControlWindow(server.getProperties(), LOG::info),
      RaftServerConfigKeys.Rpc.requestTimeout(server.getProperties()),
      tlsConfig);
}
 

private InstallSnapshotReplyProto installSnapshotImpl(InstallSnapshotRequestProto request) throws IOException {
  final RaftRpcRequestProto r = request.getServerRequest();
  final RaftPeerId leaderId = RaftPeerId.valueOf(r.getRequestorId());
  final RaftGroupId leaderGroupId = ProtoUtils.toRaftGroupId(r.getRaftGroupId());
  CodeInjectionForTesting.execute(INSTALL_SNAPSHOT, getId(),
      leaderId, request);

  assertLifeCycleState(LifeCycle.States.STARTING_OR_RUNNING);
  assertGroup(leaderId, leaderGroupId);

  // Check if install snapshot from Leader is enabled
  if (installSnapshotEnabled) {
    // Leader has sent InstallSnapshot request with SnapshotInfo. Install the snapshot.
    if (request.hasSnapshotChunk()) {
      return checkAndInstallSnapshot(request, leaderId);
    }
  } else {
    // Leader has only sent a notification to install snapshot. Inform State Machine to install snapshot.
    if (request.hasNotification()) {
      return notifyStateMachineToInstallSnapshot(request, leaderId);
    }
  }
  // There is a mismatch between configurations on leader and follower.
  final InstallSnapshotReplyProto reply = ServerProtoUtils.toInstallSnapshotReplyProto(
      leaderId, getMemberId(), InstallSnapshotResult.CONF_MISMATCH);
  LOG.error("{}: Configuration Mismatch ({}): Leader {} has it set to {} but follower {} has it set to {}",
      getMemberId(), RaftServerConfigKeys.Log.Appender.INSTALL_SNAPSHOT_ENABLED_KEY,
      leaderId, request.hasSnapshotChunk(), getId(), installSnapshotEnabled);
  return reply;
}
 

public RaftProperties getProperties() {
  RaftServerConfigKeys
      .setLeaderElectionTimeout(properties, TimeDuration.valueOf(1, TimeUnit.SECONDS));
  properties.setClass(MiniRaftCluster.STATEMACHINE_CLASS_KEY,
      SimpleStateMachine4Testing.class, StateMachine.class);
  return properties;
}
 

@Before
public void setup() throws Exception {
  RaftProperties properties = new RaftProperties();
  storageDir = getTestDir();
  RaftServerConfigKeys.setStorageDirs(properties,  Collections.singletonList(storageDir));
  this.segmentMaxSize =
      RaftServerConfigKeys.Log.segmentSizeMax(properties).getSize();
  this.preallocatedSize =
      RaftServerConfigKeys.Log.preallocatedSize(properties).getSize();
  this.bufferSize =
      RaftServerConfigKeys.Log.writeBufferSize(properties).getSizeInt();
}
 

@Before
public void setup() throws Exception {
  storageDir = getTestDir();
  RaftProperties properties = new RaftProperties();
  RaftServerConfigKeys.setStorageDirs(properties,  Collections.singletonList(storageDir));
  this.segmentMaxSize =
      RaftServerConfigKeys.Log.segmentSizeMax(properties).getSize();
  this.preallocatedSize =
      RaftServerConfigKeys.Log.preallocatedSize(properties).getSize();
  this.bufferSize =
      RaftServerConfigKeys.Log.writeBufferSize(properties).getSizeInt();
}
 

WatchRequests(Object name, RaftProperties properties) {
  this.name = name + "-" + getClass().getSimpleName();

  final TimeDuration watchTimeout = RaftServerConfigKeys.watchTimeout(properties);
  this.watchTimeoutNanos = watchTimeout.to(TimeUnit.NANOSECONDS);
  final TimeDuration watchTimeoutDenomination = RaftServerConfigKeys.watchTimeoutDenomination(properties);
  this.watchTimeoutDenominationNanos = watchTimeoutDenomination.to(TimeUnit.NANOSECONDS);
  Preconditions.assertTrue(watchTimeoutNanos.getDuration() % watchTimeoutDenominationNanos.getDuration() == 0L,
      () -> "watchTimeout (=" + watchTimeout + ") is not a multiple of watchTimeoutDenomination (="
          + watchTimeoutDenomination + ").");

  Arrays.stream(ReplicationLevel.values()).forEach(r -> queues.put(r, new WatchQueue(r)));
}
 

WatchRequests(Object name, RaftProperties properties) {
  this.name = name + "-" + getClass().getSimpleName();

  final TimeDuration watchTimeout = RaftServerConfigKeys.Watch.timeout(properties);
  this.watchTimeoutNanos = watchTimeout.to(TimeUnit.NANOSECONDS);
  final TimeDuration watchTimeoutDenomination = RaftServerConfigKeys.Watch.timeoutDenomination(properties);
  this.watchTimeoutDenominationNanos = watchTimeoutDenomination.to(TimeUnit.NANOSECONDS);
  Preconditions.assertTrue(watchTimeoutNanos.getDuration() % watchTimeoutDenominationNanos.getDuration() == 0L,
      () -> "watchTimeout (=" + watchTimeout + ") is not a multiple of watchTimeoutDenomination (="
          + watchTimeoutDenomination + ").");

  final int elementLimit = RaftServerConfigKeys.Watch.elementLimit(properties);
  Arrays.stream(ReplicationLevel.values()).forEach(r -> queues.put(r, new WatchQueue(r, elementLimit)));
}
 

@Override
public void run() throws Exception {
  int raftSegmentPreallocatedSize = 1024 * 1024 * 1024;
  RaftProperties raftProperties = new RaftProperties();
  RaftConfigKeys.Rpc.setType(raftProperties, SupportedRpcType.GRPC);
  GrpcConfigKeys.setMessageSizeMax(raftProperties,
      SizeInBytes.valueOf(raftSegmentPreallocatedSize));
  RaftServerConfigKeys.Log.Appender.setBufferByteLimit(raftProperties,
      SizeInBytes.valueOf(raftSegmentPreallocatedSize));
  RaftServerConfigKeys.Log.setWriteBufferSize(raftProperties,
      SizeInBytes.valueOf(raftSegmentPreallocatedSize));
  RaftServerConfigKeys.Log.setPreallocatedSize(raftProperties,
      SizeInBytes.valueOf(raftSegmentPreallocatedSize));
  RaftServerConfigKeys.Log.setSegmentSizeMax(raftProperties,
      SizeInBytes.valueOf(1 * 1024 * 1024 * 1024));

  RaftServerConfigKeys.Log.setMaxCachedSegmentNum(raftProperties, 2);

  RaftClientConfigKeys.Rpc.setRequestTimeout(raftProperties,
      TimeDuration.valueOf(50000, TimeUnit.MILLISECONDS));
  RaftClientConfigKeys.Async.setSchedulerThreads(raftProperties, 10);
  RaftClientConfigKeys.Async.setMaxOutstandingRequests(raftProperties, 1000);


  final RaftGroup raftGroup = RaftGroup.valueOf(RaftGroupId.valueOf(ByteString.copyFromUtf8(raftGroupId)),
      parsePeers(peers));

  RaftClient.Builder builder =
      RaftClient.newBuilder().setProperties(raftProperties);
  builder.setRaftGroup(raftGroup);
  builder.setClientRpc(new GrpcFactory(new Parameters()).newRaftClientRpc(ClientId.randomId(), raftProperties));
  RaftClient client = builder.build();

  operation(client);
}
 

private GrpcService(RaftServer raftServer, Supplier<RaftPeerId> idSupplier, int port,
    SizeInBytes grpcMessageSizeMax, SizeInBytes appenderBufferSize,
    SizeInBytes flowControlWindow,TimeDuration requestTimeoutDuration, GrpcTlsConfig tlsConfig) {
  super(idSupplier, id -> new PeerProxyMap<>(id.toString(),
      p -> new GrpcServerProtocolClient(p, flowControlWindow.getSizeInt(),
          requestTimeoutDuration, tlsConfig)));
  if (appenderBufferSize.getSize() > grpcMessageSizeMax.getSize()) {
    throw new IllegalArgumentException("Illegal configuration: "
        + RaftServerConfigKeys.Log.Appender.BUFFER_BYTE_LIMIT_KEY + " = " + appenderBufferSize
        + " > " + GrpcConfigKeys.MESSAGE_SIZE_MAX_KEY + " = " + grpcMessageSizeMax);
  }

  NettyServerBuilder nettyServerBuilder = NettyServerBuilder.forPort(port)
      .maxInboundMessageSize(grpcMessageSizeMax.getSizeInt())
      .flowControlWindow(flowControlWindow.getSizeInt())
      .addService(new GrpcServerProtocolService(idSupplier, raftServer))
      .addService(new GrpcClientProtocolService(idSupplier, raftServer))
      .addService(new GrpcAdminProtocolService(raftServer));

  if (tlsConfig != null) {
    SslContextBuilder sslContextBuilder =
        SslContextBuilder.forServer(tlsConfig.getCertChain(),
            tlsConfig.getPrivateKey());
    if (tlsConfig.getMtlsEnabled()) {
      sslContextBuilder.clientAuth(ClientAuth.REQUIRE);
      sslContextBuilder.trustManager(tlsConfig.getCertChain());
    }
    sslContextBuilder = GrpcSslContexts.configure(sslContextBuilder, OPENSSL);
    try {
      nettyServerBuilder.sslContext(sslContextBuilder.build());
    } catch (Exception ex) {
      throw new IllegalArgumentException("Failed to build SslContext, tlsConfig=" + tlsConfig, ex);
    }
  }
  server = nettyServerBuilder.build();
  addressSupplier = JavaUtils.memoize(() -> new InetSocketAddress(port != 0? port: server.getPort()));
}
 

LeaderState(RaftServerImpl server, RaftProperties properties) {
  this.name = server.getMemberId() + "-" + getClass().getSimpleName();
  this.server = server;

  stagingCatchupGap = RaftServerConfigKeys.stagingCatchupGap(properties);
  syncInterval = RaftServerConfigKeys.Rpc.sleepTime(properties);

  final ServerState state = server.getState();
  this.raftLog = state.getLog();
  this.currentTerm = state.getCurrentTerm();

  this.eventQueue = new EventQueue();
  processor = new EventProcessor();
  raftServerMetrics = server.getRaftServerMetrics();
  logAppenderMetrics = new LogAppenderMetrics(server.getMemberId());
  this.pendingRequests = new PendingRequests(server.getMemberId(), properties, raftServerMetrics);
  this.watchRequests = new WatchRequests(server.getMemberId(), properties);
  this.streamRequests = new StreamRequests(server.getMemberId());

  final RaftConfiguration conf = server.getRaftConf();
  Collection<RaftPeer> others = conf.getOtherPeers(server.getId());
  placeHolderIndex = raftLog.getNextIndex();

  senders = new SenderList();
  addSenders(others, placeHolderIndex, true);
  voterLists = divideFollowers(conf);
}
 

@Test
public void testRequestTimeout() throws Exception {
  final TimeDuration oldExpiryTime = RaftServerConfigKeys.RetryCache.expiryTime(getProperties());
  RaftServerConfigKeys.RetryCache.setExpiryTime(getProperties(), TimeDuration.valueOf(5, TimeUnit.SECONDS));
  runWithNewCluster(NUM_SERVERS, cluster -> RaftBasicTests.testRequestTimeout(true, cluster, LOG));

  //reset for the other tests
  RaftServerConfigKeys.RetryCache.setExpiryTime(getProperties(), oldExpiryTime);
}
 

private static RaftLog initRaftLog(RaftGroupMemberId memberId, RaftServerImpl server, RaftStorage storage,
    Consumer<LogEntryProto> logConsumer, long lastIndexInSnapshot, RaftProperties prop) throws IOException {
  final RaftLog log;
  if (RaftServerConfigKeys.Log.useMemory(prop)) {
    log = new MemoryRaftLog(memberId, lastIndexInSnapshot, prop);
  } else {
    log = new SegmentedRaftLog(memberId, server, storage, lastIndexInSnapshot, prop);
  }
  log.open(lastIndexInSnapshot, logConsumer);
  return log;
}
 

ServerState(RaftPeerId id, RaftGroup group, RaftProperties prop,
            RaftServerImpl server, StateMachine stateMachine)
    throws IOException {
  this.memberId = RaftGroupMemberId.valueOf(id, group.getGroupId());
  this.server = server;
  RaftConfiguration initialConf = RaftConfiguration.newBuilder()
      .setConf(group.getPeers()).build();
  configurationManager = new ConfigurationManager(initialConf);
  LOG.info("{}: {}", getMemberId(), configurationManager);

  // use full uuid string to create a subdirectory
  final File dir = chooseStorageDir(RaftServerConfigKeys.storageDir(prop),
      group.getGroupId().getUuid().toString());
  storage = new RaftStorage(dir, RaftServerConstants.StartupOption.REGULAR,
      RaftServerConfigKeys.Log.corruptionPolicy(prop));
  snapshotManager = new SnapshotManager(storage, id);

  long lastApplied = initStatemachine(stateMachine, group.getGroupId());

  // On start the leader is null, start the clock now
  leaderId = null;
  this.lastNoLeaderTime = Timestamp.currentTime();
  this.noLeaderTimeout = RaftServerConfigKeys.Notification.noLeaderTimeout(prop);

  // we cannot apply log entries to the state machine in this step, since we
  // do not know whether the local log entries have been committed.
  this.log = initRaftLog(getMemberId(), server, storage, this::setRaftConf, lastApplied, prop);

  RaftLog.Metadata metadata = log.loadMetadata();
  currentTerm.set(metadata.getTerm());
  votedFor = metadata.getVotedFor();

  stateMachineUpdater = new StateMachineUpdater(stateMachine, server, this, lastApplied, prop);
}
 

void runTestAppendEntriesTimeout(CLUSTER cluster) throws Exception {
  LOG.info("Running testAppendEntriesTimeout");
  final TimeDuration oldExpiryTime = RaftServerConfigKeys.RetryCache.expiryTime(getProperties());
  RaftServerConfigKeys.RetryCache.setExpiryTime(getProperties(), TimeDuration.valueOf(20, TimeUnit.SECONDS));
  waitForLeader(cluster);
  long time = System.currentTimeMillis();
  long waitTime = 5000;
  try (final RaftClient client = cluster.createClient()) {
    // block append requests
    cluster.getServerAliveStream()
        .filter(impl -> !impl.isLeader())
        .map(SimpleStateMachine4Testing::get)
        .forEach(SimpleStateMachine4Testing::blockWriteStateMachineData);

    CompletableFuture<RaftClientReply> replyFuture = client.sendAsync(new SimpleMessage("abc"));
    Thread.sleep(waitTime);
    // replyFuture should not be completed until append request is unblocked.
    Assert.assertTrue(!replyFuture.isDone());
    // unblock append request.
    cluster.getServerAliveStream()
        .filter(impl -> !impl.isLeader())
        .map(SimpleStateMachine4Testing::get)
        .forEach(SimpleStateMachine4Testing::unblockWriteStateMachineData);

    replyFuture.get();
    Assert.assertTrue(System.currentTimeMillis() - time > waitTime);
  }

  //reset for the other tests
  RaftServerConfigKeys.RetryCache.setExpiryTime(getProperties(), oldExpiryTime);
}
 

SegmentedRaftLogCache(Object name, RaftStorage storage, RaftProperties properties,
                              RaftLogMetrics raftLogMetrics) {
  this.name = name + "-" + getClass().getSimpleName();
  this.closedSegments = new LogSegmentList(name);
  this.storage = storage;
  this.raftLogMetrics = raftLogMetrics;
  this.raftLogMetrics.addClosedSegmentsNum(this);
  this.raftLogMetrics.addClosedSegmentsSizeInBytes(this);
  this.raftLogMetrics.addOpenSegmentSizeInBytes(this);
  this.maxCachedSegments = RaftServerConfigKeys.Log.segmentCacheNumMax(properties);
}