Java Code Examples for org.apache.ratis.util.JavaUtils#attemptRepeatedly()

static void assertFlushCount(RaftServerImpl server) throws Exception {
  final String flushTimeMetric = RaftStorageTestUtils.getLogFlushTimeMetric(server.getMemberId().toString());
  RatisMetricRegistry ratisMetricRegistry = new RaftLogMetrics(server.getMemberId().toString()).getRegistry();
  Timer tm = (Timer) ratisMetricRegistry.get(RAFT_LOG_FLUSH_TIME);
  Assert.assertNotNull(tm);

  final MetricsStateMachine stateMachine = MetricsStateMachine.get(server);
  final int expectedFlush = stateMachine.getFlushCount();

  JavaUtils.attemptRepeatedly(() -> {
    Assert.assertEquals(expectedFlush, tm.getCount());
    return null;
  }, 50, HUNDRED_MILLIS, "expectedFlush == tm.getCount()", null);

  Assert.assertTrue(tm.getMeanRate() > 0);

  // Test jmx
  ObjectName oname = new ObjectName(RATIS_APPLICATION_NAME_METRICS, "name", flushTimeMetric);
  Assert.assertEquals(expectedFlush,
      ((Long) ManagementFactory.getPlatformMBeanServer().getAttribute(oname, "Count"))
          .intValue());
}
 

public static RaftServerProxy newRaftServer(
    RaftPeerId id, StateMachine.Registry stateMachineRegistry, RaftProperties properties, Parameters parameters)
    throws IOException {
  final TimeDuration sleepTime = TimeDuration.valueOf(500, TimeUnit.MILLISECONDS);
  final RaftServerProxy proxy;
  try {
    // attempt multiple times to avoid temporary bind exception
    proxy = JavaUtils.attemptRepeatedly(
        () -> new RaftServerProxy(id, stateMachineRegistry, properties, parameters),
        5, sleepTime, "new RaftServerProxy", RaftServerProxy.LOG);
  } catch (InterruptedException e) {
    throw IOUtils.toInterruptedIOException(
        "Interrupted when creating RaftServer " + id, e);
  }
  return proxy;
}
 

public static void assertLeaderContent(MiniRaftCluster cluster) throws Exception {
  final RaftServerImpl leader = RaftTestUtil.waitForLeader(cluster);
  final RaftLog leaderLog = leader.getState().getLog();
  final long lastIndex = leaderLog.getLastEntryTermIndex().getIndex();
  final LogEntryProto e = leaderLog.get(lastIndex);
  Assert.assertTrue(e.hasMetadataEntry());

  JavaUtils.attemptRepeatedly(() -> {
    Assert.assertEquals(leaderLog.getLastCommittedIndex() - 1, e.getMetadataEntry().getCommitIndex());
    return null;
  }, 50, BaseTest.HUNDRED_MILLIS, "CheckMetadataEntry", LOG);

  SimpleStateMachine4Testing simpleStateMachine = SimpleStateMachine4Testing.get(leader);
  Assert.assertTrue("Is not notified as a leader", simpleStateMachine.isNotifiedAsLeader());
  final LogEntryProto[] entries = simpleStateMachine.getContent();
  long message = 0;
  for (int i = 0; i < entries.length; i++) {
    LOG.info("{}) {} {}", i, message, entries[i]);
    if (entries[i].hasStateMachineLogEntry()) {
      final SimpleMessage m = new SimpleMessage("m" + message++);
      Assert.assertArrayEquals(m.getContent().toByteArray(),
          entries[i].getStateMachineLogEntry().getLogData().toByteArray());
    }
  }
}
 

static RaftPeerId changeLeader(MiniRaftCluster cluster, RaftPeerId oldLeader, Function<String, Exception> constructor)
    throws Exception {
  final String name = JavaUtils.getCallerStackTraceElement().getMethodName() + "-changeLeader";
  cluster.setBlockRequestsFrom(oldLeader.toString(), true);
  try {
    return JavaUtils.attemptRepeatedly(() -> {
      final RaftPeerId newLeader = waitForLeader(cluster).getId();
      if (newLeader.equals(oldLeader)) {
        throw constructor.apply("Failed to change leader: newLeader == oldLeader == " + oldLeader);
      }
      LOG.info("Changed leader from " + oldLeader + " to " + newLeader);
      return newLeader;
    }, 20, BaseTest.HUNDRED_MILLIS, name, LOG);
  } finally {
    cluster.setBlockRequestsFrom(oldLeader.toString(), false);
  }
}
 

private void runTestRestartWithCorruptedLogEntry(CLUSTER cluster) throws Exception {
  // this is the only server
  final RaftServerImpl leader = RaftTestUtil.waitForLeader(cluster);
  final RaftPeerId id = leader.getId();

  // send a few messages
  final SimpleMessage[] messages = SimpleMessage.create(10);
  final SimpleMessage lastMessage = messages[messages.length - 1];
  try (final RaftClient client = cluster.createClient()) {
    for (SimpleMessage m : messages) {
      Assert.assertTrue(client.send(m).isSuccess());
    }

    // assert that the last message exists
    Assert.assertTrue(client.sendReadOnly(lastMessage).isSuccess());
  }

  final RaftLog log = leader.getState().getLog();
  final long size = TestSegmentedRaftLog.getOpenSegmentSize(log);
  leader.getProxy().close();

  // corrupt the log
  final File openLogFile = JavaUtils.attemptRepeatedly(() -> getOpenLogFile(leader),
      10, HUNDRED_MILLIS, id + "-getOpenLogFile", LOG);
  try(final RandomAccessFile raf = new RandomAccessFile(openLogFile, "rw")) {
    final long mid = size / 2;
    raf.seek(mid);
    for (long i = mid; i < size; i++) {
      raf.write(0);
    }
  }

  // after the log is corrupted and the server is restarted, the last entry should no longer exist.
  cluster.restartServer(id, false);
  testFailureCase("last-entry-not-found", () -> {
    try (final RaftClient client = cluster.createClient()) {
      client.sendReadOnly(lastMessage);
    }
  }, StateMachineException.class, IndexOutOfBoundsException.class);
}
 

/**
 * Keep generating writing traffic and make sure snapshots are taken.
 * We then restart the whole raft peer and check if it can correctly load
 * snapshots + raft log.
 */
@Test
public void testRestartPeer() throws Exception {
  RaftTestUtil.waitForLeader(cluster);
  final RaftPeerId leaderId = cluster.getLeader().getId();
  int i = 0;
  try(final RaftClient client = cluster.createClient(leaderId)) {
    for (; i < SNAPSHOT_TRIGGER_THRESHOLD * 2 - 1; i++) {
      RaftClientReply reply = client.send(new SimpleMessage("m" + i));
      Assert.assertTrue(reply.isSuccess());
    }
  }

  final long nextIndex = cluster.getLeader().getState().getLog().getNextIndex();
  LOG.info("nextIndex = {}", nextIndex);
  // wait for the snapshot to be done
  final List<File> snapshotFiles = getSnapshotFiles(cluster, nextIndex - SNAPSHOT_TRIGGER_THRESHOLD, nextIndex);
  JavaUtils.attemptRepeatedly(() -> {
    Assert.assertTrue(snapshotFiles.stream().anyMatch(RaftSnapshotBaseTest::exists));
    return null;
  }, 10, ONE_SECOND, "snapshotFile.exist", LOG);

  // restart the peer and check if it can correctly load snapshot
  cluster.restart(false);
  try {
    // 200 messages + two leader elections --> last committed = 201
    assertLeaderContent(cluster);
  } finally {
    cluster.shutdown();
  }
}
 

@Test
public void testLateServerStart() throws Exception {
  final int numServer = 3;
  LOG.info("Running testLateServerStart");
  final MiniRaftCluster cluster = newCluster(numServer);
  cluster.initServers();

  // start all except one servers
  final Iterator<RaftServerProxy> i = cluster.getServers().iterator();
  for(int j = 1; j < numServer; j++) {
    i.next().start();
  }

  final RaftServerImpl leader = waitForLeader(cluster);
  final TimeDuration sleepTime = TimeDuration.valueOf(3, TimeUnit.SECONDS);
  LOG.info("sleep " + sleepTime);
  sleepTime.sleep();

  // start the last server
  final RaftServerProxy lastServer = i.next();
  lastServer.start();
  final RaftPeerId lastServerLeaderId = JavaUtils.attemptRepeatedly(
      () -> Optional.ofNullable(lastServer.getImpls().iterator().next().getState().getLeaderId())
          .orElseThrow(() -> new IllegalStateException("No leader yet")),
      10, ONE_SECOND, "getLeaderId", LOG);
  LOG.info(cluster.printServers());
  Assert.assertEquals(leader.getId(), lastServerLeaderId);
}
 

private void runTestRetryOnStateMachineException(CLUSTER cluster) throws Exception {
  RaftPeerId leaderId = RaftTestUtil.waitForLeader(cluster).getId();

  cluster.getLeaderAndSendFirstMessage(true);
  long oldLastApplied = cluster.getLeader().getState().getLastAppliedIndex();

  try (final RaftClient client = cluster.createClient(leaderId)) {
    final RaftClientRpc rpc = client.getClientRpc();
    final long callId = 999;
    final SimpleMessage message = new SimpleMessage("message");
    final RaftClientRequest r = cluster.newRaftClientRequest(client.getId(), leaderId, callId, message);
    RaftClientReply reply = rpc.sendRequest(r);
    Assert.assertFalse(reply.isSuccess());
    Assert.assertNotNull(reply.getStateMachineException());

    // retry with the same callId
    for (int i = 0; i < 5; i++) {
      reply = rpc.sendRequest(r);
      Assert.assertEquals(client.getId(), reply.getClientId());
      Assert.assertEquals(callId, reply.getCallId());
      Assert.assertFalse(reply.isSuccess());
      Assert.assertNotNull(reply.getStateMachineException());
    }

    for (RaftServerImpl server : cluster.iterateServerImpls()) {
      LOG.info("check server " + server.getId());

      JavaUtils.attemptRepeatedly(() -> {
        Assert.assertNotNull(
            RaftServerTestUtil.getRetryEntry(server, client.getId(), callId));
        return null;
      }, 5, BaseTest.ONE_SECOND, "GetRetryEntry", LOG);

      final RaftLog log = server.getState().getLog();
      RaftTestUtil.logEntriesContains(log, oldLastApplied + 1, log.getNextIndex(), message);
    }

    cluster.shutdown();
  }
}
 

static RaftServerImpl waitForLeader(
    MiniRaftCluster cluster, RaftGroupId groupId, boolean expectLeader)
    throws InterruptedException {
  final String name = "waitForLeader-" + groupId + "-(expectLeader? " + expectLeader + ")";
  final int numAttempts = expectLeader? 100: 10;
  final TimeDuration sleepTime = cluster.getTimeoutMax().apply(d -> (d * 3) >> 1);
  LOG.info(cluster.printServers(groupId));

  final AtomicReference<IllegalStateException> exception = new AtomicReference<>();
  final Runnable handleNoLeaders = () -> {
    throw cluster.newIllegalStateExceptionForNoLeaders(groupId);
  };
  final Consumer<List<RaftServerImpl>> handleMultipleLeaders = leaders -> {
    final IllegalStateException ise = cluster.newIllegalStateExceptionForMultipleLeaders(groupId, leaders);
    exception.set(ise);
  };

  final RaftServerImpl leader = JavaUtils.attemptRepeatedly(() -> {
    RaftServerImpl l = cluster.getLeader(groupId, handleNoLeaders, handleMultipleLeaders);
    if (l != null && !l.isLeaderReady()) {
      throw new IllegalStateException("Leader: "+ l.getMemberId() +  " not ready");
    }
    return l;
  }, numAttempts, sleepTime, name, LOG);

  LOG.info(cluster.printServers(groupId));
  if (expectLeader) {
    return Optional.ofNullable(leader).orElseThrow(exception::get);
  } else {
    if (leader == null) {
      return null;
    } else {
      throw new IllegalStateException("expectLeader = " + expectLeader + " but leader = " + leader);
    }
  }
}
 

void runTestHandleNotLeaderException(CLUSTER cluster) throws Exception {
  final RaftPeerId oldLeader = RaftTestUtil.waitForLeader(cluster).getId();
  try(final RaftClient client = cluster.createClient(oldLeader)) {
    sendMessage("m1", client);

    // enforce leader change
    final RaftPeerId newLeader = RaftTestUtil.changeLeader(cluster, oldLeader);

    final RaftClientRpc rpc = client.getClientRpc();
    JavaUtils.attemptRepeatedly(() -> assertNotLeaderException(newLeader, "m2", oldLeader, rpc, cluster),
        10, ONE_SECOND, "assertNotLeaderException", LOG);

    sendMessage("m3", client);
  }
}
 

void runTestRestartFollower(MiniRaftCluster cluster) throws Exception {
  RaftTestUtil.waitForLeader(cluster);
  final RaftPeerId leaderId = cluster.getLeader().getId();

  // write some messages
  final AtomicInteger messageCount = new AtomicInteger();
  final Supplier<Message> newMessage = () -> new SimpleMessage("m" + messageCount.getAndIncrement());
  writeSomething(newMessage, cluster);

  // restart a follower
  RaftPeerId followerId = cluster.getFollowers().get(0).getId();
  LOG.info("Restart follower {}", followerId);
  cluster.restartServer(followerId, false);

  // write some more messages
  writeSomething(newMessage, cluster);
  final int truncatedMessageIndex = messageCount.get() - 1;

  final long leaderLastIndex = cluster.getLeader().getState().getLog().getLastEntryTermIndex().getIndex();
  // make sure the restarted follower can catchup
  final ServerState followerState = cluster.getRaftServerImpl(followerId).getState();
  JavaUtils.attemptRepeatedly(() -> {
    Assert.assertTrue(followerState.getLastAppliedIndex() >= leaderLastIndex);
    return null;
  }, 10, ONE_SECOND, "follower catchup", LOG);

  // make sure the restarted peer's log segments is correct
  final RaftServerImpl follower = cluster.restartServer(followerId, false);
  final RaftLog followerLog = follower.getState().getLog();
  final long followerLastIndex = followerLog.getLastEntryTermIndex().getIndex();
  Assert.assertTrue(followerLastIndex >= leaderLastIndex);

  final File followerOpenLogFile = getOpenLogFile(follower);
  final File leaderOpenLogFile = getOpenLogFile(cluster.getRaftServerImpl(leaderId));

  // shutdown all servers
  cluster.getServers().forEach(RaftServerProxy::close);

  // truncate log and
  assertTruncatedLog(followerId, followerOpenLogFile, followerLastIndex, cluster);
  assertTruncatedLog(leaderId, leaderOpenLogFile, leaderLastIndex, cluster);

  // restart and write something.
  cluster.restart(false);
  writeSomething(newMessage, cluster);

  // restart again and check messages.
  cluster.restart(false);
  try(final RaftClient client = cluster.createClient()) {
    for(int i = 0; i < messageCount.get(); i++) {
      if (i != truncatedMessageIndex) {
        final Message m = new SimpleMessage("m" + i);
        final RaftClientReply reply = client.sendReadOnly(m);
        Assert.assertTrue(reply.isSuccess());
        LOG.info("query {}: {} {}", m, reply, LogEntryProto.parseFrom(reply.getMessage().getContent()));
      }
    }
  }
}
 

void runTestRestartCommitIndex(MiniRaftCluster cluster) throws Exception {
  final SimpleMessage[] messages = SimpleMessage.create(100);
  final List<CompletableFuture<Void>> futures = new ArrayList<>(messages.length);
  for(int i = 0; i < messages.length; i++) {
    final CompletableFuture<Void> f = new CompletableFuture<>();
    futures.add(f);

    final SimpleMessage m = messages[i];
    new Thread(() -> {
      try (final RaftClient client = cluster.createClient()) {
        Assert.assertTrue(client.send(m).isSuccess());
      } catch (IOException e) {
        throw new IllegalStateException("Failed to send " + m, e);
      }
      f.complete(null);
    }).start();
  }
  JavaUtils.allOf(futures).get();

  final List<RaftPeerId> ids = new ArrayList<>();
  final RaftServerImpl leader = cluster.getLeader();
  final RaftLog leaderLog = leader.getState().getLog();
  final RaftPeerId leaderId = leader.getId();
  ids.add(leaderId);

  RaftTestUtil.getStateMachineLogEntries(leaderLog);

  // check that the last metadata entry is written to the log
  JavaUtils.attempt(() -> assertLastLogEntry(leader), 20, HUNDRED_MILLIS, "leader last metadata entry", LOG);

  final long lastIndex = leaderLog.getLastEntryTermIndex().getIndex();
  LOG.info("{}: leader lastIndex={}", leaderId, lastIndex);
  final LogEntryProto lastEntry = leaderLog.get(lastIndex);
  LOG.info("{}: leader lastEntry entry[{}] = {}", leaderId, lastIndex, ServerProtoUtils.toLogEntryString(lastEntry));
  final long loggedCommitIndex = lastEntry.getMetadataEntry().getCommitIndex();
  final LogEntryProto lastCommittedEntry = leaderLog.get(loggedCommitIndex);
  LOG.info("{}: leader lastCommittedEntry = entry[{}] = {}",
      leaderId, loggedCommitIndex, ServerProtoUtils.toLogEntryString(lastCommittedEntry));

  final SimpleStateMachine4Testing leaderStateMachine = SimpleStateMachine4Testing.get(leader);
  final TermIndex lastAppliedTermIndex = leaderStateMachine.getLastAppliedTermIndex();
  LOG.info("{}: leader lastAppliedTermIndex = {}", leaderId, lastAppliedTermIndex);

  // check follower logs
  for(RaftServerImpl s : cluster.iterateServerImpls()) {
    if (!s.getId().equals(leaderId)) {
      ids.add(s.getId());
      RaftTestUtil.assertSameLog(leaderLog, s.getState().getLog());
    }
  }

  // take snapshot and truncate last (metadata) entry
  leaderStateMachine.takeSnapshot();
  leaderLog.truncate(lastIndex);

  // kill all servers
  ids.forEach(cluster::killServer);

  // Restart and kill servers one by one so that they won't talk to each other.
  for(RaftPeerId id : ids) {
    cluster.restartServer(id, false);
    final RaftServerImpl server = cluster.getRaftServerImpl(id);
    final RaftLog raftLog = server.getState().getLog();
    JavaUtils.attemptRepeatedly(() -> {
      Assert.assertTrue(raftLog.getLastCommittedIndex() >= loggedCommitIndex);
      return null;
    }, 10, HUNDRED_MILLIS, id + "(commitIndex >= loggedCommitIndex)", LOG);
    JavaUtils.attemptRepeatedly(() -> {
      Assert.assertTrue(server.getState().getLastAppliedIndex() >= loggedCommitIndex);
      return null;
    }, 10, HUNDRED_MILLIS, id + "(lastAppliedIndex >= loggedCommitIndex)", LOG);
    LOG.info("{}: commitIndex={}, lastAppliedIndex={}",
        id, raftLog.getLastCommittedIndex(), server.getState().getLastAppliedIndex());
    cluster.killServer(id);
  }
}
 

private void testRestartFollower(CLUSTER cluster) throws Exception {
  leaderSnapshotInfoRef.set(null);
  int i = 0;
  final RaftServerImpl leader = RaftTestUtil.waitForLeader(cluster);
  final RaftPeerId leaderId = leader.getId();

  try (final RaftClient client = cluster.createClient(leaderId)) {
    for (; i < SNAPSHOT_TRIGGER_THRESHOLD * 2 - 1; i++) {
      final RaftClientReply reply = client.send(new RaftTestUtil.SimpleMessage("m" + i));
      Assert.assertTrue(reply.isSuccess());
    }
  }

  // wait for the snapshot to be done
  final long oldLeaderNextIndex = leader.getState().getLog().getNextIndex();
  {
    LOG.info("{}: oldLeaderNextIndex = {}", leaderId, oldLeaderNextIndex);
    final List<File> snapshotFiles = RaftSnapshotBaseTest.getSnapshotFiles(cluster,
        oldLeaderNextIndex - SNAPSHOT_TRIGGER_THRESHOLD, oldLeaderNextIndex);
    JavaUtils.attemptRepeatedly(() -> {
      Assert.assertTrue(snapshotFiles.stream().anyMatch(RaftSnapshotBaseTest::exists));
      return null;
    }, 10, ONE_SECOND, "snapshotFile.exist", LOG);
  }

  final RaftPeerId followerId = cluster.getFollowers().get(0).getId();
  cluster.killServer(followerId);

  // generate some more traffic
  try (final RaftClient client = cluster.createClient(leader.getId())) {
    Assert.assertTrue(client.send(new RaftTestUtil.SimpleMessage("m" + i)).isSuccess());
  }

  FIVE_SECONDS.sleep();
  cluster.restartServer(followerId, false);
  final RaftServerImpl follower = cluster.getRaftServerImpl(followerId);
  JavaUtils.attempt(() -> {
    final long newLeaderNextIndex = leader.getState().getLog().getNextIndex();
    LOG.info("{}: newLeaderNextIndex = {}", leaderId, newLeaderNextIndex);
    Assert.assertTrue(newLeaderNextIndex > oldLeaderNextIndex);
    Assert.assertEquals(newLeaderNextIndex, follower.getState().getLog().getNextIndex());
  }, 10, ONE_SECOND, "followerNextIndex", LOG);

}
 

/**
 * Basic test for install snapshot: start a one node cluster and let it
 * generate a snapshot. Then delete the log and restart the node, and add more
 * nodes as followers.
 */
@Test
public void testBasicInstallSnapshot() throws Exception {
  final List<LogPathAndIndex> logs;
  int i = 0;
  try {
    RaftTestUtil.waitForLeader(cluster);
    final RaftPeerId leaderId = cluster.getLeader().getId();

    try(final RaftClient client = cluster.createClient(leaderId)) {
      for (; i < SNAPSHOT_TRIGGER_THRESHOLD * 2 - 1; i++) {
        RaftClientReply reply = client.send(new SimpleMessage("m" + i));
        Assert.assertTrue(reply.isSuccess());
      }
    }

    // wait for the snapshot to be done
    RaftStorageDirectory storageDirectory = cluster.getLeader().getState()
        .getStorage().getStorageDir();

    final long nextIndex = cluster.getLeader().getState().getLog().getNextIndex();
    LOG.info("nextIndex = {}", nextIndex);
    final List<File> snapshotFiles = getSnapshotFiles(cluster, nextIndex - SNAPSHOT_TRIGGER_THRESHOLD, nextIndex);
    JavaUtils.attemptRepeatedly(() -> {
      Assert.assertTrue(snapshotFiles.stream().anyMatch(RaftSnapshotBaseTest::exists));
      return null;
    }, 10, ONE_SECOND, "snapshotFile.exist", LOG);
    verifyTakeSnapshotMetric(cluster.getLeader());
    logs = storageDirectory.getLogSegmentFiles();
  } finally {
    cluster.shutdown();
  }

  // delete the log segments from the leader
  for (LogPathAndIndex path : logs) {
    FileUtils.delete(path.getPath());
  }

  // restart the peer
  LOG.info("Restarting the cluster");
  cluster.restart(false);
  try {
    assertLeaderContent(cluster);

    // generate some more traffic
    try(final RaftClient client = cluster.createClient(cluster.getLeader().getId())) {
      Assert.assertTrue(client.send(new SimpleMessage("m" + i)).isSuccess());
    }

    // add two more peers
    MiniRaftCluster.PeerChanges change = cluster.addNewPeers(
        new String[]{"s3", "s4"}, true);
    // trigger setConfiguration
    cluster.setConfiguration(change.allPeersInNewConf);

    // Verify installSnapshot counter on leader before restart.
    verifyInstallSnapshotMetric(cluster.getLeader());
    RaftServerTestUtil.waitAndCheckNewConf(cluster, change.allPeersInNewConf, 0, null);

    Timer timer = getTakeSnapshotTimer(cluster.getLeader());
    long count = timer.getCount();

    // restart the peer and check if it can correctly handle conf change
    cluster.restartServer(cluster.getLeader().getId(), false);
    assertLeaderContent(cluster);

    // verify that snapshot was taken when stopping the server
    Assert.assertTrue(count < timer.getCount());
  } finally {
    cluster.shutdown();
  }
}
 

void runTestRevertConfigurationChange(CLUSTER cluster) throws Exception {
  RaftLog log2 = null;
  try {
    RaftTestUtil.waitForLeader(cluster);

    final RaftServerImpl leader = cluster.getLeader();
    final RaftPeerId leaderId = leader.getId();

    final RaftLog log = leader.getState().getLog();
    log2 = log;
    Thread.sleep(1000);

    // we block the incoming msg for the leader and block its requests to
    // followers, so that we force the leader change and the old leader will
    // not know
    LOG.info("start blocking the leader");
    BlockRequestHandlingInjection.getInstance().blockReplier(leaderId.toString());
    cluster.setBlockRequestsFrom(leaderId.toString(), true);

    PeerChanges change = cluster.removePeers(1, false, new ArrayList<>());

    AtomicBoolean gotNotLeader = new AtomicBoolean(false);
    final Thread clientThread = new Thread(() -> {
      try(final RaftClient client = cluster.createClient(leaderId)) {
        LOG.info("client starts to change conf");
        final RaftClientRpc sender = client.getClientRpc();
        RaftClientReply reply = sender.sendRequest(cluster.newSetConfigurationRequest(
            client.getId(), leaderId, change.allPeersInNewConf));
        if (reply.getNotLeaderException() != null) {
          gotNotLeader.set(true);
        }
      } catch (IOException e) {
        LOG.warn("Got unexpected exception when client1 changes conf", e);
      }
    });
    clientThread.start();

    // find ConfigurationEntry
    final TimeDuration sleepTime = TimeDuration.valueOf(500, TimeUnit.MILLISECONDS);
    final long confIndex = JavaUtils.attemptRepeatedly(() -> {
      final long last = log.getLastEntryTermIndex().getIndex();
      for (long i = last; i >= 1; i--) {
        if (log.get(i).hasConfigurationEntry()) {
          return i;
        }
      }
      throw new Exception("ConfigurationEntry not found: last=" + last);
    }, 10, sleepTime, "confIndex", LOG);

    // wait till the old leader persist the new conf
    JavaUtils.attemptRepeatedly(() -> {
      Assert.assertTrue(log.getFlushIndex() >= confIndex);
      return null;
    }, 10, sleepTime, "FLUSH", LOG);
    final long committed = log.getLastCommittedIndex();
    Assert.assertTrue(committed < confIndex);

    // unblock the old leader
    BlockRequestHandlingInjection.getInstance().unblockReplier(leaderId.toString());
    cluster.setBlockRequestsFrom(leaderId.toString(), false);

    // the client should get NotLeaderException
    clientThread.join(5000);
    Assert.assertTrue(gotNotLeader.get());

    // the old leader should have truncated the setConf from the log
    JavaUtils.attemptRepeatedly(() -> {
      Assert.assertTrue(log.getLastCommittedIndex() >= confIndex);
      return null;
    }, 10, ONE_SECOND, "COMMIT", LOG);
    Assert.assertTrue(log.get(confIndex).hasConfigurationEntry());
    log2 = null;
  } finally {
    RaftStorageTestUtils.printLog(log2, s -> LOG.info(s));
  }
}
 

void runTestRetryOnNewLeader(CLUSTER cluster) throws Exception {
  RaftTestUtil.waitForLeader(cluster);
  final RaftPeerId leaderId = cluster.getLeaderAndSendFirstMessage(false).getId();

  try (final RaftClient client = cluster.createClient(leaderId)) {
    RaftClientRpc rpc = client.getClientRpc();
    final long callId = 999;
    RaftClientRequest r = cluster.newRaftClientRequest(client.getId(), leaderId,
            callId, new SimpleMessage("message"));
    assertReply(rpc.sendRequest(r), client, callId);
    long oldLastApplied = cluster.getLeader().getState().getLastAppliedIndex();

    // trigger the reconfiguration, make sure the original leader is kicked out
    PeerChanges change = cluster.addNewPeers(2, true);
    RaftPeer[] allPeers = cluster.removePeers(2, true,
            asList(change.newPeers)).allPeersInNewConf;
    // trigger setConfiguration
    cluster.setConfiguration(allPeers);

    final RaftPeerId newLeaderId = JavaUtils.attemptRepeatedly(() -> {
      final RaftPeerId id = RaftTestUtil.waitForLeader(cluster).getId();
      Assert.assertNotEquals(leaderId, id);
      return id;
    }, 10, TimeDuration.valueOf(100, TimeUnit.MILLISECONDS), "wait for a leader different than " + leaderId, LOG);
    Assert.assertNotEquals(leaderId, newLeaderId);
    // same clientId and callId in the request
    r = cluster.newRaftClientRequest(client.getId(), newLeaderId,
            callId, new SimpleMessage("message"));
    rpc.addServers(Arrays.asList(change.newPeers));
    for (int i = 0; i < 10; i++) {
      try {
        assertReply(rpc.sendRequest(r), client, callId);
        LOG.info("successfully sent out the retry request_" + i);
      } catch (Exception e) {
        LOG.info("hit exception while retrying the same request: " + r, e);
      }
      Thread.sleep(100);
    }

    // check the new leader and make sure the retry did not get committed
    Assert.assertEquals(0, count(cluster.getLeader().getState().getLog(), oldLastApplied + 1));
  }
}