org.apache.ratis.protocol.StateMachineException Java Examples

/**
 * Query the n-th log entry.
 * @param request an index represented in a UTF-8 String, or an empty message.
 * @return a completed future of the n-th log entry,
 *         where n is the last applied index if the request is empty,
 *         otherwise, n is the index represented in the request.
 */
@Override
public CompletableFuture<Message> query(Message request) {
  final String string = request.getContent().toStringUtf8();
  Exception exception;
  try {
    LOG.info("query " + string);
    final LogEntryProto entry = dataMap.get(string);
    if (entry != null) {
      return CompletableFuture.completedFuture(Message.valueOf(entry.toByteString()));
    }
    exception = new IndexOutOfBoundsException(getId() + ": LogEntry not found for query " + string);
  } catch (Exception e) {
    LOG.warn("Failed request " + request, e);
    exception = e;
  }
  return JavaUtils.completeExceptionally(new StateMachineException(
      "Failed request " + request, exception));
}
 

/**
 * Query the n-th log entry.
 * @param request an index represented in a UTF-8 String, or an empty message.
 * @return a completed future of the n-th log entry,
 *         where n is the last applied index if the request is empty,
 *         otherwise, n is the index represented in the request.
 */
@Override
public CompletableFuture<Message> query(Message request) {
  final String string = request.getContent().toStringUtf8();
  Exception exception;
  try {
    LOG.info("query " + string);
    final LogEntryProto entry = dataMap.get(string);
    if (entry != null) {
      return CompletableFuture.completedFuture(Message.valueOf(entry.toByteString()));
    }
    exception = new IndexOutOfBoundsException(getId() + ": LogEntry not found for query " + string);
  } catch (Exception e) {
    LOG.warn("Failed request " + request, e);
    exception = e;
  }
  return JavaUtils.completeExceptionally(new StateMachineException(
      "Failed request " + request, exception));
}
 

private void takeSnapshot() {
  final long i;
  try {
    Timer.Context takeSnapshotTimerContext = stateMachineMetrics.getTakeSnapshotTimer().time();
    i = stateMachine.takeSnapshot();
    takeSnapshotTimerContext.stop();

    final long lastAppliedIndex = getLastAppliedIndex();
    if (i > lastAppliedIndex) {
      throw new StateMachineException(
          "Bug in StateMachine: snapshot index = " + i + " > appliedIndex = " + lastAppliedIndex
          + "; StateMachine class=" +  stateMachine.getClass().getName() + ", stateMachine=" + stateMachine);
    }
    stateMachine.getStateMachineStorage().cleanupOldSnapshots(snapshotRetentionPolicy);
  } catch (IOException e) {
    LOG.error(name + ": Failed to take snapshot", e);
    return;
  }

  if (i >= 0) {
    LOG.info("{}: Took a snapshot at index {}", name, i);
    snapshotIndex.updateIncreasingly(i, infoIndexChange);

    final long purgeIndex;
    if (purgeUptoSnapshotIndex) {
      // We can purge up to snapshot index even if all the peers do not have
      // commitIndex up to this snapshot index.
      purgeIndex = i;
    } else {
      final LongStream commitIndexStream = server.getCommitInfos().stream().mapToLong(
          CommitInfoProto::getCommitIndex);
      purgeIndex = LongStream.concat(LongStream.of(i), commitIndexStream).min().orElse(i);
    }
    raftLog.purge(purgeIndex);
  }
}
 

static CompletableFuture<ByteString> sendAsync(
    ByteString request, Function<Message, CompletableFuture<RaftClientReply>> sendFunction)
    {
  return sendFunction.apply(() -> request
  ).thenApply(reply -> {
    final StateMachineException sme = reply.getStateMachineException();
    if (sme != null) {
      throw new CompletionException("Failed to send request " + request, sme);
    }
    Preconditions.assertTrue(reply.isSuccess(), () -> "Failed " + request + ", reply=" + reply);
    return reply.getMessage().getContent();
  });
}
 

static ByteString send(
    ByteString request, CheckedFunction<Message, RaftClientReply, IOException> sendFunction)
    throws IOException {
  final RaftClientReply reply = sendFunction.apply(Message.valueOf(request));
  final StateMachineException sme = reply.getStateMachineException();
  if (sme != null) {
    throw new IOException("Failed to send request " + request, sme);
  }
  Preconditions.assertTrue(reply.isSuccess(), () -> "Failed " + request + ", reply=" + reply);
  return reply.getMessage().getContent();
}
 

@Override
public long takeSnapshot() throws IOException {
  TermIndex ti = getLastAppliedTermIndex();
  long startTime = Time.monotonicNow();
  if (!isStateMachineHealthy()) {
    String msg =
        "Failed to take snapshot " + " for " + gid + " as the stateMachine"
            + " is unhealthy. The last applied index is at " + ti;
    StateMachineException sme = new StateMachineException(msg);
    LOG.error(msg);
    throw sme;
  }
  if (ti != null && ti.getIndex() != RaftLog.INVALID_LOG_INDEX) {
    final File snapshotFile =
        storage.getSnapshotFile(ti.getTerm(), ti.getIndex());
    LOG.info("{}: Taking a snapshot at:{} file {}", gid, ti, snapshotFile);
    try (FileOutputStream fos = new FileOutputStream(snapshotFile)) {
      persistContainerSet(fos);
      fos.flush();
      // make sure the snapshot file is synced
      fos.getFD().sync();
    } catch (IOException ioe) {
      LOG.error("{}: Failed to write snapshot at:{} file {}", gid, ti,
          snapshotFile);
      throw ioe;
    }
    LOG.info("{}: Finished taking a snapshot at:{} file:{} time:{}", gid, ti,
        snapshotFile, (Time.monotonicNow() - startTime));
    return ti.getIndex();
  }
  return -1;
}
 

static CompletableFuture<ByteString> sendAsync(
    ByteString request, Function<Message, CompletableFuture<RaftClientReply>> sendFunction) {
  return sendFunction.apply(() -> request
  ).thenApply(reply -> {
    final StateMachineException sme = reply.getStateMachineException();
    if (sme != null) {
      throw new CompletionException("Failed to send request " + request, sme);
    }
    Preconditions.assertTrue(reply.isSuccess(), () -> "Failed " + request + ", reply=" + reply);
    return reply.getMessage().getContent();
  });
}
 

static ByteString send(
    ByteString request, CheckedFunction<Message, RaftClientReply, IOException> sendFunction)
    throws IOException {
  final RaftClientReply reply = sendFunction.apply(Message.valueOf(request));
  final StateMachineException sme = reply.getStateMachineException();
  if (sme != null) {
    throw new IOException("Failed to send request " + request, sme);
  }
  Preconditions.assertTrue(reply.isSuccess(), () -> "Failed " + request + ", reply=" + reply);
  return reply.getMessage().getContent();
}
 

private void runTestRestartWithCorruptedLogEntry(CLUSTER cluster) throws Exception {
  // this is the only server
  final RaftServerImpl leader = RaftTestUtil.waitForLeader(cluster);
  final RaftPeerId id = leader.getId();

  // send a few messages
  final SimpleMessage[] messages = SimpleMessage.create(10);
  final SimpleMessage lastMessage = messages[messages.length - 1];
  try (final RaftClient client = cluster.createClient()) {
    for (SimpleMessage m : messages) {
      Assert.assertTrue(client.send(m).isSuccess());
    }

    // assert that the last message exists
    Assert.assertTrue(client.sendReadOnly(lastMessage).isSuccess());
  }

  final RaftLog log = leader.getState().getLog();
  final long size = TestSegmentedRaftLog.getOpenSegmentSize(log);
  leader.getProxy().close();

  // corrupt the log
  final File openLogFile = JavaUtils.attemptRepeatedly(() -> getOpenLogFile(leader),
      10, HUNDRED_MILLIS, id + "-getOpenLogFile", LOG);
  try(final RandomAccessFile raf = new RandomAccessFile(openLogFile, "rw")) {
    final long mid = size / 2;
    raf.seek(mid);
    for (long i = mid; i < size; i++) {
      raf.write(0);
    }
  }

  // after the log is corrupted and the server is restarted, the last entry should no longer exist.
  cluster.restartServer(id, false);
  testFailureCase("last-entry-not-found", () -> {
    try (final RaftClient client = cluster.createClient()) {
      client.sendReadOnly(lastMessage);
    }
  }, StateMachineException.class, IndexOutOfBoundsException.class);
}
 

private void processReply(RaftClientReply reply) throws IOException {
  // NotLeader exception is thrown only when the raft server to which the
  // request is submitted is not the leader. The request will be rejected
  // and will eventually be executed once the request comes via the leader
  // node.
  NotLeaderException notLeaderException = reply.getNotLeaderException();
  if (notLeaderException != null) {
    throw notLeaderException;
  }
  StateMachineException stateMachineException =
      reply.getStateMachineException();
  if (stateMachineException != null) {
    throw stateMachineException;
  }
}
 

@Override
public final long append(long term, TransactionContext transaction) throws StateMachineException {
  return runner.runSequentially(() -> appendImpl(term, transaction));
}
 

void runTestStaleReadAsync(CLUSTER cluster) throws Exception {
  final int numMesssages = 10;
  try (RaftClient client = cluster.createClient()) {
    RaftTestUtil.waitForLeader(cluster);

    // submit some messages
    final List<CompletableFuture<RaftClientReply>> futures = new ArrayList<>();
    for (int i = 0; i < numMesssages; i++) {
      final String s = "" + i;
      LOG.info("sendAsync " + s);
      futures.add(client.sendAsync(new SimpleMessage(s)));
    }
    Assert.assertEquals(numMesssages, futures.size());
    final List<RaftClientReply> replies = new ArrayList<>();
    for (CompletableFuture<RaftClientReply> f : futures) {
      final RaftClientReply r = f.join();
      Assert.assertTrue(r.isSuccess());
      replies.add(r);
    }
    futures.clear();

    // Use a follower with the max commit index
    final RaftClientReply lastWriteReply = replies.get(replies.size() - 1);
    final RaftPeerId leader = lastWriteReply.getServerId();
    LOG.info("leader = " + leader);
    final Collection<CommitInfoProto> commitInfos = lastWriteReply.getCommitInfos();
    LOG.info("commitInfos = " + commitInfos);
    final CommitInfoProto followerCommitInfo = commitInfos.stream()
        .filter(info -> !RaftPeerId.valueOf(info.getServer().getId()).equals(leader))
        .max(Comparator.comparing(CommitInfoProto::getCommitIndex)).get();
    final RaftPeerId follower = RaftPeerId.valueOf(followerCommitInfo.getServer().getId());
    final long followerCommitIndex = followerCommitInfo.getCommitIndex();
    LOG.info("max follower = {}, commitIndex = {}", follower, followerCommitIndex);

    // test a failure case
    testFailureCaseAsync("sendStaleReadAsync(..) with a larger commit index",
        () -> client.sendStaleReadAsync(
            new SimpleMessage("" + Long.MAX_VALUE),
            followerCommitInfo.getCommitIndex(), follower),
        StateMachineException.class, IndexOutOfBoundsException.class);

    // test sendStaleReadAsync
    for (int i = 0; i < numMesssages; i++) {
      final RaftClientReply reply = replies.get(i);
      final String query = "" + i;
      LOG.info("query=" + query + ", reply=" + reply);
      final Message message = new SimpleMessage(query);
      final CompletableFuture<RaftClientReply> readFuture = client.sendReadOnlyAsync(message);

      futures.add(readFuture.thenCompose(r -> {
        if (reply.getLogIndex() <= followerCommitIndex) {
          LOG.info("sendStaleReadAsync, query=" + query);
          return client.sendStaleReadAsync(message, followerCommitIndex, follower);
        } else {
          return CompletableFuture.completedFuture(null);
        }
      }).thenApply(staleReadReply -> {
        if (staleReadReply == null) {
          return null;
        }

        final ByteString expected = readFuture.join().getMessage().getContent();
        final ByteString computed = staleReadReply.getMessage().getContent();
        try {
          LOG.info("query " + query + " returns "
              + LogEntryProto.parseFrom(expected).getStateMachineLogEntry().getLogData().toStringUtf8());
        } catch (InvalidProtocolBufferException e) {
          throw new CompletionException(e);
        }

        Assert.assertEquals("log entry mismatch for query=" + query, expected, computed);
        return null;
      }));
    }
    JavaUtils.allOf(futures).join();
  }
}
 

/**
 * Process the raftClientReply and return OMResponse.
 * @param omRequest
 * @param reply
 * @return OMResponse - response which is returned to client.
 * @throws ServiceException
 */
private OMResponse processReply(OMRequest omRequest, RaftClientReply reply)
    throws ServiceException {
  // NotLeader exception is thrown only when the raft server to which the
  // request is submitted is not the leader. This can happen first time
  // when client is submitting request to OM.

  if (!reply.isSuccess()) {
    NotLeaderException notLeaderException = reply.getNotLeaderException();
    if (notLeaderException != null) {
      throw new ServiceException(
          OMNotLeaderException.convertToOMNotLeaderException(
                notLeaderException, getRaftPeerId()));
    }

    LeaderNotReadyException leaderNotReadyException =
        reply.getLeaderNotReadyException();
    if (leaderNotReadyException != null) {
      throw new ServiceException(new OMLeaderNotReadyException(
          leaderNotReadyException.getMessage()));
    }

    StateMachineException stateMachineException =
        reply.getStateMachineException();
    if (stateMachineException != null) {
      OMResponse.Builder omResponse = OMResponse.newBuilder()
          .setCmdType(omRequest.getCmdType())
          .setSuccess(false)
          .setTraceID(omRequest.getTraceID());
      if (stateMachineException.getCause() != null) {
        omResponse.setMessage(stateMachineException.getCause().getMessage());
        omResponse.setStatus(
            exceptionToResponseStatus(stateMachineException.getCause()));
      } else {
        // Current Ratis is setting cause, this is an safer side check.
        LOG.error("StateMachine exception cause is not set");
        omResponse.setStatus(
            OzoneManagerProtocolProtos.Status.INTERNAL_ERROR);
        omResponse.setMessage(
            StringUtils.stringifyException(stateMachineException));
      }

      if (LOG.isDebugEnabled()) {
        LOG.debug("Error while executing ratis request. " +
            "stateMachineException: ", stateMachineException);
      }
      return omResponse.build();
    }
  }

  try {
    return OMRatisHelper.getOMResponseFromRaftClientReply(reply);
  } catch (InvalidProtocolBufferException ex) {
    if (ex.getMessage() != null) {
      throw new ServiceException(ex.getMessage(), ex);
    } else {
      throw new ServiceException(ex);
    }
  }

  // TODO: Still need to handle RaftRetry failure exception and
  //  NotReplicated exception.
}
 

@Test
public void testApplyTransactionFailure() throws Exception {
  OzoneOutputStream key =
          objectStore.getVolume(volumeName).getBucket(bucketName)
                  .createKey("ratis", 1024, ReplicationType.RATIS,
                          ReplicationFactor.ONE, new HashMap<>());
  // First write and flush creates a container in the datanode
  key.write("ratis".getBytes());
  key.flush();
  key.write("ratis".getBytes());
  KeyOutputStream groupOutputStream = (KeyOutputStream) key.
          getOutputStream();
  List<OmKeyLocationInfo> locationInfoList =
          groupOutputStream.getLocationInfoList();
  Assert.assertEquals(1, locationInfoList.size());
  OmKeyLocationInfo omKeyLocationInfo = locationInfoList.get(0);
  HddsDatanodeService dn = TestHelper.getDatanodeService(omKeyLocationInfo,
          cluster);
  int index = cluster.getHddsDatanodeIndex(dn.getDatanodeDetails());
  ContainerData containerData = dn.getDatanodeStateMachine()
                  .getContainer().getContainerSet()
                  .getContainer(omKeyLocationInfo.getContainerID())
                  .getContainerData();
  Assert.assertTrue(containerData instanceof KeyValueContainerData);
  KeyValueContainerData keyValueContainerData =
          (KeyValueContainerData) containerData;
  key.close();
  ContainerStateMachine stateMachine =
      (ContainerStateMachine) TestHelper.getStateMachine(cluster.
          getHddsDatanodes().get(index), omKeyLocationInfo.getPipeline());
  SimpleStateMachineStorage storage =
          (SimpleStateMachineStorage) stateMachine.getStateMachineStorage();
  stateMachine.takeSnapshot();
  Path parentPath = storage.findLatestSnapshot().getFile().getPath();
  // Since the snapshot threshold is set to 1, since there are
  // applyTransactions, we should see snapshots
  Assert.assertTrue(parentPath.getParent().toFile().listFiles().length > 0);
  FileInfo snapshot = storage.findLatestSnapshot().getFile();
  Assert.assertNotNull(snapshot);
  long containerID = omKeyLocationInfo.getContainerID();
  // delete the container db file
  FileUtil.fullyDelete(new File(keyValueContainerData.getContainerPath()));
  Pipeline pipeline = cluster.getStorageContainerLocationClient()
          .getContainerWithPipeline(containerID).getPipeline();
  XceiverClientSpi xceiverClient =
          xceiverClientManager.acquireClient(pipeline);
  ContainerProtos.ContainerCommandRequestProto.Builder request =
          ContainerProtos.ContainerCommandRequestProto.newBuilder();
  request.setDatanodeUuid(pipeline.getFirstNode().getUuidString());
  request.setCmdType(ContainerProtos.Type.CloseContainer);
  request.setContainerID(containerID);
  request.setCloseContainer(
          ContainerProtos.CloseContainerRequestProto.getDefaultInstance());
  // close container transaction will fail over Ratis and will initiate
  // a pipeline close action

  try {
    xceiverClient.sendCommand(request.build());
    Assert.fail("Expected exception not thrown");
  } catch (IOException e) {
    // Exception should be thrown
  }
  // Make sure the container is marked unhealthy
  Assert.assertTrue(dn.getDatanodeStateMachine()
                  .getContainer().getContainerSet().getContainer(containerID)
                  .getContainerState()
                  == ContainerProtos.ContainerDataProto.State.UNHEALTHY);
  try {
    // try to take a new snapshot, ideally it should just fail
    stateMachine.takeSnapshot();
  } catch (IOException ioe) {
    Assert.assertTrue(ioe instanceof StateMachineException);
  }
  // Make sure the latest snapshot is same as the previous one
  FileInfo latestSnapshot = storage.findLatestSnapshot().getFile();
  Assert.assertTrue(snapshot.getPath().equals(latestSnapshot.getPath()));
}
 

@Override
public final long append(long term, TransactionContext transaction) throws StateMachineException {
  return runner.runSequentially(() -> appendImpl(term, transaction));
}
 

void runTestStaleReadAsync(CLUSTER cluster) throws Exception {
  final int numMesssages = 10;
  try (RaftClient client = cluster.createClient()) {
    RaftTestUtil.waitForLeader(cluster);

    // submit some messages
    final List<CompletableFuture<RaftClientReply>> futures = new ArrayList<>();
    for (int i = 0; i < numMesssages; i++) {
      final String s = "" + i;
      LOG.info("sendAsync " + s);
      futures.add(client.sendAsync(new SimpleMessage(s)));
    }
    Assert.assertEquals(numMesssages, futures.size());
    final List<RaftClientReply> replies = new ArrayList<>();
    for (CompletableFuture<RaftClientReply> f : futures) {
      final RaftClientReply r = f.join();
      Assert.assertTrue(r.isSuccess());
      replies.add(r);
    }
    futures.clear();

    // Use a follower with the max commit index
    final RaftClientReply lastWriteReply = replies.get(replies.size() - 1);
    final RaftPeerId leader = lastWriteReply.getServerId();
    LOG.info("leader = " + leader);
    final Collection<CommitInfoProto> commitInfos = lastWriteReply.getCommitInfos();
    LOG.info("commitInfos = " + commitInfos);
    final CommitInfoProto followerCommitInfo = commitInfos.stream()
        .filter(info -> !RaftPeerId.valueOf(info.getServer().getId()).equals(leader))
        .max(Comparator.comparing(CommitInfoProto::getCommitIndex)).get();
    final RaftPeerId follower = RaftPeerId.valueOf(followerCommitInfo.getServer().getId());
    final long followerCommitIndex = followerCommitInfo.getCommitIndex();
    LOG.info("max follower = {}, commitIndex = {}", follower, followerCommitIndex);

    // test a failure case
    testFailureCaseAsync("sendStaleReadAsync(..) with a larger commit index",
        () -> client.sendStaleReadAsync(
            new SimpleMessage("" + Long.MAX_VALUE),
            followerCommitInfo.getCommitIndex(), follower),
        StateMachineException.class, IndexOutOfBoundsException.class);

    // test sendStaleReadAsync
    for (int i = 0; i < numMesssages; i++) {
      final RaftClientReply reply = replies.get(i);
      final String query = "" + i;
      LOG.info("query=" + query + ", reply=" + reply);
      final Message message = new SimpleMessage(query);
      final CompletableFuture<RaftClientReply> readFuture = client.sendReadOnlyAsync(message);

      futures.add(readFuture.thenCompose(r -> {
        if (reply.getLogIndex() <= followerCommitIndex) {
          LOG.info("sendStaleReadAsync, query=" + query);
          return client.sendStaleReadAsync(message, followerCommitIndex, follower);
        } else {
          return CompletableFuture.completedFuture(null);
        }
      }).thenApply(staleReadReply -> {
        if (staleReadReply == null) {
          return null;
        }

        final ByteString expected = readFuture.join().getMessage().getContent();
        final ByteString computed = staleReadReply.getMessage().getContent();
        try {
          LOG.info("query " + query + " returns "
              + LogEntryProto.parseFrom(expected).getStateMachineLogEntry().getLogData().toStringUtf8());
        } catch (InvalidProtocolBufferException e) {
          throw new CompletionException(e);
        }

        Assert.assertEquals("log entry mismatch for query=" + query, expected, computed);
        return null;
      }));
    }
    JavaUtils.allOf(futures).join();
  }
}
 

/**
 * Append asynchronously a log entry for the given term and transaction.
 * Used by the leader.
 *
 * Note that the underlying I/O operation is submitted but may not be completed when this method returns.
 *
 * @return the index of the new log entry.
 */
long append(long term, TransactionContext transaction) throws StateMachineException;
 

/**
 * Append asynchronously a log entry for the given term and transaction.
 * Used by the leader.
 *
 * Note that the underlying I/O operation is submitted but may not be completed when this method returns.
 *
 * @return the index of the new log entry.
 */
long append(long term, TransactionContext transaction) throws StateMachineException;