org.apache.ratis.statemachine.StateMachine Java Examples

public static RaftServerProxy newRaftServer(
    RaftPeerId id, StateMachine.Registry stateMachineRegistry, RaftProperties properties, Parameters parameters)
    throws IOException {
  final TimeDuration sleepTime = TimeDuration.valueOf(500, TimeUnit.MILLISECONDS);
  final RaftServerProxy proxy;
  try {
    // attempt multiple times to avoid temporary bind exception
    proxy = JavaUtils.attemptRepeatedly(
        () -> new RaftServerProxy(id, stateMachineRegistry, properties, parameters),
        5, sleepTime, "new RaftServerProxy", RaftServerProxy.LOG);
  } catch (InterruptedException e) {
    throw IOUtils.toInterruptedIOException(
        "Interrupted when creating RaftServer " + id, e);
  }
  return proxy;
}
 

private long initStatemachine(StateMachine sm, RaftGroupId groupId)
    throws IOException {
  sm.initialize(server.getProxy(), groupId, storage);
  storage.setStateMachineStorage(sm.getStateMachineStorage());
  SnapshotInfo snapshot = sm.getLatestSnapshot();

  if (snapshot == null || snapshot.getTermIndex().getIndex() < 0) {
    return RaftServerConstants.INVALID_LOG_INDEX;
  }

  // get the raft configuration from raft metafile
  RaftConfiguration raftConf = storage.readRaftConfiguration();
  if (raftConf != null) {
    setRaftConf(raftConf.getLogEntryIndex(), raftConf);
  }
  return snapshot.getIndex();
}
 

@Override
public void run() throws Exception {
  RaftPeerId peerId = RaftPeerId.valueOf(id);
  RaftProperties properties = new RaftProperties();

  final int port = NetUtils.createSocketAddr(getPeer(peerId).getAddress()).getPort();
  GrpcConfigKeys.Server.setPort(properties, port);
  properties.setInt(GrpcConfigKeys.OutputStream.RETRY_TIMES_KEY, Integer.MAX_VALUE);
  RaftServerConfigKeys.setStorageDir(properties, Collections.singletonList(storageDir));
  StateMachine stateMachine = new ArithmeticStateMachine();

  final RaftGroup raftGroup = RaftGroup.valueOf(RaftGroupId.valueOf(ByteString.copyFromUtf8(getRaftGroupId())),
          getPeers());
  RaftServer raftServer = RaftServer.newBuilder()
      .setServerId(RaftPeerId.valueOf(id))
      .setStateMachine(stateMachine).setProperties(properties)
      .setGroup(raftGroup)
      .build();
  raftServer.start();

  for(; raftServer.getLifeCycleState() != LifeCycle.State.CLOSED;) {
    TimeUnit.SECONDS.sleep(1);
  }
}
 

RaftServerImpl(RaftGroup group, StateMachine stateMachine, RaftServerProxy proxy) throws IOException {
  final RaftPeerId id = proxy.getId();
  LOG.info("{}: new RaftServerImpl for {} with {}", id, group, stateMachine);
  this.groupId = group.getGroupId();
  this.lifeCycle = new LifeCycle(id);
  this.stateMachine = stateMachine;
  this.role = new RoleInfo(id);

  final RaftProperties properties = proxy.getProperties();
  minTimeoutMs = RaftServerConfigKeys.Rpc.timeoutMin(properties).toIntExact(TimeUnit.MILLISECONDS);
  maxTimeoutMs = RaftServerConfigKeys.Rpc.timeoutMax(properties).toIntExact(TimeUnit.MILLISECONDS);
  rpcSlownessTimeoutMs = RaftServerConfigKeys.Rpc.slownessTimeout(properties).toIntExact(TimeUnit.MILLISECONDS);
  Preconditions.assertTrue(maxTimeoutMs > minTimeoutMs,
      "max timeout: %s, min timeout: %s", maxTimeoutMs, minTimeoutMs);
  this.proxy = proxy;

  this.state = new ServerState(id, group, properties, this, stateMachine);
  this.retryCache = initRetryCache(properties);

  this.jmxAdapter = new RaftServerJmxAdapter();
}
 

@Test
public void testServerRestartOnException() throws Exception {
  RaftProperties properties = new RaftProperties();
  final MiniRaftClusterWithGrpc cluster
      = MiniRaftClusterWithGrpc.FACTORY.newCluster(1, properties);
  cluster.start();
  RaftPeerId leaderId = RaftTestUtil.waitForLeader(cluster).getId();
  GrpcConfigKeys.Server.setPort(properties, cluster.getLeader().getServerRpc().getInetSocketAddress().getPort());
  // Create a raft server proxy with server rpc bound to a different address
  // compared to leader. This helps in locking the raft storage directory to
  // be used by next raft server proxy instance.
  final StateMachine stateMachine = cluster.getLeader().getStateMachine();
  ServerImplUtils.newRaftServer(leaderId, cluster.getGroup(), gid -> stateMachine, properties, null);
  // Close the server rpc for leader so that new raft server can be bound to it.
  cluster.getLeader().getServerRpc().close();

  // Create a raft server proxy with server rpc bound to same address as
  // the leader. This step would fail as the raft storage has been locked by
  // the raft server proxy created earlier. Raft server proxy should close
  // the rpc server on failure.
  testFailureCase("start a new server with the same address",
      () -> ServerImplUtils.newRaftServer(leaderId, cluster.getGroup(), gid -> stateMachine, properties, null).start(),
      IOException.class, OverlappingFileLockException.class);
  // Try to start a raft server rpc at the leader address.
  cluster.getServer(leaderId).getFactory().newRaftServerRpc(cluster.getServer(leaderId));
}
 

@Override
public void run() throws Exception {
  RaftPeerId peerId = RaftPeerId.valueOf(id);
  RaftProperties properties = new RaftProperties();

  RaftPeer[] peers = getPeers();
  final int port = NetUtils.createSocketAddr(getPeer(peerId).getAddress()).getPort();
  GrpcConfigKeys.Server.setPort(properties, port);
  properties.setInt(GrpcConfigKeys.OutputStream.RETRY_TIMES_KEY, Integer.MAX_VALUE);
  RaftServerConfigKeys.setStorageDirs(properties, Collections.singletonList(storageDir));
  ConfUtils.setFile(properties::setFile, FileStoreCommon.STATEMACHINE_DIR_KEY,
      storageDir);
  StateMachine stateMachine = new FileStoreStateMachine(properties);

  final RaftGroup raftGroup = RaftGroup.valueOf(RaftGroupId.valueOf(ByteString.copyFromUtf8(raftGroupId)), peers);
  RaftServer raftServer = RaftServer.newBuilder()
      .setServerId(RaftPeerId.valueOf(id))
      .setStateMachine(stateMachine).setProperties(properties)
      .setGroup(raftGroup)
      .build();
  raftServer.start();

  for(; raftServer.getLifeCycleState() != LifeCycle.State.CLOSED;) {
    TimeUnit.SECONDS.sleep(1);
  }
}
 

@Override
public void run() throws Exception {
  RaftPeerId peerId = RaftPeerId.valueOf(id);
  RaftProperties properties = new RaftProperties();

  RaftPeer[] peers = getPeers();
  final int port = NetUtils.createSocketAddr(getPeer(peerId).getAddress()).getPort();
  GrpcConfigKeys.Server.setPort(properties, port);
  properties.setInt(GrpcConfigKeys.OutputStream.RETRY_TIMES_KEY, Integer.MAX_VALUE);
  RaftServerConfigKeys.setStorageDirs(properties, Collections.singletonList(storageDir));
  StateMachine stateMachine = new ArithmeticStateMachine();

  final RaftGroup raftGroup = RaftGroup.valueOf(RaftGroupId.valueOf(ByteString.copyFromUtf8(raftGroupId)), peers);
  RaftServer raftServer = RaftServer.newBuilder()
      .setServerId(RaftPeerId.valueOf(id))
      .setStateMachine(stateMachine).setProperties(properties)
      .setGroup(raftGroup)
      .build();
  raftServer.start();

  for(; raftServer.getLifeCycleState() != LifeCycle.State.CLOSED;) {
    TimeUnit.SECONDS.sleep(1);
  }
}
 

StateMachineUpdater(StateMachine stateMachine, RaftServerImpl server,
    ServerState serverState, long lastAppliedIndex, RaftProperties properties) {
  this.name = serverState.getMemberId() + "-" + getClass().getSimpleName();
  this.infoIndexChange = s -> LOG.info("{}: {}", name, s);
  this.debugIndexChange = s -> LOG.debug("{}: {}", name, s);

  this.stateMachine = stateMachine;
  this.server = server;
  this.raftLog = serverState.getLog();

  this.appliedIndex = new RaftLogIndex("appliedIndex", lastAppliedIndex);
  this.snapshotIndex = new RaftLogIndex("snapshotIndex", lastAppliedIndex);

  final boolean autoSnapshot = RaftServerConfigKeys.Snapshot.autoTriggerEnabled(properties);
  this.autoSnapshotThreshold = autoSnapshot? RaftServerConfigKeys.Snapshot.autoTriggerThreshold(properties): null;
  final int numSnapshotFilesRetained = RaftServerConfigKeys.Snapshot.retentionFileNum(properties);
  this.snapshotRetentionPolicy = new SnapshotRetentionPolicy() {
    @Override
    public int getNumSnapshotsRetained() {
      return numSnapshotFilesRetained;
    }
  };
  this.purgeUptoSnapshotIndex = RaftServerConfigKeys.Log.purgeUptoSnapshotIndex(properties);

  updater = new Daemon(this);
}
 

@Test
public void testHandleStateMachineException() throws Exception {
  final RaftProperties prop = getProperties();
  prop.setClass(MiniRaftCluster.STATEMACHINE_CLASS_KEY,
      StateMachineWithException.class, StateMachine.class);
  final MiniRaftCluster cluster = newCluster(3);
  cluster.start();

  RaftPeerId leaderId = RaftTestUtil.waitForLeader(cluster).getId();

  try(final RaftClient client = cluster.createClient(leaderId)) {
    client.send(new RaftTestUtil.SimpleMessage("m"));
    fail("Exception expected");
  } catch (StateMachineException e) {
    e.printStackTrace();
    Assert.assertTrue(e.getCause().getMessage().contains("Fake Exception"));
  }
  cluster.shutdown();
}
 

private long initStatemachine(StateMachine sm, RaftGroupId gid)
    throws IOException {
  sm.initialize(server.getProxy(), gid, storage);
  SnapshotInfo snapshot = sm.getLatestSnapshot();

  if (snapshot == null || snapshot.getTermIndex().getIndex() < 0) {
    return RaftLog.INVALID_LOG_INDEX;
  }

  // get the raft configuration from raft metafile
  RaftConfiguration raftConf = storage.readRaftConfiguration();
  if (raftConf != null) {
    setRaftConf(raftConf.getLogEntryIndex(), raftConf);
  }
  return snapshot.getIndex();
}
 

/** For the case that all {@link RaftServerImpl} objects share the same {@link StateMachine}. */
public static RaftServerProxy newRaftServer(
    RaftPeerId id, RaftGroup group, StateMachine.Registry stateMachineRegistry,
    RaftProperties properties, Parameters parameters) throws IOException {
  RaftServerProxy.LOG.debug("newRaftServer: {}, {}", id, group);
  final RaftServerProxy proxy = newRaftServer(id, stateMachineRegistry, properties, parameters);
  proxy.initGroups(group);
  return proxy;
}
 

ServerState(RaftPeerId id, RaftGroup group, RaftProperties prop,
            RaftServerImpl server, StateMachine stateMachine)
    throws IOException {
  this.selfId = id;
  this.server = server;
  RaftConfiguration initialConf = RaftConfiguration.newBuilder()
      .setConf(group.getPeers()).build();
  configurationManager = new ConfigurationManager(initialConf);
  LOG.info("{}: {}", id, configurationManager);

  // use full uuid string to create a subdirectory
  final File dir = chooseStorageDir(RaftServerConfigKeys.storageDirs(prop),
      group.getGroupId().getUuid().toString());
  storage = new RaftStorage(dir, RaftServerConstants.StartupOption.REGULAR);
  snapshotManager = new SnapshotManager(storage, id);

  long lastApplied = initStatemachine(stateMachine, group.getGroupId());

  // On start the leader is null, start the clock now
  leaderId = null;
  this.lastNoLeaderTime = Timestamp.currentTime();
  this.leaderElectionTimeoutMs =
      RaftServerConfigKeys.leaderElectionTimeout(prop).toIntExact(TimeUnit.MILLISECONDS);

  // we cannot apply log entries to the state machine in this step, since we
  // do not know whether the local log entries have been committed.
  log = initLog(id, prop, lastApplied, this::setRaftConf);

  RaftLog.Metadata metadata = log.loadMetadata();
  currentTerm = metadata.getTerm();
  votedFor = metadata.getVotedFor();

  stateMachineUpdater = new StateMachineUpdater(stateMachine, server, log,
       lastApplied, prop);
}
 

void installSnapshot(InstallSnapshotRequestProto request) throws IOException {
  // TODO: verify that we need to install the snapshot
  StateMachine sm = server.getStateMachine();
  sm.pause(); // pause the SM to prepare for install snapshot
  snapshotManager.installSnapshot(sm, request);
  updateInstalledSnapshotIndex(ServerProtoUtils.toTermIndex(request.getSnapshotChunk().getTermIndex()));
}
 

private static RaftServerProxy newRaftServer(
    RaftPeerId id, StateMachine.Registry stateMachineRegistry, RaftProperties properties, Parameters parameters)
    throws IOException {
  final RaftServerProxy proxy;
  try {
    // attempt multiple times to avoid temporary bind exception
    proxy = JavaUtils.attempt(
        () -> new RaftServerProxy(id, stateMachineRegistry, properties, parameters),
        5, 500L, "new RaftServerProxy", RaftServerProxy.LOG);
  } catch (InterruptedException e) {
    throw IOUtils.toInterruptedIOException(
        "Interrupted when creating RaftServer " + id, e);
  }
  return proxy;
}
 

@Test
public void testStream() throws Exception {
  final RaftProperties p = getProperties();
  p.setClass(MiniRaftCluster.STATEMACHINE_CLASS_KEY,
      SimpleStateMachine4Testing.class, StateMachine.class);

  runWithNewCluster(NUM_SERVERS, this::runTestStream);
}
 

@Test
public void testStreamAsync() throws Exception {
  final RaftProperties p = getProperties();
  RaftClientConfigKeys.Stream.setSubmessageSize(p, SUBMESSAGE_SIZE);
  p.setClass(MiniRaftCluster.STATEMACHINE_CLASS_KEY,
      SimpleStateMachine4Testing.class, StateMachine.class);

  runWithNewCluster(NUM_SERVERS, this::runTestStreamAsync);
  RaftClientConfigKeys.Stream.setSubmessageSize(p);
}
 

/** For the case that all {@link RaftServerImpl} objects share the same {@link StateMachine}. */
public static RaftServerProxy newRaftServer(
    RaftPeerId id, RaftGroup group, StateMachine.Registry stateMachineRegistry,
    RaftProperties properties, Parameters parameters) throws IOException {
  RaftServerProxy.LOG.debug("newRaftServer: {}, {}", id, group);
  final RaftServerProxy proxy = newRaftServer(id, stateMachineRegistry, properties, parameters);
  proxy.initGroups(group);
  return proxy;
}
 

public RaftProperties getProperties() {
  RaftServerConfigKeys.Rpc
      .setSlownessTimeout(properties, TimeDuration.valueOf(1, TimeUnit.SECONDS));
  properties.setClass(MiniRaftCluster.STATEMACHINE_CLASS_KEY,
      SimpleStateMachine4Testing.class, StateMachine.class);
  return properties;
}
 

void installSnapshot(InstallSnapshotRequestProto request) throws IOException {
  // TODO: verify that we need to install the snapshot
  StateMachine sm = server.getStateMachine();
  sm.pause(); // pause the SM to prepare for install snapshot
  snapshotManager.installSnapshot(sm, request);
  log.syncWithSnapshot(request.getTermIndex().getIndex());
  this.latestInstalledSnapshot = ServerProtoUtils.toTermIndex(
      request.getTermIndex());
}
 

@Override
protected RaftServerProxy newRaftServer(
    RaftPeerId id, StateMachine.Registry stateMachineRegistry , RaftGroup group,
    RaftProperties properties) throws IOException {
  serverRequestReply.addPeer(id);
  client2serverRequestReply.addPeer(id);
  return ServerImplUtils.newRaftServer(id, group, stateMachineRegistry, properties, parameters);
}
 

ServerState(RaftPeerId id, RaftGroup group, RaftProperties prop,
            RaftServerImpl server, StateMachine stateMachine)
    throws IOException {
  this.memberId = RaftGroupMemberId.valueOf(id, group.getGroupId());
  this.server = server;
  RaftConfiguration initialConf = RaftConfiguration.newBuilder()
      .setConf(group.getPeers()).build();
  configurationManager = new ConfigurationManager(initialConf);
  LOG.info("{}: {}", getMemberId(), configurationManager);

  // use full uuid string to create a subdirectory
  final File dir = chooseStorageDir(RaftServerConfigKeys.storageDir(prop),
      group.getGroupId().getUuid().toString());
  storage = new RaftStorage(dir, RaftServerConstants.StartupOption.REGULAR,
      RaftServerConfigKeys.Log.corruptionPolicy(prop));
  snapshotManager = new SnapshotManager(storage, id);

  long lastApplied = initStatemachine(stateMachine, group.getGroupId());

  // On start the leader is null, start the clock now
  leaderId = null;
  this.lastNoLeaderTime = Timestamp.currentTime();
  this.noLeaderTimeout = RaftServerConfigKeys.Notification.noLeaderTimeout(prop);

  // we cannot apply log entries to the state machine in this step, since we
  // do not know whether the local log entries have been committed.
  this.log = initRaftLog(getMemberId(), server, storage, this::setRaftConf, lastApplied, prop);

  RaftLog.Metadata metadata = log.loadMetadata();
  currentTerm.set(metadata.getTerm());
  votedFor = metadata.getVotedFor();

  stateMachineUpdater = new StateMachineUpdater(stateMachine, server, this, lastApplied, prop);
}
 

RaftServerProxy(RaftPeerId id, StateMachine.Registry stateMachineRegistry,
    RaftProperties properties, Parameters parameters) {
  this.properties = properties;
  this.stateMachineRegistry = stateMachineRegistry;

  final RpcType rpcType = RaftConfigKeys.Rpc.type(properties, LOG::info);
  this.factory = ServerFactory.cast(rpcType.newFactory(parameters));

  this.serverRpc = factory.newRaftServerRpc(this);
  this.id = id != null? id: RaftPeerId.valueOf(getIdStringFrom(serverRpc));
  this.lifeCycle = new LifeCycle(this.id + "-" + getClass().getSimpleName());
}
 

public RaftProperties getProperties() {
  RaftServerConfigKeys.Rpc
      .setSlownessTimeout(properties, TimeDuration.valueOf(1, TimeUnit.SECONDS));
  properties.setClass(MiniRaftCluster.STATEMACHINE_CLASS_KEY,
      SimpleStateMachine4Testing.class, StateMachine.class);
  return properties;
}
 

StateMachineUpdater(StateMachine stateMachine, RaftServerImpl server,
    RaftLog raftLog, long lastAppliedIndex, RaftProperties properties) {
  this.properties = properties;
  this.stateMachine = stateMachine;
  this.server = server;
  this.raftLog = raftLog;

  this.lastAppliedIndex = lastAppliedIndex;
  lastSnapshotIndex = lastAppliedIndex;

  autoSnapshotEnabled = RaftServerConfigKeys.Snapshot.autoTriggerEnabled(properties);
  autoSnapshotThreshold = RaftServerConfigKeys.Snapshot.autoTriggerThreshold(properties);
  updater = new Daemon(this);
}
 

public RaftProperties getProperties() {
  RaftServerConfigKeys
      .setLeaderElectionTimeout(properties, TimeDuration.valueOf(1, TimeUnit.SECONDS));
  properties.setClass(MiniRaftCluster.STATEMACHINE_CLASS_KEY,
      SimpleStateMachine4Testing.class, StateMachine.class);
  return properties;
}
 

public static <S extends StateMachine> Collection<Object[]> getMiniRaftClusters(
    Class<S> stateMachineClass, int clusterSize, Class<?>... clusterClasses)
    throws IOException {
  final RaftProperties prop = new RaftProperties();
  prop.setClass(MiniRaftCluster.STATEMACHINE_CLASS_KEY,
      stateMachineClass, StateMachine.class);
  return getMiniRaftClusters(prop, clusterSize, clusterClasses);
}
 

@Override
protected RaftServerProxy newRaftServer(
    RaftPeerId id, StateMachine.Registry stateMachineRegistry , RaftGroup group,
    RaftProperties properties) throws IOException {
  final Configuration hconf = new Configuration(hadoopConf);
  final String address = "0.0.0.0:" + getPort(id, group);
  HadoopConfigKeys.Ipc.setAddress(hconf, address);

  return ServerImplUtils.newRaftServer(id, group, stateMachineRegistry, properties,
      HadoopFactory.newRaftParameters(hconf));
}
 

@Override
protected RaftServerProxy newRaftServer(
    RaftPeerId id, StateMachine.Registry stateMachineRegistry, RaftGroup group,
    RaftProperties properties) throws IOException {
  GrpcConfigKeys.Server.setPort(properties, getPort(id, group));
  return ServerImplUtils.newRaftServer(id, group, stateMachineRegistry, properties, null);
}
 

@Override
protected RaftServerProxy newRaftServer(
    RaftPeerId id, StateMachine.Registry stateMachineRegistry , RaftGroup group,
    RaftProperties properties) throws IOException {
  NettyConfigKeys.Server.setPort(properties, getPort(id, group));
  return ServerImplUtils.newRaftServer(id, group, stateMachineRegistry, properties, null);
}
 

/**
 * Construct a {@link TransactionContext} from a client request.
 * Used by the state machine to start a transaction
 * and send the Log entry representing the transaction data
 * to be applied to the raft log.
 */
public TransactionContextImpl(
    StateMachine stateMachine, RaftClientRequest clientRequest,
    StateMachineLogEntryProto smLogEntryProto, Object stateMachineContext) {
  this(RaftPeerRole.LEADER, stateMachine);
  this.clientRequest = clientRequest;
  this.smLogEntryProto = smLogEntryProto != null? smLogEntryProto
      : ServerProtoUtils.toStateMachineLogEntryProto(clientRequest, null, null);
  this.stateMachineContext = stateMachineContext;
}