org.apache.ratis.server.RaftServer Java Examples

@Override
public void run() throws Exception {
  RaftPeerId peerId = RaftPeerId.valueOf(id);
  RaftProperties properties = new RaftProperties();

  final int port = NetUtils.createSocketAddr(getPeer(peerId).getAddress()).getPort();
  GrpcConfigKeys.Server.setPort(properties, port);
  properties.setInt(GrpcConfigKeys.OutputStream.RETRY_TIMES_KEY, Integer.MAX_VALUE);
  RaftServerConfigKeys.setStorageDir(properties, Collections.singletonList(storageDir));
  StateMachine stateMachine = new ArithmeticStateMachine();

  final RaftGroup raftGroup = RaftGroup.valueOf(RaftGroupId.valueOf(ByteString.copyFromUtf8(getRaftGroupId())),
          getPeers());
  RaftServer raftServer = RaftServer.newBuilder()
      .setServerId(RaftPeerId.valueOf(id))
      .setStateMachine(stateMachine).setProperties(properties)
      .setGroup(raftGroup)
      .build();
  raftServer.start();

  for(; raftServer.getLifeCycleState() != LifeCycle.State.CLOSED;) {
    TimeUnit.SECONDS.sleep(1);
  }
}
 

static <SERVER extends RaftServer> void blockQueueAndSetDelay(
    Collection<SERVER> servers,
    DelayLocalExecutionInjection injection, String leaderId, int delayMs,
    TimeDuration maxTimeout) throws InterruptedException {
  // block reqeusts sent to leader if delayMs > 0
  final boolean block = delayMs > 0;
  LOG.debug("{} requests sent to leader {} and set {}ms delay for the others",
      block? "Block": "Unblock", leaderId, delayMs);
  if (block) {
    BlockRequestHandlingInjection.getInstance().blockReplier(leaderId);
  } else {
    BlockRequestHandlingInjection.getInstance().unblockReplier(leaderId);
  }

  // delay RaftServerRequest for other servers
  servers.stream().filter(s -> !s.getId().toString().equals(leaderId))
      .forEach(s -> {
        if (block) {
          injection.setDelayMs(s.getId().toString(), delayMs);
        } else {
          injection.removeDelay(s.getId().toString());
        }
      });

  Thread.sleep(3 * maxTimeout.toLong(TimeUnit.MILLISECONDS));
}
 

@Override
public void run() throws Exception {
  RaftPeerId peerId = RaftPeerId.valueOf(id);
  RaftProperties properties = new RaftProperties();

  RaftPeer[] peers = getPeers();
  final int port = NetUtils.createSocketAddr(getPeer(peerId).getAddress()).getPort();
  GrpcConfigKeys.Server.setPort(properties, port);
  properties.setInt(GrpcConfigKeys.OutputStream.RETRY_TIMES_KEY, Integer.MAX_VALUE);
  RaftServerConfigKeys.setStorageDirs(properties, Collections.singletonList(storageDir));
  ConfUtils.setFile(properties::setFile, FileStoreCommon.STATEMACHINE_DIR_KEY,
      storageDir);
  StateMachine stateMachine = new FileStoreStateMachine(properties);

  final RaftGroup raftGroup = RaftGroup.valueOf(RaftGroupId.valueOf(ByteString.copyFromUtf8(raftGroupId)), peers);
  RaftServer raftServer = RaftServer.newBuilder()
      .setServerId(RaftPeerId.valueOf(id))
      .setStateMachine(stateMachine).setProperties(properties)
      .setGroup(raftGroup)
      .build();
  raftServer.start();

  for(; raftServer.getLifeCycleState() != LifeCycle.State.CLOSED;) {
    TimeUnit.SECONDS.sleep(1);
  }
}
 

@Override
public void run() throws Exception {
  RaftPeerId peerId = RaftPeerId.valueOf(id);
  RaftProperties properties = new RaftProperties();

  RaftPeer[] peers = getPeers();
  final int port = NetUtils.createSocketAddr(getPeer(peerId).getAddress()).getPort();
  GrpcConfigKeys.Server.setPort(properties, port);
  properties.setInt(GrpcConfigKeys.OutputStream.RETRY_TIMES_KEY, Integer.MAX_VALUE);
  RaftServerConfigKeys.setStorageDirs(properties, Collections.singletonList(storageDir));
  StateMachine stateMachine = new ArithmeticStateMachine();

  final RaftGroup raftGroup = RaftGroup.valueOf(RaftGroupId.valueOf(ByteString.copyFromUtf8(raftGroupId)), peers);
  RaftServer raftServer = RaftServer.newBuilder()
      .setServerId(RaftPeerId.valueOf(id))
      .setStateMachine(stateMachine).setProperties(properties)
      .setGroup(raftGroup)
      .build();
  raftServer.start();

  for(; raftServer.getLifeCycleState() != LifeCycle.State.CLOSED;) {
    TimeUnit.SECONDS.sleep(1);
  }
}
 

@Override
public synchronized void initialize(RaftServer server, RaftGroupId groupId,
    RaftStorage raftStorage) throws IOException {
  LOG.info("Initializing " + this);
  this.groupId = groupId;
  getLifeCycle().startAndTransition(() -> {
    super.initialize(server, groupId, raftStorage);
    storage.init(raftStorage);
    loadSnapshot(storage.findLatestSnapshot());

    if (properties.getBoolean(
        RAFT_TEST_SIMPLE_STATE_MACHINE_TAKE_SNAPSHOT_KEY,
        RAFT_TEST_SIMPLE_STATE_MACHINE_TAKE_SNAPSHOT_DEFAULT)) {
      checkpointer.start();
    }
  });
}
 

private HadoopRpcService(RaftServer server, final Configuration conf) {
  super(server::getId,
      id -> new PeerProxyMap<>(id.toString(),
          p -> new Proxy<>(RaftServerProtocolPB.class, p.getAddress(), conf)));
  try {
    this.ipcServer = newRpcServer(server, conf);
  } catch (IOException e) {
    throw new RuntimeException("Failed to create Hadoop rpc server.", e);
  }
  this.ipcServerAddress = ipcServer.getListenerAddress();

  addRaftClientProtocol(server, conf);

  LOG.info(getClass().getSimpleName() + " created RPC.Server at "
      + ipcServerAddress);
}
 

private HadoopRpcService(RaftServer server, final Configuration conf) {
  super(server::getId,
      id -> new PeerProxyMap<>(id.toString(),
          p -> new Proxy<>(RaftServerProtocolPB.class, p.getAddress(), conf)));
  try {
    this.ipcServer = newRpcServer(server, conf);
  } catch (IOException e) {
    throw new RuntimeException("Failed to create Hadoop rpc server.", e);
  }
  this.ipcServerAddress = ipcServer.getListenerAddress();

  addRaftClientProtocol(server, conf);

  LOG.info(getClass().getSimpleName() + " created RPC.Server at "
      + ipcServerAddress);
}
 

@Override
public synchronized void initialize(RaftServer server, RaftGroupId groupId,
    RaftStorage raftStorage) throws IOException {
  LOG.info("Initializing " + this);
  lifeCycle.startAndTransition(() -> {
    super.initialize(server, groupId, raftStorage);
    storage.init(raftStorage);
    loadSnapshot(storage.findLatestSnapshot());

    if (properties.getBoolean(
        RAFT_TEST_SIMPLE_STATE_MACHINE_TAKE_SNAPSHOT_KEY,
        RAFT_TEST_SIMPLE_STATE_MACHINE_TAKE_SNAPSHOT_DEFAULT)) {
      checkpointer.start();
    }
  });
}
 

static <SERVER extends RaftServer> void blockQueueAndSetDelay(
    Collection<SERVER> servers,
    DelayLocalExecutionInjection injection, String leaderId, int delayMs,
    long maxTimeout) throws InterruptedException {
  // block reqeusts sent to leader if delayMs > 0
  final boolean block = delayMs > 0;
  LOG.debug("{} requests sent to leader {} and set {}ms delay for the others",
      block? "Block": "Unblock", leaderId, delayMs);
  if (block) {
    BlockRequestHandlingInjection.getInstance().blockReplier(leaderId);
  } else {
    BlockRequestHandlingInjection.getInstance().unblockReplier(leaderId);
  }

  // delay RaftServerRequest for other servers
  servers.stream().filter(s -> !s.getId().toString().equals(leaderId))
      .forEach(s -> {
        if (block) {
          injection.setDelayMs(s.getId().toString(), delayMs);
        } else {
          injection.removeDelay(s.getId().toString());
        }
      });

  Thread.sleep(3 * maxTimeout);
}
 

private RaftGroup getGroupFromGroupId(RaftServer raftServer, RaftGroupId raftGroupId)
    throws IOException {
  List<RaftGroup> x = StreamSupport.stream(raftServer.getGroups().spliterator(), false)
      .filter(group -> group.getGroupId().equals(raftGroupId)).collect(Collectors.toList());
  if (x.size() == 1) {
    return x.get(0);
  } else {
    throw new IOException(x.size() + " are group found for group id:" + raftGroupId);
  }
}
 

@Override
public void initialize(RaftServer server, RaftGroupId groupId, RaftStorage storage) throws IOException {
    this.raftServer = server;
    this.logServiceMetaDataMetrics = new LogServiceMetaDataMetrics(server.getId().toString());
    super.initialize(server, groupId, storage);
    peerHealthChecker = new Daemon(new PeerHealthChecker(),"peer-Health-Checker");
    peerHealthChecker.start();
}
 

@Override
public void initialize(RaftServer server, RaftGroupId groupId,
    RaftStorage raftStorage) throws IOException {
  super.initialize(server, groupId, raftStorage);
  this.storage.init(raftStorage);
  loadSnapshot(storage.getLatestSnapshot());
}
 

private GrpcService(RaftServer server, GrpcTlsConfig tlsConfig) {
  this(server, server::getId,
      GrpcConfigKeys.Server.port(server.getProperties()),
      GrpcConfigKeys.messageSizeMax(server.getProperties(), LOG::info),
      RaftServerConfigKeys.Log.Appender.bufferByteLimit(server.getProperties()),
      GrpcConfigKeys.flowControlWindow(server.getProperties(), LOG::info),
      RaftServerConfigKeys.Rpc.requestTimeout(server.getProperties()),
      tlsConfig);
}
 

/** Constructs a netty server with the given port. */
private NettyRpcService(RaftServer server) {
  super(server::getId, id -> new NettyRpcProxy.PeerMap(id.toString(), server.getProperties()));
  this.server = server;

  final ChannelInitializer<SocketChannel> initializer
      = new ChannelInitializer<SocketChannel>() {
    @Override
    protected void initChannel(SocketChannel ch) throws Exception {
      final ChannelPipeline p = ch.pipeline();

      p.addLast(new ProtobufVarint32FrameDecoder());
      p.addLast(new ProtobufDecoder(RaftNettyServerRequestProto.getDefaultInstance()));
      p.addLast(new ProtobufVarint32LengthFieldPrepender());
      p.addLast(new ProtobufEncoder());

      p.addLast(new InboundHandler());
    }
  };

  final int port = NettyConfigKeys.Server.port(server.getProperties());
  channelFuture = new ServerBootstrap()
      .group(bossGroup, workerGroup)
      .channel(NioServerSocketChannel.class)
      .handler(new LoggingHandler(LogLevel.INFO))
      .childHandler(initializer)
      .bind(port);
}
 

private RaftClient getClient() throws IOException {
  if (client == null) {
    try {
      RaftServer raftServer = getServer().get();
      client = RaftClient.newBuilder().setRaftGroup(getGroupFromGroupId(raftServer, getGroupId()))
          .setClientId(ClientId.randomId())
          .setProperties(raftServer.getProperties()).build();
    } catch (Exception e) {
      throw new IOException(e);
    }
  }
  return client;
}
 

@Override
public void initialize(RaftServer server, RaftGroupId groupId,
    RaftStorage raftStorage) throws IOException {
  super.initialize(server, groupId, raftStorage);
  this.storage.init(raftStorage);
  this.proxy = (RaftServerProxy) server;
  //TODO: using groupId for metric now but better to tag it with LogName
  this.logServiceMetrics = new LogServiceMetrics(groupId.toString(),
      server.getId().toString());
  this.readNextQueryTimer = logServiceMetrics.getTimer("readNextQueryTime");
  this.startIndexTimer= logServiceMetrics.getTimer("startIndexTime");
  this.sizeRequestTimer = logServiceMetrics.getTimer("sizeRequestTime");
  this.getStateTimer = logServiceMetrics.getTimer("getStateTime");
  this.lastIndexQueryTimer = logServiceMetrics.getTimer("lastIndexQueryTime");
  this.lengthQueryTimer = logServiceMetrics.getTimer("lengthQueryTime");
  this.syncRequesTimer = logServiceMetrics.getTimer("syncRequesTime");
  this.appendRequestTimer = logServiceMetrics.getTimer("appendRequestTime");
  this.getCloseLogTimer = logServiceMetrics.getTimer("getCloseLogTime");
  //archiving request time not the actual archiving time
  this.archiveLogRequestTimer = logServiceMetrics.getTimer("archiveLogRequestTime");
  this.archiveLogTimer = logServiceMetrics.getTimer("archiveLogTime");
  loadSnapshot(storage.getLatestSnapshot());
  executorService = Executors.newSingleThreadExecutor();
  this.archivalInfo =
      new ArchivalInfo(properties.get(Constants.LOG_SERVICE_ARCHIVAL_LOCATION_KEY));


}
 

public static void main(String[] args) throws IOException, InterruptedException {
  if (args.length < 1) {
    System.err.println("Usage: java -cp *.jar org.apache.ratis.examples.counter.server.CounterServer {serverIndex}");
    System.err.println("{serverIndex} could be 1, 2 or 3");
    System.exit(1);
  }

  //find current peer object based on application parameter
  RaftPeer currentPeer =
      CounterCommon.PEERS.get(Integer.parseInt(args[0]) - 1);

  //create a property object
  RaftProperties properties = new RaftProperties();

  //set the storage directory (different for each peer) in RaftProperty object
  File raftStorageDir = new File("./" + currentPeer.getId().toString());
  RaftServerConfigKeys.setStorageDir(properties,
      Collections.singletonList(raftStorageDir));

  //set the port which server listen to in RaftProperty object
  final int port = NetUtils.createSocketAddr(currentPeer.getAddress()).getPort();
  GrpcConfigKeys.Server.setPort(properties, port);

  //create the counter state machine which hold the counter value
  CounterStateMachine counterStateMachine = new CounterStateMachine();

  //create and start the Raft server
  RaftServer server = RaftServer.newBuilder()
      .setGroup(CounterCommon.RAFT_GROUP)
      .setProperties(properties)
      .setServerId(currentPeer.getId())
      .setStateMachine(counterStateMachine)
      .build();
  server.start();

  //exit when any input entered
  Scanner scanner = new Scanner(System.in);
  scanner.nextLine();
  server.close();
}
 

SimulatedServerRpc(RaftServer server,
    SimulatedRequestReply<RaftServerRequest, RaftServerReply> serverRequestReply,
    SimulatedRequestReply<RaftClientRequest, RaftClientReply> clientRequestReply) {
  this.server = (RaftServerProxy)server;

  final Supplier<String> id = () -> server.getId().toString();
  this.serverHandler = new RequestHandler<>(id, "serverHandler", serverRequestReply, serverHandlerImpl, 3);
  this.clientHandler = new RequestHandler<>(id, "clientHandler", clientRequestReply, clientHandlerImpl, 3);
}
 

private XceiverServerRatis(DatanodeDetails dd, int port,
    ContainerDispatcher dispatcher, ContainerController containerController,
    StateContext context, GrpcTlsConfig tlsConfig, ConfigurationSource conf)
    throws IOException {
  this.conf = conf;
  Objects.requireNonNull(dd, "id == null");
  datanodeDetails = dd;
  this.port = port;
  RaftProperties serverProperties = newRaftProperties();
  this.context = context;
  this.dispatcher = dispatcher;
  this.containerController = containerController;
  this.raftPeerId = RatisHelper.toRaftPeerId(dd);
  chunkExecutors = createChunkExecutors(conf);

  RaftServer.Builder builder =
      RaftServer.newBuilder().setServerId(raftPeerId)
          .setProperties(serverProperties)
          .setStateMachineRegistry(this::getStateMachine);
  if (tlsConfig != null) {
    builder.setParameters(GrpcFactory.newRaftParameters(tlsConfig));
  }
  this.server = builder.build();
  this.requestTimeout = conf.getTimeDuration(
      HddsConfigKeys.HDDS_DATANODE_RATIS_SERVER_REQUEST_TIMEOUT,
      HddsConfigKeys.HDDS_DATANODE_RATIS_SERVER_REQUEST_TIMEOUT_DEFAULT,
      TimeUnit.MILLISECONDS);
}
 

@Override
public void initialize(RaftServer server, RaftGroupId groupId, RaftStorage raftStorage)
    throws IOException {
  super.initialize(server, groupId, raftStorage);
  this.storage.init(raftStorage);
  FileUtils.createDirectories(files.getRoot());
}
 

@Override
public void initialize(RaftServer server, RaftGroupId groupId,
    RaftStorage raftStorage) throws IOException {
  super.initialize(server, groupId, raftStorage);
  this.storage.init(raftStorage);
  loadSnapshot(storage.getLatestSnapshot());
}
 

@Override
public HadoopRpcService newRaftServerRpc(RaftServer server) {
  return HadoopRpcService.newBuilder()
      .setServer(server)
      .setConf(getConf())
      .build();
}
 

private void addRaftClientProtocol(RaftServer server, Configuration conf) {
  final Class<?> protocol = CombinedClientProtocolPB.class;
  RPC.setProtocolEngine(conf, protocol, ProtobufRpcEngineShaded.class);

  final BlockingService service = CombinedClientProtocolService.newReflectiveBlockingService(
      new CombinedClientProtocolServerSideTranslatorPB(server));
  ipcServer.addProtocol(RPC.RpcKind.RPC_PROTOCOL_BUFFER, protocol, service);
}
 

public void start() throws IOException  {
    final ServerOpts opts = getServerOpts();
    if (opts.getHost() == null) {
        opts.setHost(LogServiceUtils.getHostName());
    }
    this.lifeCycle = new LifeCycle(this.id);
    RaftProperties properties = new RaftProperties();
    if(opts.getWorkingDir() != null) {
        RaftServerConfigKeys.setStorageDirs(properties, Collections.singletonList(new File(opts.getWorkingDir())));
    }
    GrpcConfigKeys.Server.setPort(properties, opts.getPort());
    NettyConfigKeys.Server.setPort(properties, opts.getPort());
    Set<RaftPeer> peers = getPeersFromQuorum(opts.getMetaQuorum());
    RaftGroupId raftMetaGroupId = RaftGroupId.valueOf(opts.getMetaGroupId());
    RaftGroup metaGroup = RaftGroup.valueOf(raftMetaGroupId, peers);
    metaStateMachine = new MetaStateMachine(raftMetaGroupId, RaftGroupId.valueOf(opts.getLogServerGroupId()));
    server = RaftServer.newBuilder()
            .setGroup(metaGroup)
            .setServerId(RaftPeerId.valueOf(id))
            .setStateMachineRegistry(raftGroupId -> {
                if(raftGroupId.equals(META_GROUP_ID)) {
                    return metaStateMachine;
                }
                return null;
            })
            .setProperties(properties).build();
    lifeCycle.startAndTransition(() -> {
        server.start();
    }, IOException.class);
}
 

@Override
public void initialize(RaftServer server, RaftGroupId groupId,
    RaftStorage raftStorage) throws IOException {
  super.initialize(server, groupId, raftStorage);
  this.storage.init(raftStorage);
  this.proxy = (RaftServerProxy) server;
  this.groupId = groupId;
  loadSnapshot(storage.getLatestSnapshot());
}
 

@Override
public GrpcService newRaftServerRpc(RaftServer server) {
  return GrpcService.newBuilder()
      .setServer(server)
      .setTlsConfig(tlsConfig)
      .build();
}
 

private GrpcService(RaftServer server, GrpcTlsConfig tlsConfig) {
  this(server, server::getId,
      GrpcConfigKeys.Server.port(server.getProperties()),
      GrpcConfigKeys.messageSizeMax(server.getProperties(), LOG::info),
      RaftServerConfigKeys.Log.Appender.bufferByteLimit(server.getProperties()),
      GrpcConfigKeys.flowControlWindow(server.getProperties(), LOG::info),
      RaftServerConfigKeys.Rpc.requestTimeout(server.getProperties()),
      tlsConfig);
}
 

private GrpcService(RaftServer raftServer, Supplier<RaftPeerId> idSupplier, int port,
    SizeInBytes grpcMessageSizeMax, SizeInBytes appenderBufferSize,
    SizeInBytes flowControlWindow,TimeDuration requestTimeoutDuration, GrpcTlsConfig tlsConfig) {
  super(idSupplier, id -> new PeerProxyMap<>(id.toString(),
      p -> new GrpcServerProtocolClient(p, flowControlWindow.getSizeInt(),
          requestTimeoutDuration, tlsConfig)));
  if (appenderBufferSize.getSize() > grpcMessageSizeMax.getSize()) {
    throw new IllegalArgumentException("Illegal configuration: "
        + RaftServerConfigKeys.Log.Appender.BUFFER_BYTE_LIMIT_KEY + " = " + appenderBufferSize
        + " > " + GrpcConfigKeys.MESSAGE_SIZE_MAX_KEY + " = " + grpcMessageSizeMax);
  }

  NettyServerBuilder nettyServerBuilder = NettyServerBuilder.forPort(port)
      .maxInboundMessageSize(grpcMessageSizeMax.getSizeInt())
      .flowControlWindow(flowControlWindow.getSizeInt())
      .addService(new GrpcServerProtocolService(idSupplier, raftServer))
      .addService(new GrpcClientProtocolService(idSupplier, raftServer))
      .addService(new GrpcAdminProtocolService(raftServer));

  if (tlsConfig != null) {
    SslContextBuilder sslContextBuilder =
        SslContextBuilder.forServer(tlsConfig.getCertChain(),
            tlsConfig.getPrivateKey());
    if (tlsConfig.getMtlsEnabled()) {
      sslContextBuilder.clientAuth(ClientAuth.REQUIRE);
      sslContextBuilder.trustManager(tlsConfig.getCertChain());
    }
    sslContextBuilder = GrpcSslContexts.configure(sslContextBuilder, OPENSSL);
    try {
      nettyServerBuilder.sslContext(sslContextBuilder.build());
    } catch (Exception ex) {
      throw new IllegalArgumentException("Failed to build SslContext, tlsConfig=" + tlsConfig, ex);
    }
  }
  server = nettyServerBuilder.build();
  addressSupplier = JavaUtils.memoize(() -> new InetSocketAddress(port != 0? port: server.getPort()));
}
 

/** Constructs a netty server with the given port. */
private NettyRpcService(RaftServer server) {
  super(server::getId, id -> new NettyRpcProxy.PeerMap(id.toString()));
  this.server = server;

  final ChannelInitializer<SocketChannel> initializer
      = new ChannelInitializer<SocketChannel>() {
    @Override
    protected void initChannel(SocketChannel ch) throws Exception {
      final ChannelPipeline p = ch.pipeline();

      p.addLast(new ProtobufVarint32FrameDecoder());
      p.addLast(new ProtobufDecoder(RaftNettyServerRequestProto.getDefaultInstance()));
      p.addLast(new ProtobufVarint32LengthFieldPrepender());
      p.addLast(new ProtobufEncoder());

      p.addLast(new InboundHandler());
    }
  };

  final int port = NettyConfigKeys.Server.port(server.getProperties());
  channelFuture = new ServerBootstrap()
      .group(bossGroup, workerGroup)
      .channel(NioServerSocketChannel.class)
      .handler(new LoggingHandler(LogLevel.INFO))
      .childHandler(initializer)
      .bind(port);
}
 

SimulatedServerRpc(RaftServer server,
    SimulatedRequestReply<RaftServerRequest, RaftServerReply> serverRequestReply,
    SimulatedRequestReply<RaftClientRequest, RaftClientReply> clientRequestReply) {
  this.server = (RaftServerProxy)server;

  final Supplier<String> id = () -> server.getId().toString();
  this.serverHandler = new RequestHandler<>(id, "serverHandler", serverRequestReply, serverHandlerImpl, 3);
  this.clientHandler = new RequestHandler<>(id, "clientHandler", clientRequestReply, clientHandlerImpl, 3);
}