org.apache.hadoop.ipc.Server Java Examples

private InetSocketAddress startAndGetRPCServerAddress(InetSocketAddress serverAddress) {
  Configuration conf = new Configuration();

  try {
    RPC.setProtocolEngine(conf,
        HAServiceProtocolPB.class, ProtobufRpcEngine.class);
    HAServiceProtocolServerSideTranslatorPB haServiceProtocolXlator =
        new HAServiceProtocolServerSideTranslatorPB(new MockHAProtocolImpl());
    BlockingService haPbService = HAServiceProtocolService
        .newReflectiveBlockingService(haServiceProtocolXlator);

    Server server = new RPC.Builder(conf)
        .setProtocol(HAServiceProtocolPB.class)
        .setInstance(haPbService)
        .setBindAddress(serverAddress.getHostName())
        .setPort(serverAddress.getPort()).build();
    server.start();
    return NetUtils.getConnectAddress(server);
  } catch (IOException e) {
    return null;
  }
}
 

@SuppressWarnings("unchecked")
@Override
public KillTaskResponse killTask(KillTaskRequest request) 
  throws IOException {
  TaskId taskId = request.getTaskId();
  UserGroupInformation callerUGI = UserGroupInformation.getCurrentUser();
  String message = "Kill task " + taskId + " received from " + callerUGI
      + " at " + Server.getRemoteAddress();
  LOG.info(message);
  verifyAndGetTask(taskId, JobACL.MODIFY_JOB);
  appContext.getEventHandler().handle(
      new TaskEvent(taskId, TaskEventType.T_KILL));
  KillTaskResponse response = 
    recordFactory.newRecordInstance(KillTaskResponse.class);
  return response;
}
 

/**
 * Creates a new instance of rpc server. If an earlier instance is already
 * running then returns the same.
 */
private RPC.Server getRpcServer(OzoneConfiguration conf) throws IOException {
  if (isOmRpcServerRunning) {
    return omRpcServer;
  }

  InetSocketAddress omNodeRpcAddr = OmUtils.getOmAddress(conf);

  final int handlerCount = conf.getInt(OZONE_OM_HANDLER_COUNT_KEY,
      OZONE_OM_HANDLER_COUNT_DEFAULT);
  RPC.setProtocolEngine(configuration, OzoneManagerProtocolPB.class,
      ProtobufRpcEngine.class);
  this.omServerProtocol = new OzoneManagerProtocolServerSideTranslatorPB(
      this, omRatisServer, omClientProtocolMetrics, isRatisEnabled);

  BlockingService omService = newReflectiveBlockingService(omServerProtocol);

  return startRpcServer(configuration, omNodeRpcAddr,
      OzoneManagerProtocolPB.class, omService,
      handlerCount);
}
 

/** Test to verify that InterDatanode RPC timesout as expected when
 *  the server DN does not respond.
 */
@Test(expected=SocketTimeoutException.class)
public void testInterDNProtocolTimeout() throws Throwable {
  final Server server = new TestServer(1, true);
  server.start();

  final InetSocketAddress addr = NetUtils.getConnectAddress(server);
  DatanodeID fakeDnId = DFSTestUtil.getLocalDatanodeID(addr.getPort());
  DatanodeInfo dInfo = new DatanodeInfo(fakeDnId);
  InterDatanodeProtocol proxy = null;

  try {
    proxy = DataNode.createInterDataNodeProtocolProxy(
        dInfo, conf, 500, false);
    proxy.initReplicaRecovery(new RecoveringBlock(
        new ExtendedBlock("bpid", 1), null, 100));
    fail ("Expected SocketTimeoutException exception, but did not get.");
  } finally {
    if (proxy != null) {
      RPC.stopProxy(proxy);
    }
    server.stop();
  }
}
 

/**
 * Get a listing of all files at 'src'.  The Object[] array
 * exists so we can return file attributes (soon to be implemented)
 */
public FileStatus[] getListing(String src) throws IOException {
  if (isPermissionEnabled) {
    if (dir.isDir(src)) {
      checkPathAccess(src, FsAction.READ_EXECUTE);
    }
    else {
      checkTraverse(src);
    }
  }
  if (auditLog.isInfoEnabled()) {
    logAuditEvent(UserGroupInformation.getCurrentUGI(),
                  Server.getRemoteIp(),
                  "listStatus", src, null, null);
  }
  return dir.getListing(src);
}
 

private static Server createMockDatanode(BlockTokenSecretManager sm,
    Token<BlockTokenIdentifier> token, Configuration conf)
    throws IOException, ServiceException {
  ClientDatanodeProtocolPB mockDN = mock(ClientDatanodeProtocolPB.class);

  BlockTokenIdentifier id = sm.createIdentifier();
  id.readFields(new DataInputStream(new ByteArrayInputStream(token
      .getIdentifier())));
  
  doAnswer(new GetLengthAnswer(sm, id)).when(mockDN)
      .getReplicaVisibleLength(any(RpcController.class),
          any(GetReplicaVisibleLengthRequestProto.class));

  RPC.setProtocolEngine(conf, ClientDatanodeProtocolPB.class,
      ProtobufRpcEngine.class);
  BlockingService service = ClientDatanodeProtocolService
      .newReflectiveBlockingService(mockDN);
  return new RPC.Builder(conf).setProtocol(ClientDatanodeProtocolPB.class)
      .setInstance(service).setBindAddress(ADDRESS).setPort(0)
      .setNumHandlers(5).setVerbose(true).setSecretManager(sm).build();
}
 

/**
 * Set owner for an existing file.
 * @throws IOException
 */
public synchronized void setOwner(String src, String username, String group
    ) throws IOException {
  PermissionChecker pc = checkOwner(src);
  if (!pc.isSuper) {
    if (username != null && !pc.user.equals(username)) {
      throw new AccessControlException("Non-super user cannot change owner.");
    }
    if (group != null && !pc.containsGroup(group)) {
      throw new AccessControlException("User does not belong to " + group
          + " .");
    }
  }
  dir.setOwner(src, username, group);
  getEditLog().logSync();
  if (auditLog.isInfoEnabled()) {
    final FileStatus stat = dir.getFileInfo(src);
    logAuditEvent(UserGroupInformation.getCurrentUGI(),
                  Server.getRemoteIp(),
                  "setOwner", src, null, stat);
  }
}
 

@SuppressWarnings("unchecked")
@Override
public KillJobResponse killJob(KillJobRequest request) 
  throws IOException {
  JobId jobId = request.getJobId();
  UserGroupInformation callerUGI = UserGroupInformation.getCurrentUser();
  String message = "Kill job " + jobId + " received from " + callerUGI
      + " at " + Server.getRemoteAddress();
  LOG.info(message);
  verifyAndGetJob(jobId, JobACL.MODIFY_JOB, false);
  appContext.getEventHandler().handle(
      new JobDiagnosticsUpdateEvent(jobId, message));
  appContext.getEventHandler().handle(
      new JobEvent(jobId, JobEventType.JOB_KILL));
  KillJobResponse response = 
    recordFactory.newRecordInstance(KillJobResponse.class);
  return response;
}
 

/**
 * Test {@link NMAuditLogger} with IP set.
 */
@Test  
public void testNMAuditLoggerWithIP() throws Exception {
  Configuration conf = new Configuration();
  // start the IPC server
  Server server = new RPC.Builder(conf).setProtocol(TestProtocol.class)
      .setInstance(new MyTestRPCServer()).setBindAddress("0.0.0.0")
      .setPort(0).setNumHandlers(5).setVerbose(true).build();

  server.start();

  InetSocketAddress addr = NetUtils.getConnectAddress(server);

  // Make a client connection and test the audit log
  TestProtocol proxy = (TestProtocol)RPC.getProxy(TestProtocol.class,
                         TestProtocol.versionID, addr, conf);
  // Start the testcase
  proxy.ping();

  server.stop();
}
 

/**
 * Test the AuditLog format for successful events.
 */
private void testSuccessLogFormatHelper(boolean checkIP, ApplicationId appId,
    ApplicationAttemptId attemptId, ContainerId containerId) {
  String sLog = RMAuditLogger.createSuccessLog(USER, OPERATION, TARGET,
      appId, attemptId, containerId);
  StringBuilder expLog = new StringBuilder();
  expLog.append("USER=test\t");
  if (checkIP) {
    InetAddress ip = Server.getRemoteIp();
    expLog.append(Keys.IP.name() + "=" + ip.getHostAddress() + "\t");
  }
  expLog.append("OPERATION=oper\tTARGET=tgt\tRESULT=SUCCESS");

  if (appId != null) {
    expLog.append("\tAPPID=app_1");
  }
  if (attemptId != null) {
    expLog.append("\tAPPATTEMPTID=app_attempt_1");
  }
  if (containerId != null) {
    expLog.append("\tCONTAINERID=container_1");
  }
  assertEquals(expLog.toString(), sLog);
}
 

@SuppressWarnings("unchecked")
@Override
public KillTaskAttemptResponse killTaskAttempt(
    KillTaskAttemptRequest request) throws IOException {
  TaskAttemptId taskAttemptId = request.getTaskAttemptId();
  UserGroupInformation callerUGI = UserGroupInformation.getCurrentUser();
  String message = "Kill task attempt " + taskAttemptId
      + " received from " + callerUGI + " at "
      + Server.getRemoteAddress();
  LOG.info(message);
  verifyAndGetAttempt(taskAttemptId, JobACL.MODIFY_JOB);
  appContext.getEventHandler().handle(
      new TaskAttemptDiagnosticsUpdateEvent(taskAttemptId, message));
  appContext.getEventHandler().handle(
      new TaskAttemptEvent(taskAttemptId, 
          TaskAttemptEventType.TA_KILL));
  KillTaskAttemptResponse response = 
    recordFactory.newRecordInstance(KillTaskAttemptResponse.class);
  return response;
}
 

@SuppressWarnings("unchecked")
@Override
public KillJobResponse killJob(KillJobRequest request) 
  throws IOException {
  JobId jobId = request.getJobId();
  UserGroupInformation callerUGI = UserGroupInformation.getCurrentUser();
  String message = "Kill job " + jobId + " received from " + callerUGI
      + " at " + Server.getRemoteAddress();
  LOG.info(message);
  verifyAndGetJob(jobId, JobACL.MODIFY_JOB, false);
  appContext.getEventHandler().handle(
      new JobDiagnosticsUpdateEvent(jobId, message));
  appContext.getEventHandler().handle(
      new JobEvent(jobId, JobEventType.JOB_KILL));
  KillJobResponse response = 
    recordFactory.newRecordInstance(KillJobResponse.class);
  return response;
}
 

/**
 * Test the AuditLog format for successful events.
 */
private void testSuccessLogFormatHelper(boolean checkIP, ApplicationId appId,
    ApplicationAttemptId attemptId, ContainerId containerId) {
  String sLog = RMAuditLogger.createSuccessLog(USER, OPERATION, TARGET,
      appId, attemptId, containerId);
  StringBuilder expLog = new StringBuilder();
  expLog.append("USER=test\t");
  if (checkIP) {
    InetAddress ip = Server.getRemoteIp();
    expLog.append(Keys.IP.name() + "=" + ip.getHostAddress() + "\t");
  }
  expLog.append("OPERATION=oper\tTARGET=tgt\tRESULT=SUCCESS");

  if (appId != null) {
    expLog.append("\tAPPID=app_1");
  }
  if (attemptId != null) {
    expLog.append("\tAPPATTEMPTID=app_attempt_1");
  }
  if (containerId != null) {
    expLog.append("\tCONTAINERID=container_1");
  }
  assertEquals(expLog.toString(), sLog);
}
 

Server createServer() {
  Configuration conf = getConfig();
  YarnRPC rpc = YarnRPC.create(conf);
  if (UserGroupInformation.isSecurityEnabled()) {
    secretManager = new LocalizerTokenSecretManager();      
  }
  
  Server server = rpc.getServer(LocalizationProtocol.class, this,
      localizationServerAddress, conf, secretManager, 
      conf.getInt(YarnConfiguration.NM_LOCALIZER_CLIENT_THREAD_COUNT, 
          YarnConfiguration.DEFAULT_NM_LOCALIZER_CLIENT_THREAD_COUNT));
  
  // Enable service authorization?
  if (conf.getBoolean(
      CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHORIZATION, 
      false)) {
    server.refreshServiceAcl(conf, new NMPolicyProvider());
  }
  
  return server;
}
 

/**
 * Get block locations within the specified range.
 * @see ClientProtocol#getBlockLocations(String, long, long)
 */
public LocatedBlocks getBlockLocations(String src, long offset, long length,
    boolean doAccessTime) throws IOException {
  if (offset < 0) {
    throw new IOException("Negative offset is not supported. File: " + src );
  }
  if (length < 0) {
    throw new IOException("Negative length is not supported. File: " + src );
  }
  final LocatedBlocks ret = getBlockLocationsInternal(src, dir.getFileINode(src),
      offset, length, Integer.MAX_VALUE, doAccessTime);  
  if (auditLog.isInfoEnabled()) {
    logAuditEvent(UserGroupInformation.getCurrentUGI(),
                  Server.getRemoteIp(),
                  "open", src, null, null);
  }
  return ret;
}
 

Server createServer() {
  Configuration conf = getConfig();
  YarnRPC rpc = YarnRPC.create(conf);
  if (UserGroupInformation.isSecurityEnabled()) {
    secretManager = new LocalizerTokenSecretManager();      
  }
  
  Server server = rpc.getServer(LocalizationProtocol.class, this,
      localizationServerAddress, conf, secretManager, 
      conf.getInt(YarnConfiguration.NM_LOCALIZER_CLIENT_THREAD_COUNT, 
          YarnConfiguration.DEFAULT_NM_LOCALIZER_CLIENT_THREAD_COUNT));
  
  // Enable service authorization?
  if (conf.getBoolean(
      CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHORIZATION, 
      false)) {
    server.refreshServiceAcl(conf, new NMPolicyProvider());
  }
  
  return server;
}
 

private void testPbServerFactory() {
  InetSocketAddress addr = new InetSocketAddress(0);
  Configuration conf = new Configuration();
  ResourceTracker instance = new ResourceTrackerTestImpl();
  Server server = null;
  try {
    server = 
      RpcServerFactoryPBImpl.get().getServer(
          ResourceTracker.class, instance, addr, conf, null, 1);
    server.start();
  } catch (YarnRuntimeException e) {
    e.printStackTrace();
    Assert.fail("Failed to create server");
  } finally {
    server.stop();
  }
}
 

private void testPbServerFactory() {
  InetSocketAddress addr = new InetSocketAddress(0);
  Configuration conf = new Configuration();
  ApplicationMasterProtocol instance = new AMRMProtocolTestImpl();
  Server server = null;
  try {
    server = 
      RpcServerFactoryPBImpl.get().getServer(
          ApplicationMasterProtocol.class, instance, addr, conf, null, 1);
    server.start();
  } catch (YarnRuntimeException e) {
    e.printStackTrace();
    Assert.fail("Failed to create server");
  } finally {
    if (server != null) {
      server.stop();
    }
  }
}
 

/**
 * Test the AuditLog format for failure events.
 */
private void testFailureLogFormatHelper(boolean checkIP, ApplicationId appId,
    ContainerId containerId) {
  String fLog =
    NMAuditLogger.createFailureLog(USER, OPERATION, TARGET, DESC, appId,
    containerId);
  StringBuilder expLog = new StringBuilder();
  expLog.append("USER=test\t");
  if (checkIP) {
    InetAddress ip = Server.getRemoteIp();
    expLog.append(Keys.IP.name() + "=" + ip.getHostAddress() + "\t");
  }
  expLog.append("OPERATION=oper\tTARGET=tgt\tRESULT=FAILURE\t");
  expLog.append("DESCRIPTION=description of an audit log");

  if (appId != null) {
    expLog.append("\tAPPID=app_1");
  }
  if (containerId != null) {
    expLog.append("\tCONTAINERID=container_1");
  }
  assertEquals(expLog.toString(), fLog);
}
 

/**
 * Mark the block belonging to datanode as corrupt.
 *
 * @param blk Block to be added to CorruptReplicasMap
 * @param dn DatanodeDescriptor which holds the corrupt replica
 */
public void addToCorruptReplicasMap(Block blk, DatanodeDescriptor dn) {
  Collection<DatanodeDescriptor> nodes = getNodes(blk);
  if (nodes == null) {
    nodes = new TreeSet<DatanodeDescriptor>();
    corruptReplicasMap.put(blk, nodes);
  }
  if (!nodes.contains(dn)) {
    nodes.add(dn);
    NameNode.stateChangeLog.info("BLOCK NameSystem.addToCorruptReplicasMap: "+
                                 blk.getBlockName() +
                                 " added as corrupt on " + dn.getName() +
                                 " by " + Server.getRemoteIp());
  } else {
    NameNode.stateChangeLog.info("BLOCK NameSystem.addToCorruptReplicasMap: "+
                                 "duplicate requested for " + 
                                 blk.getBlockName() + " to add as corrupt " +
                                 "on " + dn.getName() +
                                 " by " + Server.getRemoteIp());
  }
}
 

private void testPbServerFactory() {
  InetSocketAddress addr = new InetSocketAddress(0);
  Configuration conf = new Configuration();
  ResourceTracker instance = new ResourceTrackerTestImpl();
  Server server = null;
  try {
    server = 
      RpcServerFactoryPBImpl.get().getServer(
          ResourceTracker.class, instance, addr, conf, null, 1);
    server.start();
  } catch (YarnRuntimeException e) {
    e.printStackTrace();
    Assert.fail("Failed to create server");
  } finally {
    server.stop();
  }
}
 

/**
 * stores the modification and access time for this inode. 
 * The access time is precise upto an hour. The transaction, if needed, is
 * written to the edits log but is not flushed.
 */
public synchronized void setTimes(String src, long mtime, long atime) throws IOException {
  if (!isAccessTimeSupported() && atime != -1) {
    throw new IOException("Access time for hdfs is not configured. " +
                          " Please set dfs.support.accessTime configuration parameter.");
  }
  //
  // The caller needs to have write access to set access & modification times.
  if (isPermissionEnabled) {
    checkPathAccess(src, FsAction.WRITE);
  }
  INodeFile inode = dir.getFileINode(src);
  if (inode != null) {
    dir.setTimes(src, inode, mtime, atime, true);
    if (auditLog.isInfoEnabled()) {
      final FileStatus stat = dir.getFileInfo(src);
      logAuditEvent(UserGroupInformation.getCurrentUGI(),
                    Server.getRemoteIp(),
                    "setTimes", src, null, stat);
    }
  } else {
    throw new FileNotFoundException("File " + src + " does not exist.");
  }
}
 

@Override
public void start() {
  LOG.info("Starting application message server");
  RPC.Builder builder = new RPC.Builder(getConfig());
  builder.setProtocol(ApplicationMessageProtocol.class);
  builder.setInstance(this);
  builder.setBindAddress("0.0.0.0");
  builder.setPort(0);
  Server server;
  try {
    server = builder.build();
  } catch (Exception e) {
    LOG.error("Error starting message server!", e);
    e.printStackTrace();
    return;
  }
  server.start();

  serverAddress = NetUtils.getConnectAddress(server);
  LOG.info("Started application message server at " + serverAddress);
}
 

@Override
public AuditMessage buildAuditMessageForSuccess(
    AuditAction op, Map<String, String> auditMap) {

  return new AuditMessage.Builder()
      .setUser(getRemoteUserName())
      .atIp(Server.getRemoteAddress())
      .forOperation(op)
      .withParams(auditMap)
      .withResult(AuditEventStatus.SUCCESS)
      .build();
}
 

@Test(timeout=4000)
public void testRealUserSetup() throws IOException {
  final Configuration conf = new Configuration();
  conf.setStrings(DefaultImpersonationProvider.getTestProvider().
      getProxySuperuserGroupConfKey(REAL_USER_SHORT_NAME), "group1");
  configureSuperUserIPAddresses(conf, REAL_USER_SHORT_NAME);
  Server server = new RPC.Builder(conf).setProtocol(TestProtocol.class)
      .setInstance(new TestImpl()).setBindAddress(ADDRESS).setPort(0)
      .setNumHandlers(5).setVerbose(true).build();

  refreshConf(conf);
  try {
    server.start();

    UserGroupInformation realUserUgi = UserGroupInformation
        .createRemoteUser(REAL_USER_NAME);
    checkRemoteUgi(server, realUserUgi, conf);
    
    UserGroupInformation proxyUserUgi = UserGroupInformation.createProxyUserForTesting(
        PROXY_USER_NAME, realUserUgi, GROUP_NAMES);
    checkRemoteUgi(server, proxyUserUgi, conf);
  } catch (Exception e) {
    e.printStackTrace();
    Assert.fail();
  } finally {
    server.stop();
    if (proxy != null) {
      RPC.stopProxy(proxy);
    }
  }
}
 

/**
 * Test the AuditLog format for successful events.
 */
private void testSuccessLogFormat(boolean checkIP) {
  // check without the IP
  String sLog = AuditLogger.createSuccessLog(USER, OPERATION, TARGET);
  StringBuilder expLog = new StringBuilder();
  expLog.append("USER=test\t");
  if (checkIP) {
    InetAddress ip = Server.getRemoteIp();
    expLog.append(Keys.IP.name() + "=" + ip.getHostAddress() + "\t");
  }
  expLog.append("OPERATION=oper\tTARGET=tgt\tRESULT=SUCCESS");
  assertEquals(expLog.toString(), sLog);

}
 

/**
 * A helper api to add remote IP address
 */
static void addRemoteIP(StringBuilder b) {
  InetAddress ip = Server.getRemoteIp();
  // ip address can be null for testcases
  if (ip != null) {
    add(Keys.IP, ip.getHostAddress(), b);
  }
}
 

/**
 * Test the AuditLog format for failure events.
 */
private void testFailureLogFormat(boolean checkIP, String perm) {
  String fLog =
    AuditLogger.createFailureLog(USER, OPERATION, perm, TARGET, DESC);
  StringBuilder expLog = new StringBuilder();
  expLog.append("USER=test\t");
  if (checkIP) {
    InetAddress ip = Server.getRemoteIp();
    expLog.append(Keys.IP.name() + "=" + ip.getHostAddress() + "\t");
  }
  expLog.append("OPERATION=oper\tTARGET=tgt\tRESULT=FAILURE\t");
  expLog.append("DESCRIPTION=description of an audit log\t");
  expLog.append("PERMISSIONS=" + perm);
  assertEquals(expLog.toString(), fLog);
}
 

@Override
public Server getServer(Class protocol, Object instance,
    InetSocketAddress addr, Configuration conf,
    SecretManager<? extends TokenIdentifier> secretManager,
    int numHandlers, String portRangeConfig) {
  LOG.debug("Creating a HadoopYarnProtoRpc server for protocol " + protocol + 
      " with " + numHandlers + " handlers");
  
  return RpcFactoryProvider.getServerFactory(conf).getServer(protocol, 
      instance, addr, conf, secretManager, numHandlers, portRangeConfig);

}
 

/**
 * Mark the block belonging to datanode as corrupt.
 *
 * @param blk Block to be added to CorruptReplicasMap
 * @param dn DatanodeDescriptor which holds the corrupt replica
 * @param reason a textual reason (for logging purposes)
 * @param reasonCode the enum representation of the reason
 */
void addToCorruptReplicasMap(Block blk, DatanodeDescriptor dn,
    String reason, Reason reasonCode) {
  Map <DatanodeDescriptor, Reason> nodes = corruptReplicasMap.get(blk);
  if (nodes == null) {
    nodes = new HashMap<DatanodeDescriptor, Reason>();
    corruptReplicasMap.put(blk, nodes);
  }
  
  String reasonText;
  if (reason != null) {
    reasonText = " because " + reason;
  } else {
    reasonText = "";
  }
  
  if (!nodes.keySet().contains(dn)) {
    NameNode.blockStateChangeLog.info(
        "BLOCK NameSystem.addToCorruptReplicasMap: {} added as corrupt on "
            + "{} by {} {}", blk.getBlockName(), dn, Server.getRemoteIp(),
        reasonText);
  } else {
    NameNode.blockStateChangeLog.info(
        "BLOCK NameSystem.addToCorruptReplicasMap: duplicate requested for" +
            " {} to add as corrupt on {} by {} {}", blk.getBlockName(), dn,
            Server.getRemoteIp(), reasonText);
  }
  // Add the node or update the reason.
  nodes.put(dn, reasonCode);
}