Java Code Examples for org.apache.hadoop.yarn.server.api.protocolrecords.RegisterNodeManagerRequest#setHttpPort()

The following examples show how to use org.apache.hadoop.yarn.server.api.protocolrecords.RegisterNodeManagerRequest#setHttpPort() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: NodeManager.java    From hadoop with Apache License 2.0 6 votes vote down vote up 
public NodeManager(String hostName, int containerManagerPort, int httpPort,
    String rackName, Resource capability,
    ResourceManager resourceManager)
    throws IOException, YarnException {
  this.containerManagerAddress = hostName + ":" + containerManagerPort;
  this.nodeHttpAddress = hostName + ":" + httpPort;
  this.rackName = rackName;
  this.resourceTrackerService = resourceManager.getResourceTrackerService();
  this.capability = capability;
  Resources.addTo(available, capability);
  this.nodeId = NodeId.newInstance(hostName, containerManagerPort);
  RegisterNodeManagerRequest request = recordFactory
      .newRecordInstance(RegisterNodeManagerRequest.class);
  request.setHttpPort(httpPort);
  request.setResource(capability);
  request.setNodeId(this.nodeId);
  request.setNMVersion(YarnVersionInfo.getVersion());
  resourceTrackerService.registerNodeManager(request);
  this.resourceManager = resourceManager;
  resourceManager.getResourceScheduler().getNodeReport(this.nodeId);
}
Example 2
Source File: MockNM.java    From hadoop with Apache License 2.0 6 votes vote down vote up 
public RegisterNodeManagerResponse registerNode(
    List<NMContainerStatus> containerReports,
    List<ApplicationId> runningApplications) throws Exception {
  RegisterNodeManagerRequest req = Records.newRecord(
      RegisterNodeManagerRequest.class);
  req.setNodeId(nodeId);
  req.setHttpPort(httpPort);
  Resource resource = BuilderUtils.newResource(memory, vCores, gCores);
  req.setResource(resource);
  req.setContainerStatuses(containerReports);
  req.setNMVersion(version);
  req.setRunningApplications(runningApplications);
  RegisterNodeManagerResponse registrationResponse =
      resourceTracker.registerNodeManager(req);
  this.currentContainerTokenMasterKey =
      registrationResponse.getContainerTokenMasterKey();
  this.currentNMTokenMasterKey = registrationResponse.getNMTokenMasterKey();
  return registrationResponse;    
}
Example 3
Source File: TestResourceTrackerService.java    From hadoop with Apache License 2.0 6 votes vote down vote up 
@Test
public void testNodeRegistrationSuccess() throws Exception {
  writeToHostsFile("host2");
  Configuration conf = new Configuration();
  conf.set(YarnConfiguration.RM_NODES_INCLUDE_FILE_PATH, hostFile
      .getAbsolutePath());
  rm = new MockRM(conf);
  rm.start();

  ResourceTrackerService resourceTrackerService = rm.getResourceTrackerService();
  RegisterNodeManagerRequest req = Records.newRecord(
      RegisterNodeManagerRequest.class);
  NodeId nodeId = NodeId.newInstance("host2", 1234);
  Resource capability = BuilderUtils.newResource(1024, 1, 1);
  req.setResource(capability);
  req.setNodeId(nodeId);
  req.setHttpPort(1234);
  req.setNMVersion(YarnVersionInfo.getVersion());
  // trying to register a invalid node.
  RegisterNodeManagerResponse response = resourceTrackerService.registerNodeManager(req);
  Assert.assertEquals(NodeAction.NORMAL,response.getNodeAction());
}
Example 4
Source File: TestResourceTrackerService.java    From hadoop with Apache License 2.0 6 votes vote down vote up 
@Test
public void testNodeRegistrationFailure() throws Exception {
  writeToHostsFile("host1");
  Configuration conf = new Configuration();
  conf.set(YarnConfiguration.RM_NODES_INCLUDE_FILE_PATH, hostFile
      .getAbsolutePath());
  rm = new MockRM(conf);
  rm.start();
  
  ResourceTrackerService resourceTrackerService = rm.getResourceTrackerService();
  RegisterNodeManagerRequest req = Records.newRecord(
      RegisterNodeManagerRequest.class);
  NodeId nodeId = NodeId.newInstance("host2", 1234);
  req.setNodeId(nodeId);
  req.setHttpPort(1234);
  // trying to register a invalid node.
  RegisterNodeManagerResponse response = resourceTrackerService.registerNodeManager(req);
  Assert.assertEquals(NodeAction.SHUTDOWN,response.getNodeAction());
  Assert
    .assertEquals(
      "Disallowed NodeManager from  host2, Sending SHUTDOWN signal to the NodeManager.",
      response.getDiagnosticsMessage());
}
Example 5
Source File: NodeManager.java    From big-c with Apache License 2.0 6 votes vote down vote up 
public NodeManager(String hostName, int containerManagerPort, int httpPort,
    String rackName, Resource capability,
    ResourceManager resourceManager)
    throws IOException, YarnException {
  this.containerManagerAddress = hostName + ":" + containerManagerPort;
  this.nodeHttpAddress = hostName + ":" + httpPort;
  this.rackName = rackName;
  this.resourceTrackerService = resourceManager.getResourceTrackerService();
  this.capability = capability;
  Resources.addTo(available, capability);
  this.nodeId = NodeId.newInstance(hostName, containerManagerPort);
  RegisterNodeManagerRequest request = recordFactory
      .newRecordInstance(RegisterNodeManagerRequest.class);
  request.setHttpPort(httpPort);
  request.setResource(capability);
  request.setNodeId(this.nodeId);
  request.setNMVersion(YarnVersionInfo.getVersion());
  resourceTrackerService.registerNodeManager(request);
  this.resourceManager = resourceManager;
  resourceManager.getResourceScheduler().getNodeReport(this.nodeId);
}
Example 6
Source File: MockNM.java    From big-c with Apache License 2.0 6 votes vote down vote up 
public RegisterNodeManagerResponse registerNode(
    List<NMContainerStatus> containerReports,
    List<ApplicationId> runningApplications) throws Exception {
  RegisterNodeManagerRequest req = Records.newRecord(
      RegisterNodeManagerRequest.class);
  req.setNodeId(nodeId);
  req.setHttpPort(httpPort);
  Resource resource = BuilderUtils.newResource(memory, vCores);
  req.setResource(resource);
  req.setContainerStatuses(containerReports);
  req.setNMVersion(version);
  req.setRunningApplications(runningApplications);
  RegisterNodeManagerResponse registrationResponse =
      resourceTracker.registerNodeManager(req);
  this.currentContainerTokenMasterKey =
      registrationResponse.getContainerTokenMasterKey();
  this.currentNMTokenMasterKey = registrationResponse.getNMTokenMasterKey();
  return registrationResponse;    
}
Example 7
Source File: TestResourceTrackerService.java    From big-c with Apache License 2.0 6 votes vote down vote up 
@Test
public void testNodeRegistrationSuccess() throws Exception {
  writeToHostsFile("host2");
  Configuration conf = new Configuration();
  conf.set(YarnConfiguration.RM_NODES_INCLUDE_FILE_PATH, hostFile
      .getAbsolutePath());
  rm = new MockRM(conf);
  rm.start();

  ResourceTrackerService resourceTrackerService = rm.getResourceTrackerService();
  RegisterNodeManagerRequest req = Records.newRecord(
      RegisterNodeManagerRequest.class);
  NodeId nodeId = NodeId.newInstance("host2", 1234);
  Resource capability = BuilderUtils.newResource(1024, 1);
  req.setResource(capability);
  req.setNodeId(nodeId);
  req.setHttpPort(1234);
  req.setNMVersion(YarnVersionInfo.getVersion());
  // trying to register a invalid node.
  RegisterNodeManagerResponse response = resourceTrackerService.registerNodeManager(req);
  Assert.assertEquals(NodeAction.NORMAL,response.getNodeAction());
}
Example 8
Source File: TestResourceTrackerService.java    From big-c with Apache License 2.0 6 votes vote down vote up 
@Test
public void testNodeRegistrationFailure() throws Exception {
  writeToHostsFile("host1");
  Configuration conf = new Configuration();
  conf.set(YarnConfiguration.RM_NODES_INCLUDE_FILE_PATH, hostFile
      .getAbsolutePath());
  rm = new MockRM(conf);
  rm.start();
  
  ResourceTrackerService resourceTrackerService = rm.getResourceTrackerService();
  RegisterNodeManagerRequest req = Records.newRecord(
      RegisterNodeManagerRequest.class);
  NodeId nodeId = NodeId.newInstance("host2", 1234);
  req.setNodeId(nodeId);
  req.setHttpPort(1234);
  // trying to register a invalid node.
  RegisterNodeManagerResponse response = resourceTrackerService.registerNodeManager(req);
  Assert.assertEquals(NodeAction.SHUTDOWN,response.getNodeAction());
  Assert
    .assertEquals(
      "Disallowed NodeManager from  host2, Sending SHUTDOWN signal to the NodeManager.",
      response.getDiagnosticsMessage());
}
Example 9
Source File: TestNMReconnect.java    From hadoop with Apache License 2.0 5 votes vote down vote up 
@Test
public void testReconnect() throws Exception {
  String hostname1 = "localhost1";
  Resource capability = BuilderUtils.newResource(1024, 1, 1);

  RegisterNodeManagerRequest request1 = recordFactory
      .newRecordInstance(RegisterNodeManagerRequest.class);
  NodeId nodeId1 = NodeId.newInstance(hostname1, 0);
  request1.setNodeId(nodeId1);
  request1.setHttpPort(0);
  request1.setResource(capability);
  resourceTrackerService.registerNodeManager(request1);

  Assert.assertEquals(RMNodeEventType.STARTED, rmNodeEvents.get(0).getType());

  rmNodeEvents.clear();
  resourceTrackerService.registerNodeManager(request1);
  Assert.assertEquals(RMNodeEventType.RECONNECTED,
      rmNodeEvents.get(0).getType());

  rmNodeEvents.clear();
  resourceTrackerService.registerNodeManager(request1);
  capability = BuilderUtils.newResource(1024, 2, 2);
  request1.setResource(capability);
  Assert.assertEquals(RMNodeEventType.RECONNECTED,
      rmNodeEvents.get(0).getType());
}
Example 10
Source File: TestResourceTrackerService.java    From hadoop with Apache License 2.0 5 votes vote down vote up 
@Test
public void testNodeRegistrationVersionLessThanRM() throws Exception {
  writeToHostsFile("host2");
  Configuration conf = new Configuration();
  conf.set(YarnConfiguration.RM_NODES_INCLUDE_FILE_PATH, hostFile
      .getAbsolutePath());
  conf.set(YarnConfiguration.RM_NODEMANAGER_MINIMUM_VERSION,"EqualToRM" );
  rm = new MockRM(conf);
  rm.start();
  String nmVersion = "1.9.9";

  ResourceTrackerService resourceTrackerService = rm.getResourceTrackerService();
  RegisterNodeManagerRequest req = Records.newRecord(
      RegisterNodeManagerRequest.class);
  NodeId nodeId = NodeId.newInstance("host2", 1234);
  Resource capability = BuilderUtils.newResource(1024, 1, 1);
  req.setResource(capability);
  req.setNodeId(nodeId);
  req.setHttpPort(1234);
  req.setNMVersion(nmVersion);
  // trying to register a invalid node.
  RegisterNodeManagerResponse response = resourceTrackerService.registerNodeManager(req);
  Assert.assertEquals(NodeAction.SHUTDOWN,response.getNodeAction());
  Assert.assertTrue("Diagnostic message did not contain: 'Disallowed NodeManager " +
      "Version "+ nmVersion + ", is less than the minimum version'",
      response.getDiagnosticsMessage().contains("Disallowed NodeManager Version " +
          nmVersion + ", is less than the minimum version "));

}
Example 11
Source File: TestNMReconnect.java    From big-c with Apache License 2.0 5 votes vote down vote up 
@Test
public void testReconnect() throws Exception {
  String hostname1 = "localhost1";
  Resource capability = BuilderUtils.newResource(1024, 1);

  RegisterNodeManagerRequest request1 = recordFactory
      .newRecordInstance(RegisterNodeManagerRequest.class);
  NodeId nodeId1 = NodeId.newInstance(hostname1, 0);
  request1.setNodeId(nodeId1);
  request1.setHttpPort(0);
  request1.setResource(capability);
  resourceTrackerService.registerNodeManager(request1);

  Assert.assertEquals(RMNodeEventType.STARTED, rmNodeEvents.get(0).getType());

  rmNodeEvents.clear();
  resourceTrackerService.registerNodeManager(request1);
  Assert.assertEquals(RMNodeEventType.RECONNECTED,
      rmNodeEvents.get(0).getType());

  rmNodeEvents.clear();
  resourceTrackerService.registerNodeManager(request1);
  capability = BuilderUtils.newResource(1024, 2);
  request1.setResource(capability);
  Assert.assertEquals(RMNodeEventType.RECONNECTED,
      rmNodeEvents.get(0).getType());
}
Example 12
Source File: TestResourceTrackerService.java    From big-c with Apache License 2.0 5 votes vote down vote up 
@Test
public void testNodeRegistrationVersionLessThanRM() throws Exception {
  writeToHostsFile("host2");
  Configuration conf = new Configuration();
  conf.set(YarnConfiguration.RM_NODES_INCLUDE_FILE_PATH, hostFile
      .getAbsolutePath());
  conf.set(YarnConfiguration.RM_NODEMANAGER_MINIMUM_VERSION,"EqualToRM" );
  rm = new MockRM(conf);
  rm.start();
  String nmVersion = "1.9.9";

  ResourceTrackerService resourceTrackerService = rm.getResourceTrackerService();
  RegisterNodeManagerRequest req = Records.newRecord(
      RegisterNodeManagerRequest.class);
  NodeId nodeId = NodeId.newInstance("host2", 1234);
  Resource capability = BuilderUtils.newResource(1024, 1);
  req.setResource(capability);
  req.setNodeId(nodeId);
  req.setHttpPort(1234);
  req.setNMVersion(nmVersion);
  // trying to register a invalid node.
  RegisterNodeManagerResponse response = resourceTrackerService.registerNodeManager(req);
  Assert.assertEquals(NodeAction.SHUTDOWN,response.getNodeAction());
  Assert.assertTrue("Diagnostic message did not contain: 'Disallowed NodeManager " +
      "Version "+ nmVersion + ", is less than the minimum version'",
      response.getDiagnosticsMessage().contains("Disallowed NodeManager Version " +
          nmVersion + ", is less than the minimum version "));

}
Example 13
Source File: TestRMNMRPCResponseId.java    From hadoop with Apache License 2.0 4 votes vote down vote up 
@Test
public void testRPCResponseId() throws IOException, YarnException {
  String node = "localhost";
  Resource capability = BuilderUtils.newResource(1024, 1, 1);
  RegisterNodeManagerRequest request = recordFactory.newRecordInstance(RegisterNodeManagerRequest.class);
  nodeId = NodeId.newInstance(node, 1234);
  request.setNodeId(nodeId);
  request.setHttpPort(0);
  request.setResource(capability);

  RegisterNodeManagerRequest request1 = recordFactory
      .newRecordInstance(RegisterNodeManagerRequest.class);
  request1.setNodeId(nodeId);
  request1.setHttpPort(0);
  request1.setResource(capability);
  resourceTrackerService.registerNodeManager(request1);

  org.apache.hadoop.yarn.server.api.records.NodeStatus nodeStatus = recordFactory.
    newRecordInstance(org.apache.hadoop.yarn.server.api.records.NodeStatus.class);
  nodeStatus.setNodeId(nodeId);
  NodeHealthStatus nodeHealthStatus = recordFactory.newRecordInstance(NodeHealthStatus.class);
  nodeHealthStatus.setIsNodeHealthy(true);
  nodeStatus.setNodeHealthStatus(nodeHealthStatus);
  NodeHeartbeatRequest nodeHeartBeatRequest = recordFactory
      .newRecordInstance(NodeHeartbeatRequest.class);
  nodeHeartBeatRequest.setNodeStatus(nodeStatus);

  nodeStatus.setResponseId(0);
  NodeHeartbeatResponse response = resourceTrackerService.nodeHeartbeat(
      nodeHeartBeatRequest);
  Assert.assertTrue(response.getResponseId() == 1);

  nodeStatus.setResponseId(response.getResponseId());
  response = resourceTrackerService.nodeHeartbeat(nodeHeartBeatRequest);
  Assert.assertTrue(response.getResponseId() == 2);   

  /* try calling with less response id */
  response = resourceTrackerService.nodeHeartbeat(nodeHeartBeatRequest);
  Assert.assertTrue(response.getResponseId() == 2);

  nodeStatus.setResponseId(0);
  response = resourceTrackerService.nodeHeartbeat(nodeHeartBeatRequest);
  Assert.assertTrue(NodeAction.RESYNC.equals(response.getNodeAction()));
  Assert.assertEquals("Too far behind rm response id:2 nm response id:0",
    response.getDiagnosticsMessage());
}
Example 14
Source File: TestNMExpiry.java    From hadoop with Apache License 2.0 4 votes vote down vote up 
@Test
public void testNMExpiry() throws Exception {
  String hostname1 = "localhost1";
  String hostname2 = "localhost2";
  String hostname3 = "localhost3";
  Resource capability = BuilderUtils.newResource(1024, 1, 1);

  RegisterNodeManagerRequest request1 = recordFactory
      .newRecordInstance(RegisterNodeManagerRequest.class);
  NodeId nodeId1 = NodeId.newInstance(hostname1, 0);
  request1.setNodeId(nodeId1);
  request1.setHttpPort(0);
  request1.setResource(capability);
  resourceTrackerService.registerNodeManager(request1);

  RegisterNodeManagerRequest request2 = recordFactory
      .newRecordInstance(RegisterNodeManagerRequest.class);
  NodeId nodeId2 = NodeId.newInstance(hostname2, 0);
  request2.setNodeId(nodeId2);
  request2.setHttpPort(0);
  request2.setResource(capability);
  resourceTrackerService.registerNodeManager(request2);
  
  int waitCount = 0;
  while(ClusterMetrics.getMetrics().getNumLostNMs()!=2 && waitCount ++<20){
    synchronized (this) {
      wait(100);
    }
  }
  Assert.assertEquals(2, ClusterMetrics.getMetrics().getNumLostNMs());

  request3 = recordFactory
      .newRecordInstance(RegisterNodeManagerRequest.class);
  NodeId nodeId3 = NodeId.newInstance(hostname3, 0);
  request3.setNodeId(nodeId3);
  request3.setHttpPort(0);
  request3.setResource(capability);
  resourceTrackerService
      .registerNodeManager(request3);

  /* test to see if hostanme 3 does not expire */
  stopT = false;
  new ThirdNodeHeartBeatThread().start();
  Assert.assertEquals(2,ClusterMetrics.getMetrics().getNumLostNMs());
  stopT = true;
}
Example 15
Source File: TestRMNMRPCResponseId.java    From big-c with Apache License 2.0 4 votes vote down vote up 
@Test
public void testRPCResponseId() throws IOException, YarnException {
  String node = "localhost";
  Resource capability = BuilderUtils.newResource(1024, 1);
  RegisterNodeManagerRequest request = recordFactory.newRecordInstance(RegisterNodeManagerRequest.class);
  nodeId = NodeId.newInstance(node, 1234);
  request.setNodeId(nodeId);
  request.setHttpPort(0);
  request.setResource(capability);

  RegisterNodeManagerRequest request1 = recordFactory
      .newRecordInstance(RegisterNodeManagerRequest.class);
  request1.setNodeId(nodeId);
  request1.setHttpPort(0);
  request1.setResource(capability);
  resourceTrackerService.registerNodeManager(request1);

  org.apache.hadoop.yarn.server.api.records.NodeStatus nodeStatus = recordFactory.
    newRecordInstance(org.apache.hadoop.yarn.server.api.records.NodeStatus.class);
  nodeStatus.setNodeId(nodeId);
  NodeHealthStatus nodeHealthStatus = recordFactory.newRecordInstance(NodeHealthStatus.class);
  nodeHealthStatus.setIsNodeHealthy(true);
  nodeStatus.setNodeHealthStatus(nodeHealthStatus);
  NodeHeartbeatRequest nodeHeartBeatRequest = recordFactory
      .newRecordInstance(NodeHeartbeatRequest.class);
  nodeHeartBeatRequest.setNodeStatus(nodeStatus);

  nodeStatus.setResponseId(0);
  NodeHeartbeatResponse response = resourceTrackerService.nodeHeartbeat(
      nodeHeartBeatRequest);
  Assert.assertTrue(response.getResponseId() == 1);

  nodeStatus.setResponseId(response.getResponseId());
  response = resourceTrackerService.nodeHeartbeat(nodeHeartBeatRequest);
  Assert.assertTrue(response.getResponseId() == 2);   

  /* try calling with less response id */
  response = resourceTrackerService.nodeHeartbeat(nodeHeartBeatRequest);
  Assert.assertTrue(response.getResponseId() == 2);

  nodeStatus.setResponseId(0);
  response = resourceTrackerService.nodeHeartbeat(nodeHeartBeatRequest);
  Assert.assertTrue(NodeAction.RESYNC.equals(response.getNodeAction()));
  Assert.assertEquals("Too far behind rm response id:2 nm response id:0",
    response.getDiagnosticsMessage());
}
Example 16
Source File: TestNMExpiry.java    From big-c with Apache License 2.0 4 votes vote down vote up 
@Test
public void testNMExpiry() throws Exception {
  String hostname1 = "localhost1";
  String hostname2 = "localhost2";
  String hostname3 = "localhost3";
  Resource capability = BuilderUtils.newResource(1024, 1);

  RegisterNodeManagerRequest request1 = recordFactory
      .newRecordInstance(RegisterNodeManagerRequest.class);
  NodeId nodeId1 = NodeId.newInstance(hostname1, 0);
  request1.setNodeId(nodeId1);
  request1.setHttpPort(0);
  request1.setResource(capability);
  resourceTrackerService.registerNodeManager(request1);

  RegisterNodeManagerRequest request2 = recordFactory
      .newRecordInstance(RegisterNodeManagerRequest.class);
  NodeId nodeId2 = NodeId.newInstance(hostname2, 0);
  request2.setNodeId(nodeId2);
  request2.setHttpPort(0);
  request2.setResource(capability);
  resourceTrackerService.registerNodeManager(request2);
  
  int waitCount = 0;
  while(ClusterMetrics.getMetrics().getNumLostNMs()!=2 && waitCount ++<20){
    synchronized (this) {
      wait(100);
    }
  }
  Assert.assertEquals(2, ClusterMetrics.getMetrics().getNumLostNMs());

  request3 = recordFactory
      .newRecordInstance(RegisterNodeManagerRequest.class);
  NodeId nodeId3 = NodeId.newInstance(hostname3, 0);
  request3.setNodeId(nodeId3);
  request3.setHttpPort(0);
  request3.setResource(capability);
  resourceTrackerService
      .registerNodeManager(request3);

  /* test to see if hostanme 3 does not expire */
  stopT = false;
  new ThirdNodeHeartBeatThread().start();
  Assert.assertEquals(2,ClusterMetrics.getMetrics().getNumLostNMs());
  stopT = true;
}