org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodesRequest Java Examples

The following examples show how to use org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodesRequest. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: ClientRMService.java    From hadoop with Apache License 2.0 6 votes vote down vote up 
@Override
public GetClusterNodesResponse getClusterNodes(GetClusterNodesRequest request)
    throws YarnException {
  GetClusterNodesResponse response = 
    recordFactory.newRecordInstance(GetClusterNodesResponse.class);
  EnumSet<NodeState> nodeStates = request.getNodeStates();
  if (nodeStates == null || nodeStates.isEmpty()) {
    nodeStates = EnumSet.allOf(NodeState.class);
  }
  Collection<RMNode> nodes = RMServerUtils.queryRMNodes(rmContext,
      nodeStates);
  
  List<NodeReport> nodeReports = new ArrayList<NodeReport>(nodes.size());
  for (RMNode nodeInfo : nodes) {
    nodeReports.add(createNodeReports(nodeInfo));
  }
  response.setNodeReports(nodeReports);
  return response;
}
Example #2
Source File: ClientRMService.java    From big-c with Apache License 2.0 6 votes vote down vote up 
@Override
public GetClusterNodesResponse getClusterNodes(GetClusterNodesRequest request)
    throws YarnException {
  GetClusterNodesResponse response = 
    recordFactory.newRecordInstance(GetClusterNodesResponse.class);
  EnumSet<NodeState> nodeStates = request.getNodeStates();
  if (nodeStates == null || nodeStates.isEmpty()) {
    nodeStates = EnumSet.allOf(NodeState.class);
  }
  Collection<RMNode> nodes = RMServerUtils.queryRMNodes(rmContext,
      nodeStates);
  
  List<NodeReport> nodeReports = new ArrayList<NodeReport>(nodes.size());
  for (RMNode nodeInfo : nodes) {
    nodeReports.add(createNodeReports(nodeInfo));
  }
  response.setNodeReports(nodeReports);
  return response;
}
Example #3
Source File: YarnClientImpl.java    From hadoop with Apache License 2.0 5 votes vote down vote up 
@Override
public List<NodeReport> getNodeReports(NodeState... states) throws YarnException,
    IOException {
  EnumSet<NodeState> statesSet = (states.length == 0) ?
      EnumSet.allOf(NodeState.class) : EnumSet.noneOf(NodeState.class);
  for (NodeState state : states) {
    statesSet.add(state);
  }
  GetClusterNodesRequest request = GetClusterNodesRequest
      .newInstance(statesSet);
  GetClusterNodesResponse response = rmClient.getClusterNodes(request);
  return response.getNodeReports();
}
Example #4
Source File: ProtocolHATestBase.java    From hadoop with Apache License 2.0 5 votes vote down vote up 
@Override
public GetClusterNodesResponse getClusterNodes(
    GetClusterNodesRequest request)
    throws YarnException {
  resetStartFailoverFlag(true);

  // make sure failover has been triggered
  Assert.assertTrue(waittingForFailOver());

  // create GetClusterNodesResponse with fake ClusterNodeLists
  GetClusterNodesResponse response =
      GetClusterNodesResponse.newInstance(createFakeNodeReports());
  return response;
}
Example #5
Source File: ApplicationClientProtocolPBClientImpl.java    From big-c with Apache License 2.0 5 votes vote down vote up 
@Override
public GetClusterNodesResponse
    getClusterNodes(GetClusterNodesRequest request)
        throws YarnException, IOException {
  GetClusterNodesRequestProto requestProto =
      ((GetClusterNodesRequestPBImpl) request).getProto();
  try {
    return new GetClusterNodesResponsePBImpl(proxy.getClusterNodes(null,
      requestProto));
  } catch (ServiceException e) {
    RPCUtil.unwrapAndThrowException(e);
    return null;
  }
}
Example #6
Source File: ApplicationClientProtocolPBClientImpl.java    From hadoop with Apache License 2.0 5 votes vote down vote up 
@Override
public GetClusterNodesResponse
    getClusterNodes(GetClusterNodesRequest request)
        throws YarnException, IOException {
  GetClusterNodesRequestProto requestProto =
      ((GetClusterNodesRequestPBImpl) request).getProto();
  try {
    return new GetClusterNodesResponsePBImpl(proxy.getClusterNodes(null,
      requestProto));
  } catch (ServiceException e) {
    RPCUtil.unwrapAndThrowException(e);
    return null;
  }
}
Example #7
Source File: YarnClientImpl.java    From big-c with Apache License 2.0 5 votes vote down vote up 
@Override
public List<NodeReport> getNodeReports(NodeState... states) throws YarnException,
    IOException {
  EnumSet<NodeState> statesSet = (states.length == 0) ?
      EnumSet.allOf(NodeState.class) : EnumSet.noneOf(NodeState.class);
  for (NodeState state : states) {
    statesSet.add(state);
  }
  GetClusterNodesRequest request = GetClusterNodesRequest
      .newInstance(statesSet);
  GetClusterNodesResponse response = rmClient.getClusterNodes(request);
  return response.getNodeReports();
}
Example #8
Source File: ProtocolHATestBase.java    From big-c with Apache License 2.0 5 votes vote down vote up 
@Override
public GetClusterNodesResponse getClusterNodes(
    GetClusterNodesRequest request)
    throws YarnException {
  resetStartFailoverFlag(true);

  // make sure failover has been triggered
  Assert.assertTrue(waittingForFailOver());

  // create GetClusterNodesResponse with fake ClusterNodeLists
  GetClusterNodesResponse response =
      GetClusterNodesResponse.newInstance(createFakeNodeReports());
  return response;
}
Example #9
Source File: StramMiniClusterTest.java    From attic-apex-core with Apache License 2.0 5 votes vote down vote up 
private void checkNodeState() throws YarnException
{
  GetClusterNodesRequest request = Records.newRecord(GetClusterNodesRequest.class);
  ClientRMService clientRMService = yarnCluster.getResourceManager().getClientRMService();
  GetClusterNodesResponse response = clientRMService.getClusterNodes(request);
  List<NodeReport> nodeReports = response.getNodeReports();
  LOG.info("{}", nodeReports);

  for (NodeReport nr: nodeReports) {
    if (!nr.getNodeState().isUnusable()) {
      return;
    }
  }
  fail("Yarn Mini cluster should have at least one usable node.");
}
Example #10
Source File: TestClientRMService.java    From hadoop with Apache License 2.0 4 votes vote down vote up 
@Test
public void testGetClusterNodes() throws Exception {
  MockRM rm = new MockRM() {
    protected ClientRMService createClientRMService() {
      return new ClientRMService(this.rmContext, scheduler,
        this.rmAppManager, this.applicationACLsManager, this.queueACLsManager,
        this.getRMContext().getRMDelegationTokenSecretManager());
    };
  };
  rm.start();
  RMNodeLabelsManager labelsMgr = rm.getRMContext().getNodeLabelManager();
  labelsMgr.addToCluserNodeLabels(ImmutableSet.of("x", "y"));

  // Add a healthy node with label = x
  MockNM node = rm.registerNode("host1:1234", 1024);
  Map<NodeId, Set<String>> map = new HashMap<NodeId, Set<String>>();
  map.put(node.getNodeId(), ImmutableSet.of("x"));
  labelsMgr.replaceLabelsOnNode(map);
  rm.sendNodeStarted(node);
  node.nodeHeartbeat(true);
  
  // Add and lose a node with label = y
  MockNM lostNode = rm.registerNode("host2:1235", 1024);
  rm.sendNodeStarted(lostNode);
  lostNode.nodeHeartbeat(true);
  rm.NMwaitForState(lostNode.getNodeId(), NodeState.RUNNING);
  rm.sendNodeLost(lostNode);

  // Create a client.
  Configuration conf = new Configuration();
  YarnRPC rpc = YarnRPC.create(conf);
  InetSocketAddress rmAddress = rm.getClientRMService().getBindAddress();
  LOG.info("Connecting to ResourceManager at " + rmAddress);
  ApplicationClientProtocol client =
      (ApplicationClientProtocol) rpc
        .getProxy(ApplicationClientProtocol.class, rmAddress, conf);

  // Make call
  GetClusterNodesRequest request =
      GetClusterNodesRequest.newInstance(EnumSet.of(NodeState.RUNNING));
  List<NodeReport> nodeReports =
      client.getClusterNodes(request).getNodeReports();
  Assert.assertEquals(1, nodeReports.size());
  Assert.assertNotSame("Node is expected to be healthy!", NodeState.UNHEALTHY,
      nodeReports.get(0).getNodeState());
  
  // Check node's label = x
  Assert.assertTrue(nodeReports.get(0).getNodeLabels().contains("x"));

  // Now make the node unhealthy.
  node.nodeHeartbeat(false);

  // Call again
  nodeReports = client.getClusterNodes(request).getNodeReports();
  Assert.assertEquals("Unhealthy nodes should not show up by default", 0,
      nodeReports.size());
  
  // Change label of host1 to y
  map = new HashMap<NodeId, Set<String>>();
  map.put(node.getNodeId(), ImmutableSet.of("y"));
  labelsMgr.replaceLabelsOnNode(map);
  
  // Now query for UNHEALTHY nodes
  request = GetClusterNodesRequest.newInstance(EnumSet.of(NodeState.UNHEALTHY));
  nodeReports = client.getClusterNodes(request).getNodeReports();
  Assert.assertEquals(1, nodeReports.size());
  Assert.assertEquals("Node is expected to be unhealthy!", NodeState.UNHEALTHY,
      nodeReports.get(0).getNodeState());
  
  Assert.assertTrue(nodeReports.get(0).getNodeLabels().contains("y"));
  
  // Remove labels of host1
  map = new HashMap<NodeId, Set<String>>();
  map.put(node.getNodeId(), ImmutableSet.of("y"));
  labelsMgr.removeLabelsFromNode(map);
  
  // Query all states should return all nodes
  rm.registerNode("host3:1236", 1024);
  request = GetClusterNodesRequest.newInstance(EnumSet.allOf(NodeState.class));
  nodeReports = client.getClusterNodes(request).getNodeReports();
  Assert.assertEquals(3, nodeReports.size());
  
  // All host1-3's label should be empty (instead of null)
  for (NodeReport report : nodeReports) {
    Assert.assertTrue(report.getNodeLabels() != null
        && report.getNodeLabels().isEmpty());
  }
  
  rpc.stopProxy(client, conf);
  rm.close();
}
Example #11
Source File: TestYARNRunner.java    From hadoop with Apache License 2.0 4 votes vote down vote up 
@Test(timeout=20000)
public void testResourceMgrDelegate() throws Exception {
  /* we not want a mock of resource mgr delegate */
  final ApplicationClientProtocol clientRMProtocol = mock(ApplicationClientProtocol.class);
  ResourceMgrDelegate delegate = new ResourceMgrDelegate(conf) {
    @Override
    protected void serviceStart() throws Exception {
      assertTrue(this.client instanceof YarnClientImpl);
      ((YarnClientImpl) this.client).setRMClient(clientRMProtocol);
    }
  };
  /* make sure kill calls finish application master */
  when(clientRMProtocol.forceKillApplication(any(KillApplicationRequest.class)))
  .thenReturn(KillApplicationResponse.newInstance(true));
  delegate.killApplication(appId);
  verify(clientRMProtocol).forceKillApplication(any(KillApplicationRequest.class));

  /* make sure getalljobs calls get all applications */
  when(clientRMProtocol.getApplications(any(GetApplicationsRequest.class))).
  thenReturn(recordFactory.newRecordInstance(GetApplicationsResponse.class));
  delegate.getAllJobs();
  verify(clientRMProtocol).getApplications(any(GetApplicationsRequest.class));

  /* make sure getapplication report is called */
  when(clientRMProtocol.getApplicationReport(any(GetApplicationReportRequest.class)))
  .thenReturn(recordFactory.newRecordInstance(GetApplicationReportResponse.class));
  delegate.getApplicationReport(appId);
  verify(clientRMProtocol).getApplicationReport(any(GetApplicationReportRequest.class));

  /* make sure metrics is called */
  GetClusterMetricsResponse clusterMetricsResponse = recordFactory.newRecordInstance
      (GetClusterMetricsResponse.class);
  clusterMetricsResponse.setClusterMetrics(recordFactory.newRecordInstance(
      YarnClusterMetrics.class));
  when(clientRMProtocol.getClusterMetrics(any(GetClusterMetricsRequest.class)))
  .thenReturn(clusterMetricsResponse);
  delegate.getClusterMetrics();
  verify(clientRMProtocol).getClusterMetrics(any(GetClusterMetricsRequest.class));

  when(clientRMProtocol.getClusterNodes(any(GetClusterNodesRequest.class))).
  thenReturn(recordFactory.newRecordInstance(GetClusterNodesResponse.class));
  delegate.getActiveTrackers();
  verify(clientRMProtocol).getClusterNodes(any(GetClusterNodesRequest.class));
  
  GetNewApplicationResponse newAppResponse = recordFactory.newRecordInstance(
      GetNewApplicationResponse.class);
  newAppResponse.setApplicationId(appId);
  when(clientRMProtocol.getNewApplication(any(GetNewApplicationRequest.class))).
  thenReturn(newAppResponse);
  delegate.getNewJobID();
  verify(clientRMProtocol).getNewApplication(any(GetNewApplicationRequest.class));
  
  GetQueueInfoResponse queueInfoResponse = recordFactory.newRecordInstance(
      GetQueueInfoResponse.class);
  queueInfoResponse.setQueueInfo(recordFactory.newRecordInstance(QueueInfo.class));
  when(clientRMProtocol.getQueueInfo(any(GetQueueInfoRequest.class))).
  thenReturn(queueInfoResponse);
  delegate.getQueues();
  verify(clientRMProtocol).getQueueInfo(any(GetQueueInfoRequest.class));

  GetQueueUserAclsInfoResponse aclResponse = recordFactory.newRecordInstance(
      GetQueueUserAclsInfoResponse.class);
  when(clientRMProtocol.getQueueUserAcls(any(GetQueueUserAclsInfoRequest.class)))
  .thenReturn(aclResponse);
  delegate.getQueueAclsForCurrentUser();
  verify(clientRMProtocol).getQueueUserAcls(any(GetQueueUserAclsInfoRequest.class));
}
Example #12
Source File: TestClientRedirect.java    From hadoop with Apache License 2.0 4 votes vote down vote up 
@Override
public GetClusterNodesResponse getClusterNodes(
    GetClusterNodesRequest request) throws IOException {
  return null;
}
Example #13
Source File: TestClientRMService.java    From big-c with Apache License 2.0 4 votes vote down vote up 
@Test
public void testGetClusterNodes() throws Exception {
  MockRM rm = new MockRM() {
    protected ClientRMService createClientRMService() {
      return new ClientRMService(this.rmContext, scheduler,
        this.rmAppManager, this.applicationACLsManager, this.queueACLsManager,
        this.getRMContext().getRMDelegationTokenSecretManager());
    };
  };
  rm.start();
  RMNodeLabelsManager labelsMgr = rm.getRMContext().getNodeLabelManager();
  labelsMgr.addToCluserNodeLabels(ImmutableSet.of("x", "y"));

  // Add a healthy node with label = x
  MockNM node = rm.registerNode("host1:1234", 1024);
  Map<NodeId, Set<String>> map = new HashMap<NodeId, Set<String>>();
  map.put(node.getNodeId(), ImmutableSet.of("x"));
  labelsMgr.replaceLabelsOnNode(map);
  rm.sendNodeStarted(node);
  node.nodeHeartbeat(true);
  
  // Add and lose a node with label = y
  MockNM lostNode = rm.registerNode("host2:1235", 1024);
  rm.sendNodeStarted(lostNode);
  lostNode.nodeHeartbeat(true);
  rm.NMwaitForState(lostNode.getNodeId(), NodeState.RUNNING);
  rm.sendNodeLost(lostNode);

  // Create a client.
  Configuration conf = new Configuration();
  YarnRPC rpc = YarnRPC.create(conf);
  InetSocketAddress rmAddress = rm.getClientRMService().getBindAddress();
  LOG.info("Connecting to ResourceManager at " + rmAddress);
  ApplicationClientProtocol client =
      (ApplicationClientProtocol) rpc
        .getProxy(ApplicationClientProtocol.class, rmAddress, conf);

  // Make call
  GetClusterNodesRequest request =
      GetClusterNodesRequest.newInstance(EnumSet.of(NodeState.RUNNING));
  List<NodeReport> nodeReports =
      client.getClusterNodes(request).getNodeReports();
  Assert.assertEquals(1, nodeReports.size());
  Assert.assertNotSame("Node is expected to be healthy!", NodeState.UNHEALTHY,
      nodeReports.get(0).getNodeState());
  
  // Check node's label = x
  Assert.assertTrue(nodeReports.get(0).getNodeLabels().contains("x"));

  // Now make the node unhealthy.
  node.nodeHeartbeat(false);

  // Call again
  nodeReports = client.getClusterNodes(request).getNodeReports();
  Assert.assertEquals("Unhealthy nodes should not show up by default", 0,
      nodeReports.size());
  
  // Change label of host1 to y
  map = new HashMap<NodeId, Set<String>>();
  map.put(node.getNodeId(), ImmutableSet.of("y"));
  labelsMgr.replaceLabelsOnNode(map);
  
  // Now query for UNHEALTHY nodes
  request = GetClusterNodesRequest.newInstance(EnumSet.of(NodeState.UNHEALTHY));
  nodeReports = client.getClusterNodes(request).getNodeReports();
  Assert.assertEquals(1, nodeReports.size());
  Assert.assertEquals("Node is expected to be unhealthy!", NodeState.UNHEALTHY,
      nodeReports.get(0).getNodeState());
  
  Assert.assertTrue(nodeReports.get(0).getNodeLabels().contains("y"));
  
  // Remove labels of host1
  map = new HashMap<NodeId, Set<String>>();
  map.put(node.getNodeId(), ImmutableSet.of("y"));
  labelsMgr.removeLabelsFromNode(map);
  
  // Query all states should return all nodes
  rm.registerNode("host3:1236", 1024);
  request = GetClusterNodesRequest.newInstance(EnumSet.allOf(NodeState.class));
  nodeReports = client.getClusterNodes(request).getNodeReports();
  Assert.assertEquals(3, nodeReports.size());
  
  // All host1-3's label should be empty (instead of null)
  for (NodeReport report : nodeReports) {
    Assert.assertTrue(report.getNodeLabels() != null
        && report.getNodeLabels().isEmpty());
  }
  
  rpc.stopProxy(client, conf);
  rm.close();
}
Example #14
Source File: TestYARNRunner.java    From big-c with Apache License 2.0 4 votes vote down vote up 
@Test(timeout=20000)
public void testResourceMgrDelegate() throws Exception {
  /* we not want a mock of resource mgr delegate */
  final ApplicationClientProtocol clientRMProtocol = mock(ApplicationClientProtocol.class);
  ResourceMgrDelegate delegate = new ResourceMgrDelegate(conf) {
    @Override
    protected void serviceStart() throws Exception {
      assertTrue(this.client instanceof YarnClientImpl);
      ((YarnClientImpl) this.client).setRMClient(clientRMProtocol);
    }
  };
  /* make sure kill calls finish application master */
  when(clientRMProtocol.forceKillApplication(any(KillApplicationRequest.class)))
  .thenReturn(KillApplicationResponse.newInstance(true));
  delegate.killApplication(appId);
  verify(clientRMProtocol).forceKillApplication(any(KillApplicationRequest.class));

  /* make sure getalljobs calls get all applications */
  when(clientRMProtocol.getApplications(any(GetApplicationsRequest.class))).
  thenReturn(recordFactory.newRecordInstance(GetApplicationsResponse.class));
  delegate.getAllJobs();
  verify(clientRMProtocol).getApplications(any(GetApplicationsRequest.class));

  /* make sure getapplication report is called */
  when(clientRMProtocol.getApplicationReport(any(GetApplicationReportRequest.class)))
  .thenReturn(recordFactory.newRecordInstance(GetApplicationReportResponse.class));
  delegate.getApplicationReport(appId);
  verify(clientRMProtocol).getApplicationReport(any(GetApplicationReportRequest.class));

  /* make sure metrics is called */
  GetClusterMetricsResponse clusterMetricsResponse = recordFactory.newRecordInstance
      (GetClusterMetricsResponse.class);
  clusterMetricsResponse.setClusterMetrics(recordFactory.newRecordInstance(
      YarnClusterMetrics.class));
  when(clientRMProtocol.getClusterMetrics(any(GetClusterMetricsRequest.class)))
  .thenReturn(clusterMetricsResponse);
  delegate.getClusterMetrics();
  verify(clientRMProtocol).getClusterMetrics(any(GetClusterMetricsRequest.class));

  when(clientRMProtocol.getClusterNodes(any(GetClusterNodesRequest.class))).
  thenReturn(recordFactory.newRecordInstance(GetClusterNodesResponse.class));
  delegate.getActiveTrackers();
  verify(clientRMProtocol).getClusterNodes(any(GetClusterNodesRequest.class));
  
  GetNewApplicationResponse newAppResponse = recordFactory.newRecordInstance(
      GetNewApplicationResponse.class);
  newAppResponse.setApplicationId(appId);
  when(clientRMProtocol.getNewApplication(any(GetNewApplicationRequest.class))).
  thenReturn(newAppResponse);
  delegate.getNewJobID();
  verify(clientRMProtocol).getNewApplication(any(GetNewApplicationRequest.class));
  
  GetQueueInfoResponse queueInfoResponse = recordFactory.newRecordInstance(
      GetQueueInfoResponse.class);
  queueInfoResponse.setQueueInfo(recordFactory.newRecordInstance(QueueInfo.class));
  when(clientRMProtocol.getQueueInfo(any(GetQueueInfoRequest.class))).
  thenReturn(queueInfoResponse);
  delegate.getQueues();
  verify(clientRMProtocol).getQueueInfo(any(GetQueueInfoRequest.class));

  GetQueueUserAclsInfoResponse aclResponse = recordFactory.newRecordInstance(
      GetQueueUserAclsInfoResponse.class);
  when(clientRMProtocol.getQueueUserAcls(any(GetQueueUserAclsInfoRequest.class)))
  .thenReturn(aclResponse);
  delegate.getQueueAclsForCurrentUser();
  verify(clientRMProtocol).getQueueUserAcls(any(GetQueueUserAclsInfoRequest.class));
}
Example #15
Source File: TestClientRedirect.java    From big-c with Apache License 2.0 4 votes vote down vote up 
@Override
public GetClusterNodesResponse getClusterNodes(
    GetClusterNodesRequest request) throws IOException {
  return null;
}
Example #16
Source File: ApplicationClientProtocol.java    From hadoop with Apache License 2.0 3 votes vote down vote up 
/**
 * <p>The interface used by clients to get a report of all nodes
 * in the cluster from the <code>ResourceManager</code>.</p>
 * 
 * <p>The <code>ResourceManager</code> responds with a 
 * {@link GetClusterNodesResponse} which includes the 
 * {@link NodeReport} for all the nodes in the cluster.</p>
 * 
 * @param request request for report on all nodes
 * @return report on all nodes
 * @throws YarnException
 * @throws IOException
 */
@Public
@Stable
@Idempotent
public GetClusterNodesResponse getClusterNodes(
    GetClusterNodesRequest request) 
throws YarnException, IOException;
Example #17
Source File: ApplicationClientProtocol.java    From big-c with Apache License 2.0 3 votes vote down vote up 
/**
 * <p>The interface used by clients to get a report of all nodes
 * in the cluster from the <code>ResourceManager</code>.</p>
 * 
 * <p>The <code>ResourceManager</code> responds with a 
 * {@link GetClusterNodesResponse} which includes the 
 * {@link NodeReport} for all the nodes in the cluster.</p>
 * 
 * @param request request for report on all nodes
 * @return report on all nodes
 * @throws YarnException
 * @throws IOException
 */
@Public
@Stable
@Idempotent
public GetClusterNodesResponse getClusterNodes(
    GetClusterNodesRequest request) 
throws YarnException, IOException;
Example #18
Source File: RMCommunicator.java    From jumbune with GNU Lesser General Public License v3.0 2 votes vote down vote up 
/**
 * Get the Node Reports for all the nodes of the cluster.
 *
 * @return the node reports
 * @throws YarnException the yarn exception
 * @throws IOException Signals that an I/O exception has occurred.
 * @return, the list of node report
 */
public List<NodeReport> getNodeReports() throws YarnException, IOException{
	return proxy.getClusterNodes(GetClusterNodesRequest.newInstance())
				.getNodeReports();		
}