org.apache.hadoop.yarn.api.records.NodeReport Java Examples

@Override
public AllocateResponse allocate(AllocateRequest request)
    throws YarnException, IOException {
  lastAsk = request.getAskList();
  for (ResourceRequest req : lastAsk) {
    if (ResourceRequest.ANY.equals(req.getResourceName())) {
      Priority priority = req.getPriority();
      if (priority.equals(RMContainerAllocator.PRIORITY_MAP)) {
        lastAnyAskMap = req.getNumContainers();
      } else if (priority.equals(RMContainerAllocator.PRIORITY_REDUCE)){
        lastAnyAskReduce = req.getNumContainers();
      }
    }
  }
  AllocateResponse response =  AllocateResponse.newInstance(
      request.getResponseId(),
      containersToComplete, containersToAllocate,
      Collections.<NodeReport>emptyList(),
      Resource.newInstance(512000, 1024, 1024), null, 10, null,
      Collections.<NMToken>emptyList());
  containersToComplete.clear();
  containersToAllocate.clear();
  return response;
}
 

@SuppressWarnings("deprecation")
@Test
public void testAllocateResponseWithoutIncDecContainers() {
  AllocateResponse r =
      AllocateResponse.newInstance(3, new ArrayList<ContainerStatus>(),
          new ArrayList<Container>(), new ArrayList<NodeReport>(), null,
          AMCommand.AM_RESYNC, 3, null, new ArrayList<NMToken>(), null, null);

  // serde
  AllocateResponseProto p = ((AllocateResponsePBImpl) r).getProto();
  r = new AllocateResponsePBImpl(p);

  // check value
  Assert.assertEquals(0, r.getIncreasedContainers().size());
  Assert.assertEquals(0, r.getDecreasedContainers().size());
}
 

private List<NodeReport> getNodeReports(int noOfNodes, NodeState state,
    boolean emptyNodeLabel) {
  List<NodeReport> nodeReports = new ArrayList<NodeReport>();

  for (int i = 0; i < noOfNodes; i++) {
    Set<String> nodeLabels = null;
    if (!emptyNodeLabel) {
      // node labels is not ordered, but when we output it, it should be
      // ordered
      nodeLabels = ImmutableSet.of("c", "b", "a", "x", "z", "y");
    }
    NodeReport nodeReport = NodeReport.newInstance(NodeId
      .newInstance("host" + i, 0), state, "host" + 1 + ":8888",
        "rack1", Records.newRecord(Resource.class), Records
            .newRecord(Resource.class), 0, "", 0, nodeLabels);
    nodeReports.add(nodeReport);
  }
  return nodeReports;
}
 

private ClusterResourceDescription getCurrentFreeClusterResources(YarnClient yarnClient) throws YarnException, IOException {
	List<NodeReport> nodes = yarnClient.getNodeReports(NodeState.RUNNING);

	int totalFreeMemory = 0;
	int containerLimit = 0;
	int[] nodeManagersFree = new int[nodes.size()];

	for (int i = 0; i < nodes.size(); i++) {
		NodeReport rep = nodes.get(i);
		int free = rep.getCapability().getMemory() - (rep.getUsed() != null ? rep.getUsed().getMemory() : 0);
		nodeManagersFree[i] = free;
		totalFreeMemory += free;
		if (free > containerLimit) {
			containerLimit = free;
		}
	}
	return new ClusterResourceDescription(totalFreeMemory, containerLimit, nodeManagersFree);
}
 

private NodeReport createNodeReports(RMNode rmNode) {    
  SchedulerNodeReport schedulerNodeReport = 
      scheduler.getNodeReport(rmNode.getNodeID());
  Resource used = BuilderUtils.newResource(0, 0);
  int numContainers = 0;
  if (schedulerNodeReport != null) {
    used = schedulerNodeReport.getUsedResource();
    numContainers = schedulerNodeReport.getNumContainers();
  } 
  
  NodeReport report =
      BuilderUtils.newNodeReport(rmNode.getNodeID(), rmNode.getState(),
          rmNode.getHttpAddress(), rmNode.getRackName(), used,
          rmNode.getTotalCapability(), numContainers,
          rmNode.getHealthReport(), rmNode.getLastHealthReportTime(),
          rmNode.getNodeLabels());

  return report;
}
 

public static NodeReport newNodeReport(NodeId nodeId, NodeState nodeState,
    String httpAddress, String rackName, Resource used, Resource capability,
    int numContainers, String healthReport, long lastHealthReportTime,
    Set<String> nodeLabels) {
  NodeReport nodeReport = recordFactory.newRecordInstance(NodeReport.class);
  nodeReport.setNodeId(nodeId);
  nodeReport.setNodeState(nodeState);
  nodeReport.setHttpAddress(httpAddress);
  nodeReport.setRackName(rackName);
  nodeReport.setUsed(used);
  nodeReport.setCapability(capability);
  nodeReport.setNumContainers(numContainers);
  nodeReport.setHealthReport(healthReport);
  nodeReport.setLastHealthReportTime(lastHealthReportTime);
  nodeReport.setNodeLabels(nodeLabels);
  return nodeReport;
}
 

public static AllocateResponse newAllocateResponse(int responseId,
    List<ContainerStatus> completedContainers,
    List<Container> allocatedContainers, List<NodeReport> updatedNodes,
    Resource availResources, AMCommand command, int numClusterNodes,
    PreemptionMessage preempt) {
  AllocateResponse response = recordFactory
      .newRecordInstance(AllocateResponse.class);
  response.setNumClusterNodes(numClusterNodes);
  response.setResponseId(responseId);
  response.setCompletedContainersStatuses(completedContainers);
  response.setAllocatedContainers(allocatedContainers);
  response.setUpdatedNodes(updatedNodes);
  response.setAvailableResources(availResources);
  response.setAMCommand(command);
  response.setPreemptionMessage(preempt);

  return response;
}
 

@Override
public GetClusterNodesResponse getClusterNodes(GetClusterNodesRequest request)
    throws YarnException {
  GetClusterNodesResponse response = 
    recordFactory.newRecordInstance(GetClusterNodesResponse.class);
  EnumSet<NodeState> nodeStates = request.getNodeStates();
  if (nodeStates == null || nodeStates.isEmpty()) {
    nodeStates = EnumSet.allOf(NodeState.class);
  }
  Collection<RMNode> nodes = RMServerUtils.queryRMNodes(rmContext,
      nodeStates);
  
  List<NodeReport> nodeReports = new ArrayList<NodeReport>(nodes.size());
  for (RMNode nodeInfo : nodes) {
    nodeReports.add(createNodeReports(nodeInfo));
  }
  response.setNodeReports(nodeReports);
  return response;
}
 

private NodeReport createNodeReports(RMNode rmNode) {    
  SchedulerNodeReport schedulerNodeReport = 
      scheduler.getNodeReport(rmNode.getNodeID());
  Resource used = BuilderUtils.newResource(0, 0, 0);
  int numContainers = 0;
  if (schedulerNodeReport != null) {
    used = schedulerNodeReport.getUsedResource();
    numContainers = schedulerNodeReport.getNumContainers();
  } 
  
  NodeReport report =
      BuilderUtils.newNodeReport(rmNode.getNodeID(), rmNode.getState(),
          rmNode.getHttpAddress(), rmNode.getRackName(), used,
          rmNode.getTotalCapability(), numContainers,
          rmNode.getHealthReport(), rmNode.getLastHealthReportTime(),
          rmNode.getNodeLabels());

  return report;
}
 

/**
 * Lists the nodes matching the given node states
 * 
 * @param nodeStates
 * @throws YarnException
 * @throws IOException
 */
private void listClusterNodes(Set<NodeState> nodeStates) 
          throws YarnException, IOException {
  PrintWriter writer = new PrintWriter(
      new OutputStreamWriter(sysout, Charset.forName("UTF-8")));
  List<NodeReport> nodesReport = client.getNodeReports(
                                     nodeStates.toArray(new NodeState[0]));
  writer.println("Total Nodes:" + nodesReport.size());
  writer.printf(NODES_PATTERN, "Node-Id", "Node-State", "Node-Http-Address",
      "Number-of-Running-Containers");
  for (NodeReport nodeReport : nodesReport) {
    writer.printf(NODES_PATTERN, nodeReport.getNodeId(), nodeReport
        .getNodeState(), nodeReport.getHttpAddress(), nodeReport
        .getNumContainers());
  }
  writer.flush();
}
 

@Public
@Stable
public static AllocateResponse newInstance(int responseId,
    List<ContainerStatus> completedContainers,
    List<Container> allocatedContainers, List<NodeReport> updatedNodes,
    Resource availResources, AMCommand command, int numClusterNodes,
    PreemptionMessage preempt, List<NMToken> nmTokens) {
  AllocateResponse response = Records.newRecord(AllocateResponse.class);
  response.setNumClusterNodes(numClusterNodes);
  response.setResponseId(responseId);
  response.setCompletedContainersStatuses(completedContainers);
  response.setAllocatedContainers(allocatedContainers);
  response.setUpdatedNodes(updatedNodes);
  response.setAvailableResources(availResources);
  response.setAMCommand(command);
  response.setPreemptionMessage(preempt);
  response.setNMTokens(nmTokens);
  return response;
}
 

@Public
@Stable
public static AllocateResponse newInstance(int responseId,
    List<ContainerStatus> completedContainers,
    List<Container> allocatedContainers, List<NodeReport> updatedNodes,
    Resource availResources, AMCommand command, int numClusterNodes,
    PreemptionMessage preempt, List<NMToken> nmTokens,
    List<ContainerResourceIncrease> increasedContainers,
    List<ContainerResourceDecrease> decreasedContainers) {
  AllocateResponse response = newInstance(responseId, completedContainers,
      allocatedContainers, updatedNodes, availResources, command,
      numClusterNodes, preempt, nmTokens);
  response.setIncreasedContainers(increasedContainers);
  response.setDecreasedContainers(decreasedContainers);
  return response;
}
 

/**
 * Assign host to container given affinity and anti-affinity constraints and resource availibility on node
 * @param host
 * @param antiHosts
 * @param antiPreferredHosts
 * @param grpObj
 * @param nodeLocalSet
 * @param aggrMemory
 * @param vCores
 * @return
 */
public String assignHost(String host, List<String> antiHosts, List<String> antiPreferredHosts, HostOperatorSet grpObj, Set<PTOperator> nodeLocalSet, int aggrMemory, int vCores)
{
  for (Map.Entry<String, NodeReport> nodeEntry : nodeReportMap.entrySet()) {
    if (nodeEntry.getValue().getNodeState() == NodeState.RUNNING) {
      int memAvailable = nodeEntry.getValue().getCapability().getMemory() - nodeEntry.getValue().getUsed().getMemory();
      int vCoresAvailable = nodeEntry.getValue().getCapability().getVirtualCores() - nodeEntry.getValue().getUsed().getVirtualCores();
      if (memAvailable >= aggrMemory && vCoresAvailable >= vCores && !antiHosts.contains(nodeEntry.getKey()) && !antiPreferredHosts.contains(nodeEntry.getKey())) {
        host = nodeEntry.getKey();
        grpObj.setHost(host);
        nodeLocalMapping.put(nodeLocalSet, host);

        return host;
      }
    }
  }
  return null;
}
 

/** {@inheritDoc} */
@Override public synchronized void onNodesUpdated(List<NodeReport> updated) {
    for (NodeReport node : updated) {
        // If node unusable.
        if (node.getNodeState().isUnusable()) {
            for (IgniteContainer cont : containers.values()) {
                if (cont.nodeId().equals(node.getNodeId())) {
                    containers.remove(cont.id());

                    log.log(Level.WARNING, "Node is unusable. Node: {0}, state: {1}.",
                        new Object[]{node.getNodeId().getHost(), node.getNodeState()});
                }
            }

            log.log(Level.WARNING, "Node is unusable. Node: {0}, state: {1}.",
                new Object[]{node.getNodeId().getHost(), node.getNodeState()});
        }
    }
}
 

private List<NodeReport> getNodeReports(
    List<NodeReport> nodeReports,
    Set<NodeState> nodeStates) {
  List<NodeReport> reports = new ArrayList<NodeReport>();

  for (NodeReport nodeReport : nodeReports) {
    if (nodeStates.contains(nodeReport.getNodeState())) {
      reports.add(nodeReport);
    }
  }
  return reports;
}
 

@SuppressWarnings("unchecked")
@Override
public synchronized void nodesUpdated(List<NodeReport> updatedNodes) {
  for (NodeReport nr : updatedNodes) {
    // Scheduler will find out from the node, if at all.
    // Relying on the RM to not allocate containers on an unhealthy node.
    eventHandler.handle(new AMNodeEventStateChanged(nr));
  }
}
 

@Override
public String getClusterDescription() {

	try {
		ByteArrayOutputStream baos = new ByteArrayOutputStream();
		PrintStream ps = new PrintStream(baos);

		YarnClusterMetrics metrics = yarnClient.getYarnClusterMetrics();

		ps.append("NodeManagers in the ClusterClient " + metrics.getNumNodeManagers());
		List<NodeReport> nodes = yarnClient.getNodeReports(NodeState.RUNNING);
		final String format = "|%-16s |%-16s %n";
		ps.printf("|Property         |Value          %n");
		ps.println("+---------------------------------------+");
		int totalMemory = 0;
		int totalCores = 0;
		for (NodeReport rep : nodes) {
			final Resource res = rep.getCapability();
			totalMemory += res.getMemory();
			totalCores += res.getVirtualCores();
			ps.format(format, "NodeID", rep.getNodeId());
			ps.format(format, "Memory", res.getMemory() + " MB");
			ps.format(format, "vCores", res.getVirtualCores());
			ps.format(format, "HealthReport", rep.getHealthReport());
			ps.format(format, "Containers", rep.getNumContainers());
			ps.println("+---------------------------------------+");
		}
		ps.println("Summary: totalMemory " + totalMemory + " totalCores " + totalCores);
		List<QueueInfo> qInfo = yarnClient.getAllQueues();
		for (QueueInfo q : qInfo) {
			ps.println("Queue: " + q.getQueueName() + ", Current Capacity: " + q.getCurrentCapacity() + " Max Capacity: " +
				q.getMaximumCapacity() + " Applications: " + q.getApplications().size());
		}
		return baos.toString();
	} catch (Exception e) {
		throw new RuntimeException("Couldn't get cluster description", e);
	}
}
 

private AllocateResponse createAllocateResponse(
    List<ContainerStatus> completed, List<Container> allocated,
    List<NMToken> nmTokens) {
  AllocateResponse response =
      AllocateResponse.newInstance(0, completed, allocated,
          new ArrayList<NodeReport>(), null, null, 1, null, nmTokens);
  return response;
}
 

@Override
public void onNodesUpdated(List<NodeReport> updatedNodes) {
  if (isStopped.get()) {
    return;
  }
  // ignore bad nodes for now
  // upcall to app must be outside locks
  appClientDelegate.nodesUpdated(updatedNodes);
}
 

private static NodeReport generateNodeReport(NodeId nodeId, NodeState nodeState) {
  NodeReport nodeReport = mock(NodeReport.class);
  doReturn(nodeId).when(nodeReport).getNodeId();
  doReturn(nodeState).when(nodeReport).getNodeState();
  String httpAddress = nodeId.getHost() + ":3433";
  doReturn(httpAddress).when(nodeReport).getHttpAddress();
  doReturn("/default-rack").when(nodeReport).getRackName();
  doReturn(Resource.newInstance(0, 0)).when(nodeReport).getUsed();
  doReturn(Resource.newInstance(10240, 12)).when(nodeReport).getCapability();
  doReturn(10).when(nodeReport).getNumContainers();
  doReturn(nodeState.toString()).when(nodeReport).getHealthReport();
  long healthReportTime = System.currentTimeMillis();
  doReturn(healthReportTime).when(nodeReport).getLastHealthReportTime();
  return nodeReport;
}
 

public AllocateResponse createFakeAllocateResponse() {
  return AllocateResponse.newInstance(-1,
      new ArrayList<ContainerStatus>(),
      new ArrayList<Container>(), new ArrayList<NodeReport>(),
      Resource.newInstance(1024, 2), null, 1,
      null, new ArrayList<NMToken>());
}
 

@SuppressWarnings("unchecked")
private void handleUpdatedNodes(AllocateResponse response) {
  // send event to the job about on updated nodes
  List<NodeReport> updatedNodes = response.getUpdatedNodes();
  if (!updatedNodes.isEmpty()) {

    // send event to the job to act upon completed tasks
    eventHandler.handle(new JobUpdatedNodesEvent(getJob().getID(),
        updatedNodes));

    // act upon running tasks
    HashSet<NodeId> unusableNodes = new HashSet<NodeId>();
    for (NodeReport nr : updatedNodes) {
      NodeState nodeState = nr.getNodeState();
      if (nodeState.isUnusable()) {
        unusableNodes.add(nr.getNodeId());
      }
    }
    for (int i = 0; i < 2; ++i) {
      HashMap<TaskAttemptId, Container> taskSet = i == 0 ? assignedRequests.maps
          : assignedRequests.reduces;
      // kill running containers
      for (Map.Entry<TaskAttemptId, Container> entry : taskSet.entrySet()) {
        TaskAttemptId tid = entry.getKey();
        NodeId taskAttemptNodeId = entry.getValue().getNodeId();
        if (unusableNodes.contains(taskAttemptNodeId)) {
          LOG.info("Killing taskAttempt:" + tid
              + " because it is running on unusable node:"
              + taskAttemptNodeId);
          eventHandler.handle(new TaskAttemptKillEvent(tid,
              "TaskAttempt killed because it ran on unusable node"
                  + taskAttemptNodeId));
        }
      }
    }
  }
}
 

@Private
@Unstable
public static GetClusterNodesResponse
    newInstance(List<NodeReport> nodeReports) {
  GetClusterNodesResponse response =
      Records.newRecord(GetClusterNodesResponse.class);
  response.setNodeReports(nodeReports);
  return response;
}
 

public AMNodeEventStateChanged(NodeReport nodeReport, int sourceId) {
  super(nodeReport.getNodeId(), sourceId,
        (nodeReport.getNodeState().isUnusable() ? 
            AMNodeEventType.N_TURNED_UNHEALTHY :
            AMNodeEventType.N_TURNED_HEALTHY));
  this.nodeReport = nodeReport;
}
 

public static TaskTrackerInfo[] fromYarnNodes(List<NodeReport> nodes) {
  List<TaskTrackerInfo> taskTrackers = new ArrayList<TaskTrackerInfo>();
  for (NodeReport node : nodes) {
    taskTrackers.add(fromYarn(node));
  }
  return taskTrackers.toArray(new TaskTrackerInfo[nodes.size()]);
}
 

/**
 * Gets the container utilization alert.
 *
 * @param cluster the cluster
 * @param rmCommunicator 
 * @return the container utilization alert
 * @throws IOException Signals that an I/O exception has occurred.
 * @throws YarnException the yarn exception
 */
public List <AlertInfo> getContainerUtilizationAlert(Cluster cluster, RMCommunicator rmCommunicator){

	List<AlertInfo> containerAlert = new ArrayList<AlertInfo>();
	float containerBasedonVcores, containerBasedonMemory, container;

	List<NodeReport> nodeReports;
	try {
		nodeReports = rmCommunicator.getNodeReports();
	
	for(NodeReport nr:nodeReports){

		if (nr.getNumContainers() > 0){
			containerBasedonVcores = nr.getCapability().getVirtualCores()/getMinimumParameterMandatoryForContainer(ExtendedConstants.YARN_CONTAINER_MINIMUM_VCORE,1,cluster);
			containerBasedonMemory = nr.getCapability().getMemory()/getMinimumParameterMandatoryForContainer(ExtendedConstants.YARN_CONTAINER_MINIMUM_MEMORY,1,cluster);
			container = Math.min(containerBasedonVcores, containerBasedonMemory);
			if (((nr.getNumContainers()/container)*100) > 80){
				String host = nr.getNodeId().getHost();
				if(!host.contains(".")){
					host = InetAddress.getByName(host).getHostAddress();
				}
				AlertInfo alertInfo = new AlertInfo(ExtendedConstants.CRITICAL_LEVEL,host,"Container capacity is over utilized", getDate() );
				containerAlert.add(alertInfo);
			}
		}
	}} catch (YarnException | IOException e) {
		LOGGER.error("Unable to get container utilization alert due to : ",e);
	}

	return containerAlert;
}
 

@Test(timeout = 15000)
public void testGetClusterNodesOnHA() throws Exception {
  List<NodeReport> reports = client.getNodeReports(NodeState.RUNNING);
  Assert.assertTrue(reports != null && !reports.isEmpty());
  Assert.assertEquals(cluster.createFakeNodeReports(),
      reports);
}
 

@SuppressWarnings("unchecked")
public synchronized void nodesUpdated(int schedulerId, List<NodeReport> updatedNodes) {
  for (NodeReport nr : updatedNodes) {
    // Scheduler will find out from the node, if at all.
    // Relying on the RM to not allocate containers on an unhealthy node.
    eventHandler.handle(new AMNodeEventStateChanged(nr, schedulerId));
  }
}
 

@Override
public void setNodeReports(List<NodeReport> nodeManagers) {
  if (nodeManagers == null) {
    builder.clearNodeReports();
  }
  this.nodeManagerInfoList = nodeManagers;
}
 

@Override
public void setNodeReports(List<NodeReport> nodeManagers) {
  if (nodeManagers == null) {
    builder.clearNodeReports();
  }
  this.nodeManagerInfoList = nodeManagers;
}