Java Code Examples for org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer#getContainerId()

The following examples show how to use org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer#getContainerId() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: CapacityScheduler.java    From big-c with Apache License 2.0 6 votes vote down vote up 
public void resumeContainer(RMContainer cont){
 if (LOG.isDebugEnabled()) {
     LOG.debug("RESUME_CONTAINER: container" + cont.toString());
 }
	
 //send this resource update info to NodeManager
 NodeId nodeId = cont.getContainer().getNodeId();
 ContainerId containerId = cont.getContainerId();
 //get current resource after preemption
 Resource currentResource = cont.getCurrentUsedResource();
 
 NodeContainerUpdate nodeContainerUpdate= NodeContainerUpdate.newInstance(containerId, 
		                                  currentResource.getMemory(), currentResource.getVirtualCores(),false,true);
 
 LOG.info("get container   "+containerId+"to resume "+"currentresource:   "+currentResource+"on host: "+cont.getAllocatedNode().getHost());
 //check if the resource is right
 if(nodeContainerUpdateMap.get(nodeId) == null){
  ConcurrentLinkedQueue<NodeContainerUpdate> listNodeContainerUpdate = new  ConcurrentLinkedQueue<NodeContainerUpdate>();
  listNodeContainerUpdate.add(nodeContainerUpdate);
  nodeContainerUpdateMap.put(nodeId, listNodeContainerUpdate);
 }else{
  nodeContainerUpdateMap.get(nodeId).add(nodeContainerUpdate);
 }
}
Example 2
Source File: CapacityScheduler.java    From hadoop with Apache License 2.0 5 votes vote down vote up 
/**
 * Process node labels update on a node.
 * 
 * TODO: Currently capacity scheduler will kill containers on a node when
 * labels on the node changed. It is a simply solution to ensure guaranteed
 * capacity on labels of queues. When YARN-2498 completed, we can let
 * preemption policy to decide if such containers need to be killed or just
 * keep them running.
 */
private synchronized void updateLabelsOnNode(NodeId nodeId,
    Set<String> newLabels) {
  FiCaSchedulerNode node = nodes.get(nodeId);
  if (null == node) {
    return;
  }
  
  // labels is same, we don't need do update
  if (node.getLabels().size() == newLabels.size()
      && node.getLabels().containsAll(newLabels)) {
    return;
  }
  
  // Kill running containers since label is changed
  for (RMContainer rmContainer : node.getRunningContainers()) {
    ContainerId containerId = rmContainer.getContainerId();
    completedContainer(rmContainer, 
        ContainerStatus.newInstance(containerId,
            ContainerState.COMPLETE, 
            String.format(
                "Container=%s killed since labels on the node=%s changed",
                containerId.toString(), nodeId.toString()),
            ContainerExitStatus.KILLED_BY_RESOURCEMANAGER),
        RMContainerEventType.KILL);
  }
  
  // Unreserve container on this node
  RMContainer reservedContainer = node.getReservedContainer();
  if (null != reservedContainer) {
    dropContainerReservation(reservedContainer);
  }
  
  // Update node labels after we've done this
  node.updateLabels(newLabels);
}
Example 3
Source File: CapacityScheduler.java    From big-c with Apache License 2.0 5 votes vote down vote up 
/**
 * Process node labels update on a node.
 * 
 * TODO: Currently capacity scheduler will kill containers on a node when
 * labels on the node changed. It is a simply solution to ensure guaranteed
 * capacity on labels of queues. When YARN-2498 completed, we can let
 * preemption policy to decide if such containers need to be killed or just
 * keep them running.
 */
private synchronized void updateLabelsOnNode(NodeId nodeId,
    Set<String> newLabels) {
  FiCaSchedulerNode node = nodes.get(nodeId);
  if (null == node) {
    return;
  }
  
  // labels is same, we don't need do update
  if (node.getLabels().size() == newLabels.size()
      && node.getLabels().containsAll(newLabels)) {
    return;
  }
  
  // Kill running containers since label is changed
  for (RMContainer rmContainer : node.getRunningContainers()) {
    ContainerId containerId = rmContainer.getContainerId();
    completedContainer(rmContainer, 
        ContainerStatus.newInstance(containerId,
            ContainerState.COMPLETE, 
            String.format(
                "Container=%s killed since labels on the node=%s changed",
                containerId.toString(), nodeId.toString()),
            ContainerExitStatus.KILLED_BY_RESOURCEMANAGER),
        RMContainerEventType.KILL);
  }
  
  // Unreserve container on this node
  RMContainer reservedContainer = node.getReservedContainer();
  if (null != reservedContainer) {
    dropContainerReservation(reservedContainer);
  }
  
  // Update node labels after we've done this
  node.updateLabels(newLabels);
}
Example 4
Source File: ResourceSchedulerWrapper.java    From hadoop with Apache License 2.0 4 votes vote down vote up 
private void updateQueueWithNodeUpdate(
        NodeUpdateSchedulerEventWrapper eventWrapper) {
  RMNodeWrapper node = (RMNodeWrapper) eventWrapper.getRMNode();
  List<UpdatedContainerInfo> containerList = node.getContainerUpdates();
  for (UpdatedContainerInfo info : containerList) {
    for (ContainerStatus status : info.getCompletedContainers()) {
      ContainerId containerId = status.getContainerId();
      SchedulerAppReport app = scheduler.getSchedulerAppInfo(
              containerId.getApplicationAttemptId());

      if (app == null) {
        // this happens for the AM container
        // The app have already removed when the NM sends the release
        // information.
        continue;
      }

      String queue =
          appQueueMap.get(containerId.getApplicationAttemptId()
            .getApplicationId());
      int releasedMemory = 0, releasedVCores = 0;
      if (status.getExitStatus() == ContainerExitStatus.SUCCESS) {
        for (RMContainer rmc : app.getLiveContainers()) {
          if (rmc.getContainerId() == containerId) {
            releasedMemory += rmc.getContainer().getResource().getMemory();
            releasedVCores += rmc.getContainer()
                    .getResource().getVirtualCores();
            break;
          }
        }
      } else if (status.getExitStatus() == ContainerExitStatus.ABORTED) {
        if (preemptionContainerMap.containsKey(containerId)) {
          Resource preResource = preemptionContainerMap.get(containerId);
          releasedMemory += preResource.getMemory();
          releasedVCores += preResource.getVirtualCores();
          preemptionContainerMap.remove(containerId);
        }
      }
      // update queue counters
      updateQueueMetrics(queue, releasedMemory, releasedVCores);
    }
  }
}
Example 5
Source File: FiCaSchedulerApp.java    From big-c with Apache License 2.0 4 votes vote down vote up 
synchronized public boolean containerResume(RMContainer rmContainer,Resource toResume){
 
 
 
 ContainerId containerId = rmContainer.getContainerId();
 
 if(isStopped){
  return false;
 }
 
 if(!isSuspending){
  return false;
 }
 
 if(!containersSuspended.contains(containerId)){
  return false;
 }
 //add resumed resource
 rmContainer.addResumedResource(toResume);
 
 //we try to update its resource consumption
 rmContainer.handle(
            new RMContainerEvent(containerId,RMContainerEventType.RESUME)  
     );

 //if all of its resource has been resumed
 if(!rmContainer.isSuspending()){
 //delete contaienr from containersSuspended
 this.containersSuspended.remove(containerId);
 }  
 
 //update resource usage
 queue.getMetrics().allocateResources(getUser(), 1, toResume, true);
 //update app resource usage
 Resources.addTo(currentConsumption,toResume);
 //inform RMContainer
 if(this.containersSuspended.size() == 0){  
  isSuspending = false;
  LOG.info("application "+this.getApplicationId()+"has been out of the suspended list");
 }
 
 LOG.info("app "+this.getApplicationAttemptId()+" consume resource"+currentConsumption);

   if (LOG.isDebugEnabled()) {
     LOG.debug("allocate: applicationAttemptId=" 
         + this.getApplicationAttemptId() 
         + " container=" + containerId + " host="
         + rmContainer.getContainer().getNodeId().getHost() 
       );
   }
 
 return true;
 
}
Example 6
Source File: CapacityScheduler.java    From big-c with Apache License 2.0 4 votes vote down vote up 
@Override
 public void suspendContianer(RMContainer cont, Resource toPreempt) {
 	// TODO Auto-generated method stub
if (LOG.isDebugEnabled()) {
      LOG.debug("SUSPEND_CONTAINER: container" + cont.toString());
}

LOG.info("capacity scheduler try to preempt "+cont.getContainerId()+" resource: "+toPreempt);

if(toPreempt == null){
    LOG.info("preempted resource can not be null");
    return;
 }
 if(!Resources.greaterThan(getResourceCalculator(), clusterResource, 
		                        toPreempt,Resources.none())){
     LOG.info("preempted resource is none");
     return;
 }
   //set preempted resource
    cont.addPreemptedResource(toPreempt);
   //mark this container to be preempted
 completedContainer(cont, SchedulerUtils.createPreemptedContainerStatus(
	      cont.getContainerId(), SchedulerUtils.PREEMPTED_CONTAINER),
	      RMContainerEventType.SUSPEND);
 //send this resource update info to NodeManager
 NodeId nodeId = cont.getContainer().getNodeId();
 ContainerId containerId = cont.getContainerId();
 //get current resource after preemption
 Resource currentResource = cont.getCurrentUsedResource();
 NodeContainerUpdate nodeContainerUpdate= NodeContainerUpdate.newInstance(containerId, 
		                                  currentResource.getMemory(), currentResource.getVirtualCores(),true,false);
 
 LOG.info("get container   "+containerId+" to suspend "+" on host "+nodeId.getHost()+" currentresource:   "+currentResource);
 if(nodeContainerUpdateMap.get(nodeId) == null){
	 ConcurrentLinkedQueue<NodeContainerUpdate> listNodeContainerUpdate = new  ConcurrentLinkedQueue<NodeContainerUpdate>();
	 listNodeContainerUpdate.add(nodeContainerUpdate);
	 nodeContainerUpdateMap.put(nodeId, listNodeContainerUpdate);
 }else{
	 nodeContainerUpdateMap.get(nodeId).add(nodeContainerUpdate);
 }
 }
Example 7
Source File: ResourceSchedulerWrapper.java    From big-c with Apache License 2.0 4 votes vote down vote up 
private void updateQueueWithNodeUpdate(
        NodeUpdateSchedulerEventWrapper eventWrapper) {
  RMNodeWrapper node = (RMNodeWrapper) eventWrapper.getRMNode();
  List<UpdatedContainerInfo> containerList = node.getContainerUpdates();
  for (UpdatedContainerInfo info : containerList) {
    for (ContainerStatus status : info.getCompletedContainers()) {
      ContainerId containerId = status.getContainerId();
      SchedulerAppReport app = scheduler.getSchedulerAppInfo(
              containerId.getApplicationAttemptId());

      if (app == null) {
        // this happens for the AM container
        // The app have already removed when the NM sends the release
        // information.
        continue;
      }

      String queue =
          appQueueMap.get(containerId.getApplicationAttemptId()
            .getApplicationId());
      int releasedMemory = 0, releasedVCores = 0;
      if (status.getExitStatus() == ContainerExitStatus.SUCCESS) {
        for (RMContainer rmc : app.getLiveContainers()) {
          if (rmc.getContainerId() == containerId) {
            releasedMemory += rmc.getContainer().getResource().getMemory();
            releasedVCores += rmc.getContainer()
                    .getResource().getVirtualCores();
            break;
          }
        }
      } else if (status.getExitStatus() == ContainerExitStatus.ABORTED) {
        if (preemptionContainerMap.containsKey(containerId)) {
          Resource preResource = preemptionContainerMap.get(containerId);
          releasedMemory += preResource.getMemory();
          releasedVCores += preResource.getVirtualCores();
          preemptionContainerMap.remove(containerId);
        }
      }
      // update queue counters
      updateQueueMetrics(queue, releasedMemory, releasedVCores);
    }
  }
}