Java Code Examples for org.apache.hadoop.yarn.util.resource.Resources#equals()

The following examples show how to use org.apache.hadoop.yarn.util.resource.Resources#equals() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: FSParentQueue.java    From hadoop with Apache License 2.0 6 votes vote down vote up 
@Override
public void updateDemand() {
  // Compute demand by iterating through apps in the queue
  // Limit demand to maxResources
  Resource maxRes = scheduler.getAllocationConfiguration()
      .getMaxResources(getName());
  demand = Resources.createResource(0);
  for (FSQueue childQueue : childQueues) {
    childQueue.updateDemand();
    Resource toAdd = childQueue.getDemand();
    if (LOG.isDebugEnabled()) {
      LOG.debug("Counting resource from " + childQueue.getName() + " " + 
          toAdd + "; Total resource consumption for " + getName() +
          " now " + demand);
    }
    demand = Resources.add(demand, toAdd);
    demand = Resources.componentwiseMin(demand, maxRes);
    if (Resources.equals(demand, maxRes)) {
      break;
    }
  }
  if (LOG.isDebugEnabled()) {
    LOG.debug("The updated demand for " + getName() + " is " + demand +
        "; the max is " + maxRes);
  }    
}
Example 2
Source File: FSParentQueue.java    From hadoop with Apache License 2.0 6 votes vote down vote up 
@Override
public Resource assignContainer(FSSchedulerNode node) {
  Resource assigned = Resources.none();

  // If this queue is over its limit, reject
  if (!assignContainerPreCheck(node)) {
    return assigned;
  }

  Collections.sort(childQueues, policy.getComparator());
  for (FSQueue child : childQueues) {
    assigned = child.assignContainer(node);
    if (!Resources.equals(assigned, Resources.none())) {
      break;
    }
  }
  return assigned;
}
Example 3
Source File: FSParentQueue.java    From big-c with Apache License 2.0 6 votes vote down vote up 
@Override
public void updateDemand() {
  // Compute demand by iterating through apps in the queue
  // Limit demand to maxResources
  Resource maxRes = scheduler.getAllocationConfiguration()
      .getMaxResources(getName());
  demand = Resources.createResource(0);
  for (FSQueue childQueue : childQueues) {
    childQueue.updateDemand();
    Resource toAdd = childQueue.getDemand();
    if (LOG.isDebugEnabled()) {
      LOG.debug("Counting resource from " + childQueue.getName() + " " + 
          toAdd + "; Total resource consumption for " + getName() +
          " now " + demand);
    }
    demand = Resources.add(demand, toAdd);
    demand = Resources.componentwiseMin(demand, maxRes);
    if (Resources.equals(demand, maxRes)) {
      break;
    }
  }
  if (LOG.isDebugEnabled()) {
    LOG.debug("The updated demand for " + getName() + " is " + demand +
        "; the max is " + maxRes);
  }    
}
Example 4
Source File: FSParentQueue.java    From big-c with Apache License 2.0 6 votes vote down vote up 
@Override
public Resource assignContainer(FSSchedulerNode node) {
  Resource assigned = Resources.none();

  // If this queue is over its limit, reject
  if (!assignContainerPreCheck(node)) {
    return assigned;
  }

  Collections.sort(childQueues, policy.getComparator());
  for (FSQueue child : childQueues) {
    assigned = child.assignContainer(node);
    if (!Resources.equals(assigned, Resources.none())) {
      break;
    }
  }
  return assigned;
}
Example 5
Source File: NodeLabel.java    From hadoop with Apache License 2.0 5 votes vote down vote up 
@Override
public boolean equals(Object obj) {
  if (obj instanceof NodeLabel) {
    NodeLabel other = (NodeLabel) obj;
    return Resources.equals(resource, other.getResource())
        && StringUtils.equals(labelName, other.getLabelName())
        && (other.getNumActiveNMs() == numActiveNMs); 
  }
  return false;
}
Example 6
Source File: CSQueueUtils.java    From hadoop with Apache License 2.0 5 votes vote down vote up 
@Lock(CSQueue.class)
public static void updateQueueStatistics(
    final ResourceCalculator calculator,
    final CSQueue childQueue, final CSQueue parentQueue, 
    final Resource clusterResource, final Resource minimumAllocation) {
  Resource queueLimit = Resources.none();
  Resource usedResources = childQueue.getUsedResources();
  
  float absoluteUsedCapacity = 0.0f;
  float usedCapacity = 0.0f;

  if (Resources.greaterThan(
      calculator, clusterResource, clusterResource, Resources.none())) {
    queueLimit = 
        Resources.multiply(clusterResource, childQueue.getAbsoluteCapacity());
    absoluteUsedCapacity = 
        Resources.divide(calculator, clusterResource, 
            usedResources, clusterResource);
    usedCapacity = 
        Resources.equals(queueLimit, Resources.none()) ? 0 :
        Resources.divide(calculator, clusterResource, 
            usedResources, queueLimit);
  }

  childQueue.setUsedCapacity(usedCapacity);
  childQueue.setAbsoluteUsedCapacity(absoluteUsedCapacity);
  
  Resource available = Resources.subtract(queueLimit, usedResources);
  childQueue.getMetrics().setAvailableResourcesToQueue(
      Resources.max(
          calculator, 
          clusterResource, 
          available, 
          Resources.none()
          )
      );
 }
Example 7
Source File: TestGreedyReservationAgent.java    From hadoop with Apache License 2.0 5 votes vote down vote up 
private boolean check(ReservationAllocation cs, long start, long end,
    int containers, int mem, int cores) {

  boolean res = true;
  for (long i = start; i < end; i++) {
    res = res
        && Resources.equals(cs.getResourcesAtTime(i),
            Resource.newInstance(mem * containers, cores * containers, cores * containers));
  }
  return res;
}
Example 8
Source File: NodeLabel.java    From big-c with Apache License 2.0 5 votes vote down vote up 
@Override
public boolean equals(Object obj) {
  if (obj instanceof NodeLabel) {
    NodeLabel other = (NodeLabel) obj;
    return Resources.equals(resource, other.getResource())
        && StringUtils.equals(labelName, other.getLabelName())
        && (other.getNumActiveNMs() == numActiveNMs); 
  }
  return false;
}
Example 9
Source File: RMContainerImpl.java    From big-c with Apache License 2.0 5 votes vote down vote up 
@Override
  public RMContainerState transition(RMContainerImpl container,
      RMContainerEvent event) {
   container.resumeTime.add(System.currentTimeMillis());
if(Resources.equals(container.getPreemptedResource(),Resources.none())){
//if all the preempted resource has been resumed  
   container.isSuspending = false;
   return RMContainerState.RUNNING;
   }else{
  	return RMContainerState.DEHYDRATED;
   }

}
Example 10
Source File: FiCaSchedulerApp.java    From big-c with Apache License 2.0 5 votes vote down vote up 
@Override
  public synchronized Collection<RMContainer> getUnPreemtedContainers() {
	    Collection<RMContainer> returnList = new ArrayList<RMContainer>();
	    for(RMContainer rmContainer : liveContainers.values()){
	    	//skip if all resource has been preempted
	    	if(Resources.equals(rmContainer.getCurrentUsedResource(),Resources.none())){
	    		continue;
	    	}
	    	
	    	returnList.add(rmContainer);
	    }
	     
	    return returnList;
}
Example 11
Source File: TestGreedyReservationAgent.java    From big-c with Apache License 2.0 5 votes vote down vote up 
private boolean check(ReservationAllocation cs, long start, long end,
    int containers, int mem, int cores) {

  boolean res = true;
  for (long i = start; i < end; i++) {
    res = res
        && Resources.equals(cs.getResourcesAtTime(i),
            Resource.newInstance(mem * containers, cores * containers));
  }
  return res;
}
Example 12
Source File: YarnNodeCapacityManager.java    From incubator-myriad with Apache License 2.0 5 votes vote down vote up 
/**
 * 1. Updates {@link RMNode#getTotalCapability()} with newCapacity.
 * 2. Sends out a {@link NodeResourceUpdateSchedulerEvent} that's handled by YARN's scheduler.
 * The scheduler updates the corresponding {@link SchedulerNode} with the newCapacity.
 *
 * @param rmNode
 * @param newCapacity
 */
@SuppressWarnings("unchecked")
public void setNodeCapacity(RMNode rmNode, Resource newCapacity) {
  //NOOP prevent YARN warning changing to same size
  if ((Resources.equals(rmNode.getTotalCapability(), newCapacity))) {
    return;
  }
  if (yarnScheduler.getSchedulerNode(rmNode.getNodeID()) == null) {
    LOGGER.info("Yarn Scheduler doesn't have node {}, probably UNHEALTHY", rmNode.getNodeID());
    return;
  }
  yarnSchedulerLock.lock();
  try {
    if (newCapacity.getMemory() < 0 || newCapacity.getVirtualCores() < 0) {
      Resource zeroed = ResourceUtils.componentwiseMax(ZERO_RESOURCE, newCapacity);
      rmNode.getTotalCapability().setMemory(zeroed.getMemory());
      rmNode.getTotalCapability().setVirtualCores(zeroed.getVirtualCores());
      LOGGER.warn("Asked to set Node {} to a value less than zero!  Had {}, setting to {}.",
          rmNode.getHttpAddress(), rmNode.getTotalCapability().toString(), zeroed.toString());
    } else {
      rmNode.getTotalCapability().setMemory(newCapacity.getMemory());
      rmNode.getTotalCapability().setVirtualCores(newCapacity.getVirtualCores());
      if (LOGGER.isInfoEnabled()) {
        LOGGER.info("Setting capacity for node {} to {}", rmNode.getHostName(), newCapacity);
      }
    }
    // updates the scheduler with the new capacity for the NM.
    // the event is handled by the scheduler asynchronously
    rmContext.getDispatcher().getEventHandler().handle(new NodeResourceUpdateSchedulerEvent(rmNode, ResourceOption.newInstance(
        rmNode.getTotalCapability(), RMNode.OVER_COMMIT_TIMEOUT_MILLIS_DEFAULT)));
  } finally {
    yarnSchedulerLock.unlock();
  }
}
Example 13
Source File: Application.java    From hadoop with Apache License 2.0 4 votes vote down vote up 
private synchronized void assign(Priority priority, NodeType type, 
    List<Container> containers) throws IOException, YarnException {
  for (Iterator<Container> i=containers.iterator(); i.hasNext();) {
    Container container = i.next();
    String host = container.getNodeId().toString();
    
    if (Resources.equals(requestSpec.get(priority), container.getResource())) { 
      // See which task can use this container
      for (Iterator<Task> t=tasks.get(priority).iterator(); t.hasNext();) {
        Task task = t.next();
        if (task.getState() == State.PENDING && task.canSchedule(type, host)) {
          NodeManager nodeManager = getNodeManager(host);
          
          task.start(nodeManager, container.getId());
          i.remove();
          
          // Track application resource usage
          Resources.addTo(used, container.getResource());
          
          LOG.info("Assigned container (" + container + ") of type " + type +
              " to task " + task.getTaskId() + " at priority " + priority + 
              " on node " + nodeManager.getHostName() +
              ", currently using " + used + " resources");

          // Update resource requests
          updateResourceRequests(requests.get(priority), type, task);

          // Launch the container
          StartContainerRequest scRequest =
              StartContainerRequest.newInstance(createCLC(),
                container.getContainerToken());
          List<StartContainerRequest> list =
              new ArrayList<StartContainerRequest>();
          list.add(scRequest);
          StartContainersRequest allRequests =
              StartContainersRequest.newInstance(list);
          nodeManager.startContainers(allRequests);
          break;
        }
      }
    }
  }
}
Example 14
Source File: CSQueueUtils.java    From big-c with Apache License 2.0 4 votes vote down vote up 
@Lock(CSQueue.class)
public static void updateQueueStatistics(
    final ResourceCalculator calculator,
    final CSQueue childQueue, final CSQueue parentQueue, 
    final Resource clusterResource, final Resource minimumAllocation) {
  Resource queueLimit = Resources.none();
  Resource usedResources = childQueue.getUsedResources();
  
  float absoluteUsedCapacity = 0.0f;
  float usedCapacity = 0.0f;

  if (Resources.greaterThan(
      calculator, clusterResource, clusterResource, Resources.none())) {
    queueLimit = 
        Resources.multiply(clusterResource, childQueue.getAbsoluteCapacity());
    
    //absoluteUsedCapacity = 
    //    Resources.divide(calculator, clusterResource, 
    //        usedResources, clusterResource);
    
    absoluteUsedCapacity = (float) ((usedResources.getVirtualCores()*1.0)/(clusterResource.getVirtualCores()*1.0));
  
    usedCapacity = 
        Resources.equals(queueLimit, Resources.none()) ? 0 :
        Resources.divide(calculator, clusterResource, 
            usedResources, queueLimit);
  }

  childQueue.setUsedCapacity(usedCapacity);
  childQueue.setAbsoluteUsedCapacity(absoluteUsedCapacity);
  
  Resource available = Resources.subtract(queueLimit, usedResources);
  childQueue.getMetrics().setAvailableResourcesToQueue(
      Resources.max(
          calculator, 
          clusterResource, 
          available, 
          Resources.none()
          )
      );
 }
Example 15
Source File: ProportionalCapacityPreemptionPolicy.java    From big-c with Apache License 2.0 4 votes vote down vote up 
Resource offer(Resource avail, ResourceCalculator rc,
    Resource clusterResource) {
 	
  Resource absMaxCapIdealAssignedDelta = Resources.componentwiseMax(
                  Resources.subtract(maxCapacity, idealAssigned),
                  Resource.newInstance(0, 0));
  
  // remain = avail - min(avail, (max - assigned), (current + pending - assigned))
  // we have bug here. in some case:
  //(current + pending - assigned).core > avail.core
  //(current + pending - assigned).memo < avail.memo
  //so we get least cores of the three and least memory of the three
  Resource possibleAccepted = 
      Resources.mins(rc, clusterResource, 
          absMaxCapIdealAssignedDelta,
      Resources.mins(rc, clusterResource, avail, Resources.subtract(
          Resources.add(current, pending), idealAssigned)));
  
  //final allocation resource
  Resource finalAccepted = Resources.clone(possibleAccepted);
  //in extrame case where avail cores are more less than the available memory, it may preempt mroe memory
  //Max:      1310720   320
  //avail:    542634    26
  //Delta:    734280    60  
  //Pending:  525312    120
  //current:  576512    260
  //ideal:    576512    260
  //then the accepted will be (525312,26) in which the memory is far more beyond the requirement
  
  if(isSuspended){
	  
  if(dominantResource == Resources.CPU && !Resources.equals(pending,Resources.none())){
	  //pending must be either none() or resource(int ,int)
	  if(avail.getVirtualCores() == 0){
	      //if the dominant resource is cpu, we will stop allocation even we have memory
		  finalAccepted.setMemory(0);
		  //but if we still have more available memory, we can allocate, to avoid preemption
		  //we set memory to current usage
		  int gapMemory = current.getMemory() - idealAssigned.getMemory();
		  if(gapMemory > 0 && possibleAccepted.getMemory() > gapMemory){
			  finalAccepted.setMemory(gapMemory);
			  LOG.info("gap memory: "+gapMemory);
		  }
		  
	  }else{
	  double memoryRatio   = pending.getMemory()*1.0/pending.getVirtualCores();
	  int    ratioedMemory = (int)(memoryRatio*possibleAccepted.getVirtualCores());
	  finalAccepted.setMemory(ratioedMemory < possibleAccepted.getMemory() ? 
			                   ratioedMemory:possibleAccepted.getMemory());
	  }
	  LOG.info("queue: "+queueName+" cpu dominant ");
	  
	  if(finalAccepted.getMemory() < possibleAccepted.getMemory()){
		  LOG.info("previous memory: "+possibleAccepted.getMemory()+"  final memory: "+finalAccepted.getMemory());
	  }
	  
	  
  }else if(dominantResource == Resources.MEMORY && !Resources.equals(pending, Resources.none())){
	  if(avail.getMemory() == 0){
		  finalAccepted.setVirtualCores(0);
		  int gapCores = current.getVirtualCores() - idealAssigned.getVirtualCores();
		  if(gapCores > 0 && possibleAccepted.getVirtualCores() > gapCores){
			  finalAccepted.setVirtualCores(gapCores);
			  LOG.info("gap cores: "+gapCores);
		  }
		  
	  }else{
	  double cpuRatio   = pending.getVirtualCores()*1.0/pending.getMemory();
	  int    ratioedcpu = (int)(cpuRatio*possibleAccepted.getMemory());
	  finalAccepted.setVirtualCores(ratioedcpu < possibleAccepted.getMemory() ? 
			                 ratioedcpu:possibleAccepted.getMemory());
	  }
	  LOG.info("queue: "+queueName+" memory dominant ");
  }else{
	  LOG.info("queue: "+queueName+" empty ");
  }
  
  }
 
  LOG.info("queueName:   "+queueName);
  LOG.info("beforeideal: "+idealAssigned);  
  Resource remain = Resources.subtract(avail, finalAccepted);
  Resources.addTo(idealAssigned, finalAccepted);
  LOG.info("avaul:       "+avail);
  LOG.info("absMaxDelta: "+absMaxCapIdealAssignedDelta);
  LOG.info("max:         "+maxCapacity);
  LOG.info("current:     "+current);
  LOG.info("pending:     "+pending);
  LOG.info("acceped:     "+finalAccepted);
  LOG.info("ideal:       "+idealAssigned);
  
  return remain;
  
}
Example 16
Source File: Application.java    From big-c with Apache License 2.0 4 votes vote down vote up 
private synchronized void assign(Priority priority, NodeType type, 
    List<Container> containers) throws IOException, YarnException {
  for (Iterator<Container> i=containers.iterator(); i.hasNext();) {
    Container container = i.next();
    String host = container.getNodeId().toString();
    
    if (Resources.equals(requestSpec.get(priority), container.getResource())) { 
      // See which task can use this container
      for (Iterator<Task> t=tasks.get(priority).iterator(); t.hasNext();) {
        Task task = t.next();
        if (task.getState() == State.PENDING && task.canSchedule(type, host)) {
          NodeManager nodeManager = getNodeManager(host);
          
          task.start(nodeManager, container.getId());
          i.remove();
          
          // Track application resource usage
          Resources.addTo(used, container.getResource());
          
          LOG.info("Assigned container (" + container + ") of type " + type +
              " to task " + task.getTaskId() + " at priority " + priority + 
              " on node " + nodeManager.getHostName() +
              ", currently using " + used + " resources");

          // Update resource requests
          updateResourceRequests(requests.get(priority), type, task);

          // Launch the container
          StartContainerRequest scRequest =
              StartContainerRequest.newInstance(createCLC(),
                container.getContainerToken());
          List<StartContainerRequest> list =
              new ArrayList<StartContainerRequest>();
          list.add(scRequest);
          StartContainersRequest allRequests =
              StartContainersRequest.newInstance(list);
          nodeManager.startContainers(allRequests);
          break;
        }
      }
    }
  }
}