Java Code Examples for org.apache.hadoop.yarn.util.resource.Resources#subtract()

The following examples show how to use org.apache.hadoop.yarn.util.resource.Resources#subtract() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: AbstractSchedulerPlanFollower.java    From hadoop with Apache License 2.0 6 votes vote down vote up
private Resource getUnallocatedReservedResources(
    ReservationAllocation reservation) {
  Resource resResource;
  Resource reservationResource = planFollower
      .getReservationQueueResourceIfExists
          (plan, reservation.getReservationId());
  if (reservationResource != null) {
    resResource =
        Resources.subtract(
            reservation.getResourcesAtTime(now),
            reservationResource);
  } else {
    resResource = reservation.getResourcesAtTime(now);
  }
  return resResource;
}
 
Example 2
Source File: AbstractYarnScheduler.java    From big-c with Apache License 2.0 6 votes vote down vote up
public synchronized void adjusctContianerResouce(ContainerId contianerId, Resource resource){
 
 RMContainer container = getRMContainer(contianerId);
 //get original resource
 Resource allocatedResource = container.getAllocatedResource();
 Resource marginalResource=Resources.subtract(allocatedResource, resource);
 
 if(marginalResource.getMemory() > 0 ){		  
  LOG.info("increase memory on continaer"+contianerId.getContainerId()+"by"+marginalResource.getMemory());		  
 }else{		  
  LOG.info("decrease memory on continaer"+contianerId.getContainerId()+"by"+marginalResource.getMemory());
 }
 
 if(marginalResource.getVirtualCores() > 0){
  LOG.info("increase cores on contianer"+contianerId.getContainerId()+"by"+marginalResource.getVirtualCores());
 }else{
  LOG.info("dncrease cores on contianer"+contianerId.getContainerId()+"by"+marginalResource.getVirtualCores());
 }
 
 SchedulerNode schedulerNode = getSchedulerNode(container.getAllocatedNode());
 //now inform the schedulerNode to adjust resurce assumption
 schedulerNode.addAvailableResource(marginalResource);
 //now inform RMContinaer to adjuct its allocatedResource
 container.handle(new RMContainerResourceUpdateEvent(contianerId, RMContainerEventType.RESOURCE_UPDATE, resource));
 	  
}
 
Example 3
Source File: FSAppAttempt.java    From hadoop with Apache License 2.0 5 votes vote down vote up
/**
 * Headroom depends on resources in the cluster, current usage of the
 * queue, queue's fair-share and queue's max-resources.
 */
@Override
public Resource getHeadroom() {
  final FSQueue queue = (FSQueue) this.queue;
  SchedulingPolicy policy = queue.getPolicy();

  Resource queueFairShare = queue.getFairShare();
  Resource queueUsage = queue.getResourceUsage();
  Resource clusterResource = this.scheduler.getClusterResource();
  Resource clusterUsage = this.scheduler.getRootQueueMetrics()
      .getAllocatedResources();

  Resource clusterAvailableResources =
      Resources.subtract(clusterResource, clusterUsage);
  Resource queueMaxAvailableResources =
      Resources.subtract(queue.getMaxShare(), queueUsage);
  Resource maxAvailableResource = Resources.componentwiseMin(
      clusterAvailableResources, queueMaxAvailableResources);

  Resource headroom = policy.getHeadroom(queueFairShare,
      queueUsage, maxAvailableResource);
  if (LOG.isDebugEnabled()) {
    LOG.debug("Headroom calculation for " + this.getName() + ":" +
        "Min(" +
        "(queueFairShare=" + queueFairShare +
        " - queueUsage=" + queueUsage + ")," +
        " maxAvailableResource=" + maxAvailableResource +
        "Headroom=" + headroom);
  }
  return headroom;
}
 
Example 4
Source File: ParentQueue.java    From hadoop with Apache License 2.0 5 votes vote down vote up
private ResourceLimits getResourceLimitsOfChild(CSQueue child,
    Resource clusterResource, ResourceLimits parentLimits) {
  // Set resource-limit of a given child, child.limit =
  // min(my.limit - my.used + child.used, child.max)

  // Parent available resource = parent-limit - parent-used-resource
  Resource parentMaxAvailableResource =
      Resources.subtract(parentLimits.getLimit(), getUsedResources());

  // Child's limit = parent-available-resource + child-used
  Resource childLimit =
      Resources.add(parentMaxAvailableResource, child.getUsedResources());

  // Get child's max resource
  Resource childConfiguredMaxResource =
      Resources.multiplyAndNormalizeDown(resourceCalculator, labelManager
          .getResourceByLabel(RMNodeLabelsManager.NO_LABEL, clusterResource),
          child.getAbsoluteMaximumCapacity(), minimumAllocation);

  // Child's limit should be capped by child configured max resource
  childLimit =
      Resources.min(resourceCalculator, clusterResource, childLimit,
          childConfiguredMaxResource);

  // Normalize before return
  childLimit =
      Resources.roundDown(resourceCalculator, childLimit, minimumAllocation);

  return new ResourceLimits(childLimit);
}
 
Example 5
Source File: ParentQueue.java    From big-c with Apache License 2.0 5 votes vote down vote up
private ResourceLimits getResourceLimitsOfChild(CSQueue child,
     Resource clusterResource, ResourceLimits parentLimits) {
   // Set resource-limit of a given child, child.limit =
   // min(my.limit - my.used + child.used, child.max)//why ?
// my.limit-my-used+child.used 等于在父队列中除去别的子对列所用资源后剩下的资源,有可能别的子对列所用资源超过了他们自身的限制

   // Parent available resource = parent-limit - parent-used-resource
   Resource parentMaxAvailableResource =
       Resources.subtract(parentLimits.getLimit(), getUsedResources());
   
   // Child's limit = parent-available-resource + child-used
   Resource childLimit =
       Resources.add(parentMaxAvailableResource, child.getUsedResources());

   // Get child's max resource
   Resource childConfiguredMaxResource =
       Resources.multiplyAndNormalizeDown(resourceCalculator, labelManager
           .getResourceByLabel(RMNodeLabelsManager.NO_LABEL, clusterResource),
           child.getAbsoluteMaximumCapacity(), minimumAllocation);

   // Child's limit should be capped by child configured max resource
   childLimit =
       Resources.min(resourceCalculator, clusterResource, childLimit,
           childConfiguredMaxResource);

   // Normalize before return
   childLimit =
       Resources.roundDown(resourceCalculator, childLimit, minimumAllocation);

   return new ResourceLimits(childLimit);
 }
 
Example 6
Source File: FSAppAttempt.java    From big-c with Apache License 2.0 5 votes vote down vote up
/**
 * Headroom depends on resources in the cluster, current usage of the
 * queue, queue's fair-share and queue's max-resources.
 */
@Override
public Resource getHeadroom() {
  final FSQueue queue = (FSQueue) this.queue;
  SchedulingPolicy policy = queue.getPolicy();

  Resource queueFairShare = queue.getFairShare();
  Resource queueUsage = queue.getResourceUsage();
  Resource clusterResource = this.scheduler.getClusterResource();
  Resource clusterUsage = this.scheduler.getRootQueueMetrics()
      .getAllocatedResources();

  Resource clusterAvailableResources =
      Resources.subtract(clusterResource, clusterUsage);
  Resource queueMaxAvailableResources =
      Resources.subtract(queue.getMaxShare(), queueUsage);
  Resource maxAvailableResource = Resources.componentwiseMin(
      clusterAvailableResources, queueMaxAvailableResources);

  Resource headroom = policy.getHeadroom(queueFairShare,
      queueUsage, maxAvailableResource);
  if (LOG.isDebugEnabled()) {
    LOG.debug("Headroom calculation for " + this.getName() + ":" +
        "Min(" +
        "(queueFairShare=" + queueFairShare +
        " - queueUsage=" + queueUsage + ")," +
        " maxAvailableResource=" + maxAvailableResource +
        "Headroom=" + headroom);
  }
  return headroom;
}
 
Example 7
Source File: RMContainerImpl.java    From big-c with Apache License 2.0 5 votes vote down vote up
public Resource getCurrentUsedResource(){
 if(isSuspending){
  return Resources.subtract(container.getResource(), preempted);
 }else{
  return container.getResource();
 }
}
 
Example 8
Source File: TestWorkPreservingRMRestart.java    From hadoop with Apache License 2.0 5 votes vote down vote up
private void checkCSQueue(MockRM rm,
    SchedulerApplication<SchedulerApplicationAttempt> app,
    Resource clusterResource, Resource queueResource, Resource usedResource,
    int numContainers)
    throws Exception {
  checkCSLeafQueue(rm, app, clusterResource, queueResource, usedResource,
      numContainers);

  LeafQueue queue = (LeafQueue) app.getQueue();
  Resource availableResources =
      Resources.subtract(queueResource, usedResource);
  // ************ check app headroom ****************
  SchedulerApplicationAttempt schedulerAttempt = app.getCurrentAppAttempt();
  assertEquals(availableResources, schedulerAttempt.getHeadroom());

  // ************* check Queue metrics ************
  QueueMetrics queueMetrics = queue.getMetrics();
  assertMetrics(queueMetrics, 1, 0, 1, 0, 2, availableResources.getMemory(),
      availableResources.getVirtualCores(), usedResource.getMemory(),
      usedResource.getVirtualCores());

  // ************ check user metrics ***********
  QueueMetrics userMetrics =
      queueMetrics.getUserMetrics(app.getUser());
  assertMetrics(userMetrics, 1, 0, 1, 0, 2, availableResources.getMemory(),
      availableResources.getVirtualCores(), usedResource.getMemory(),
      usedResource.getVirtualCores());
}
 
Example 9
Source File: ProportionalCapacityPreemptionPolicy.java    From big-c with Apache License 2.0 4 votes vote down vote up
Resource offer(Resource avail, ResourceCalculator rc,
    Resource clusterResource) {
 	
  Resource absMaxCapIdealAssignedDelta = Resources.componentwiseMax(
                  Resources.subtract(maxCapacity, idealAssigned),
                  Resource.newInstance(0, 0));
  
  // remain = avail - min(avail, (max - assigned), (current + pending - assigned))
  // we have bug here. in some case:
  //(current + pending - assigned).core > avail.core
  //(current + pending - assigned).memo < avail.memo
  //so we get least cores of the three and least memory of the three
  Resource possibleAccepted = 
      Resources.mins(rc, clusterResource, 
          absMaxCapIdealAssignedDelta,
      Resources.mins(rc, clusterResource, avail, Resources.subtract(
          Resources.add(current, pending), idealAssigned)));
  
  //final allocation resource
  Resource finalAccepted = Resources.clone(possibleAccepted);
  //in extrame case where avail cores are more less than the available memory, it may preempt mroe memory
  //Max:      1310720   320
  //avail:    542634    26
  //Delta:    734280    60  
  //Pending:  525312    120
  //current:  576512    260
  //ideal:    576512    260
  //then the accepted will be (525312,26) in which the memory is far more beyond the requirement
  
  if(isSuspended){
	  
  if(dominantResource == Resources.CPU && !Resources.equals(pending,Resources.none())){
	  //pending must be either none() or resource(int ,int)
	  if(avail.getVirtualCores() == 0){
	      //if the dominant resource is cpu, we will stop allocation even we have memory
		  finalAccepted.setMemory(0);
		  //but if we still have more available memory, we can allocate, to avoid preemption
		  //we set memory to current usage
		  int gapMemory = current.getMemory() - idealAssigned.getMemory();
		  if(gapMemory > 0 && possibleAccepted.getMemory() > gapMemory){
			  finalAccepted.setMemory(gapMemory);
			  LOG.info("gap memory: "+gapMemory);
		  }
		  
	  }else{
	  double memoryRatio   = pending.getMemory()*1.0/pending.getVirtualCores();
	  int    ratioedMemory = (int)(memoryRatio*possibleAccepted.getVirtualCores());
	  finalAccepted.setMemory(ratioedMemory < possibleAccepted.getMemory() ? 
			                   ratioedMemory:possibleAccepted.getMemory());
	  }
	  LOG.info("queue: "+queueName+" cpu dominant ");
	  
	  if(finalAccepted.getMemory() < possibleAccepted.getMemory()){
		  LOG.info("previous memory: "+possibleAccepted.getMemory()+"  final memory: "+finalAccepted.getMemory());
	  }
	  
	  
  }else if(dominantResource == Resources.MEMORY && !Resources.equals(pending, Resources.none())){
	  if(avail.getMemory() == 0){
		  finalAccepted.setVirtualCores(0);
		  int gapCores = current.getVirtualCores() - idealAssigned.getVirtualCores();
		  if(gapCores > 0 && possibleAccepted.getVirtualCores() > gapCores){
			  finalAccepted.setVirtualCores(gapCores);
			  LOG.info("gap cores: "+gapCores);
		  }
		  
	  }else{
	  double cpuRatio   = pending.getVirtualCores()*1.0/pending.getMemory();
	  int    ratioedcpu = (int)(cpuRatio*possibleAccepted.getMemory());
	  finalAccepted.setVirtualCores(ratioedcpu < possibleAccepted.getMemory() ? 
			                 ratioedcpu:possibleAccepted.getMemory());
	  }
	  LOG.info("queue: "+queueName+" memory dominant ");
  }else{
	  LOG.info("queue: "+queueName+" empty ");
  }
  
  }
 
  LOG.info("queueName:   "+queueName);
  LOG.info("beforeideal: "+idealAssigned);  
  Resource remain = Resources.subtract(avail, finalAccepted);
  Resources.addTo(idealAssigned, finalAccepted);
  LOG.info("avaul:       "+avail);
  LOG.info("absMaxDelta: "+absMaxCapIdealAssignedDelta);
  LOG.info("max:         "+maxCapacity);
  LOG.info("current:     "+current);
  LOG.info("pending:     "+pending);
  LOG.info("acceped:     "+finalAccepted);
  LOG.info("ideal:       "+idealAssigned);
  
  return remain;
  
}
 
Example 10
Source File: TestFSAppAttempt.java    From big-c with Apache License 2.0 4 votes vote down vote up
@Test
public void testHeadroom() {
  final FairScheduler mockScheduler = Mockito.mock(FairScheduler.class);
  Mockito.when(mockScheduler.getClock()).thenReturn(scheduler.getClock());

  final FSLeafQueue mockQueue = Mockito.mock(FSLeafQueue.class);

  final Resource queueMaxResources = Resource.newInstance(5 * 1024, 3);
  final Resource queueFairShare = Resources.createResource(4096, 2);
  final Resource queueUsage = Resource.newInstance(2048, 2);

  final Resource queueStarvation =
      Resources.subtract(queueFairShare, queueUsage);
  final Resource queueMaxResourcesAvailable =
      Resources.subtract(queueMaxResources, queueUsage);

  final Resource clusterResource = Resources.createResource(8192, 8);
  final Resource clusterUsage = Resources.createResource(2048, 2);
  final Resource clusterAvailable =
      Resources.subtract(clusterResource, clusterUsage);

  final QueueMetrics fakeRootQueueMetrics = Mockito.mock(QueueMetrics.class);

  Mockito.when(mockQueue.getMaxShare()).thenReturn(queueMaxResources);
  Mockito.when(mockQueue.getFairShare()).thenReturn(queueFairShare);
  Mockito.when(mockQueue.getResourceUsage()).thenReturn(queueUsage);
  Mockito.when(mockScheduler.getClusterResource()).thenReturn
      (clusterResource);
  Mockito.when(fakeRootQueueMetrics.getAllocatedResources()).thenReturn
      (clusterUsage);
  Mockito.when(mockScheduler.getRootQueueMetrics()).thenReturn
      (fakeRootQueueMetrics);

  ApplicationAttemptId applicationAttemptId = createAppAttemptId(1, 1);
  RMContext rmContext = resourceManager.getRMContext();
  FSAppAttempt schedulerApp =
      new FSAppAttempt(mockScheduler, applicationAttemptId, "user1", mockQueue ,
          null, rmContext);

  // Min of Memory and CPU across cluster and queue is used in
  // DominantResourceFairnessPolicy
  Mockito.when(mockQueue.getPolicy()).thenReturn(SchedulingPolicy
      .getInstance(DominantResourceFairnessPolicy.class));
  verifyHeadroom(schedulerApp,
      min(queueStarvation.getMemory(),
          clusterAvailable.getMemory(),
          queueMaxResourcesAvailable.getMemory()),
      min(queueStarvation.getVirtualCores(),
          clusterAvailable.getVirtualCores(),
          queueMaxResourcesAvailable.getVirtualCores())
  );

  // Fair and Fifo ignore CPU of queue, so use cluster available CPU
  Mockito.when(mockQueue.getPolicy()).thenReturn(SchedulingPolicy
      .getInstance(FairSharePolicy.class));
  verifyHeadroom(schedulerApp,
      min(queueStarvation.getMemory(),
          clusterAvailable.getMemory(),
          queueMaxResourcesAvailable.getMemory()),
      Math.min(
          clusterAvailable.getVirtualCores(),
          queueMaxResourcesAvailable.getVirtualCores())
  );

  Mockito.when(mockQueue.getPolicy()).thenReturn(SchedulingPolicy
      .getInstance(FifoPolicy.class));
  verifyHeadroom(schedulerApp,
      min(queueStarvation.getMemory(),
          clusterAvailable.getMemory(),
          queueMaxResourcesAvailable.getMemory()),
      Math.min(
          clusterAvailable.getVirtualCores(),
          queueMaxResourcesAvailable.getVirtualCores())
  );
}
 
Example 11
Source File: RMContainerAllocator.java    From big-c with Apache License 2.0 4 votes vote down vote up
@Private
@VisibleForTesting
void preemptReducesIfNeeded() {
  if (reduceResourceRequest.equals(Resources.none())) {
    return; // no reduces
  }
  //check if reduces have taken over the whole cluster and there are 
  //unassigned maps
  if (scheduledRequests.maps.size() > 0) {
    Resource resourceLimit = getResourceLimit();
    Resource availableResourceForMap =
        Resources.subtract(
          resourceLimit,
          Resources.multiply(reduceResourceRequest,
            assignedRequests.reduces.size()
                - assignedRequests.preemptionWaitingReduces.size()));
    // availableMemForMap must be sufficient to run at least 1 map
    if (ResourceCalculatorUtils.computeAvailableContainers(availableResourceForMap,
      mapResourceRequest, getSchedulerResourceTypes()) <= 0) {
      // to make sure new containers are given to maps and not reduces
      // ramp down all scheduled reduces if any
      // (since reduces are scheduled at higher priority than maps)
      LOG.info("Ramping down all scheduled reduces:"
          + scheduledRequests.reduces.size());
      for (ContainerRequest req : scheduledRequests.reduces.values()) {
        pendingReduces.add(req);
      }
      scheduledRequests.reduces.clear();
 
      //do further checking to find the number of map requests that were
      //hanging around for a while
      int hangingMapRequests = getNumOfHangingRequests(scheduledRequests.maps);
      if (hangingMapRequests > 0) {
        // preempt for making space for at least one map
        int preemptionReduceNumForOneMap =
            ResourceCalculatorUtils.divideAndCeilContainers(mapResourceRequest,
              reduceResourceRequest, getSchedulerResourceTypes());
        int preemptionReduceNumForPreemptionLimit =
            ResourceCalculatorUtils.divideAndCeilContainers(
              Resources.multiply(resourceLimit, maxReducePreemptionLimit),
              reduceResourceRequest, getSchedulerResourceTypes());
        int preemptionReduceNumForAllMaps =
            ResourceCalculatorUtils.divideAndCeilContainers(
              Resources.multiply(mapResourceRequest, hangingMapRequests),
              reduceResourceRequest, getSchedulerResourceTypes());
        int toPreempt =
            Math.min(Math.max(preemptionReduceNumForOneMap,
              preemptionReduceNumForPreemptionLimit),
              preemptionReduceNumForAllMaps);

        LOG.info("Going to preempt " + toPreempt
            + " due to lack of space for maps");
        assignedRequests.preemptReduce(toPreempt);
      }
    }
  }
}
 
Example 12
Source File: CSQueueUtils.java    From big-c with Apache License 2.0 4 votes vote down vote up
@Lock(CSQueue.class)
public static void updateQueueStatistics(
    final ResourceCalculator calculator,
    final CSQueue childQueue, final CSQueue parentQueue, 
    final Resource clusterResource, final Resource minimumAllocation) {
  Resource queueLimit = Resources.none();
  Resource usedResources = childQueue.getUsedResources();
  
  float absoluteUsedCapacity = 0.0f;
  float usedCapacity = 0.0f;

  if (Resources.greaterThan(
      calculator, clusterResource, clusterResource, Resources.none())) {
    queueLimit = 
        Resources.multiply(clusterResource, childQueue.getAbsoluteCapacity());
    
    //absoluteUsedCapacity = 
    //    Resources.divide(calculator, clusterResource, 
    //        usedResources, clusterResource);
    
    absoluteUsedCapacity = (float) ((usedResources.getVirtualCores()*1.0)/(clusterResource.getVirtualCores()*1.0));
  
    usedCapacity = 
        Resources.equals(queueLimit, Resources.none()) ? 0 :
        Resources.divide(calculator, clusterResource, 
            usedResources, queueLimit);
  }

  childQueue.setUsedCapacity(usedCapacity);
  childQueue.setAbsoluteUsedCapacity(absoluteUsedCapacity);
  
  Resource available = Resources.subtract(queueLimit, usedResources);
  childQueue.getMetrics().setAvailableResourcesToQueue(
      Resources.max(
          calculator, 
          clusterResource, 
          available, 
          Resources.none()
          )
      );
 }
 
Example 13
Source File: RLESparseResourceAllocation.java    From big-c with Apache License 2.0 4 votes vote down vote up
/**
 * Removes a resource for the specified interval
 * 
 * @param reservationInterval the interval for which the resource is to be
 *          removed
 * @param capacity the resource to be removed
 * @return true if removal is successful, false otherwise
 */
public boolean removeInterval(ReservationInterval reservationInterval,
    ReservationRequest capacity) {
  Resource totCap =
      Resources.multiply(capacity.getCapability(),
          (float) capacity.getNumContainers());
  if (totCap.equals(ZERO_RESOURCE)) {
    return true;
  }
  writeLock.lock();
  try {
    long startKey = reservationInterval.getStartTime();
    long endKey = reservationInterval.getEndTime();
    // update the start key
    NavigableMap<Long, Resource> ticks =
        cumulativeCapacity.headMap(endKey, false);
    // Decrease all the capacities of overlapping intervals
    SortedMap<Long, Resource> overlapSet = ticks.tailMap(startKey);
    if (overlapSet != null && !overlapSet.isEmpty()) {
      Resource updatedCapacity = Resource.newInstance(0, 0);
      long currentKey = -1;
      for (Iterator<Entry<Long, Resource>> overlapEntries =
          overlapSet.entrySet().iterator(); overlapEntries.hasNext();) {
        Entry<Long, Resource> entry = overlapEntries.next();
        currentKey = entry.getKey();
        updatedCapacity = Resources.subtract(entry.getValue(), totCap);
        // update each entry between start and end key
        cumulativeCapacity.put(currentKey, updatedCapacity);
      }
      // Remove the first overlap entry if it is same as previous after
      // updation
      Long firstKey = overlapSet.firstKey();
      if (isSameAsPrevious(firstKey, overlapSet.get(firstKey))) {
        cumulativeCapacity.remove(firstKey);
      }
      // Remove the next entry if it is same as end entry after updation
      if ((currentKey != -1) && (isSameAsNext(currentKey, updatedCapacity))) {
        cumulativeCapacity.remove(cumulativeCapacity.higherKey(currentKey));
      }
    }
    return true;
  } finally {
    writeLock.unlock();
  }
}
 
Example 14
Source File: RMContainerAllocator.java    From hadoop with Apache License 2.0 4 votes vote down vote up
@Private
@VisibleForTesting
void preemptReducesIfNeeded() {
  if (reduceResourceRequest.equals(Resources.none())) {
    return; // no reduces
  }
  //check if reduces have taken over the whole cluster and there are 
  //unassigned maps
  if (scheduledRequests.maps.size() > 0) {
    Resource resourceLimit = getResourceLimit();
    Resource availableResourceForMap =
        Resources.subtract(
          resourceLimit,
          Resources.multiply(reduceResourceRequest,
            assignedRequests.reduces.size()
                - assignedRequests.preemptionWaitingReduces.size()));
    // availableMemForMap must be sufficient to run at least 1 map
    if (ResourceCalculatorUtils.computeAvailableContainers(availableResourceForMap,
      mapResourceRequest, getSchedulerResourceTypes()) <= 0) {
      // to make sure new containers are given to maps and not reduces
      // ramp down all scheduled reduces if any
      // (since reduces are scheduled at higher priority than maps)
      LOG.info("Ramping down all scheduled reduces:"
          + scheduledRequests.reduces.size());
      for (ContainerRequest req : scheduledRequests.reduces.values()) {
        pendingReduces.add(req);
      }
      scheduledRequests.reduces.clear();
 
      //do further checking to find the number of map requests that were
      //hanging around for a while
      int hangingMapRequests = getNumOfHangingRequests(scheduledRequests.maps);
      if (hangingMapRequests > 0) {
        // preempt for making space for at least one map
        int preemptionReduceNumForOneMap =
            ResourceCalculatorUtils.divideAndCeilContainers(mapResourceRequest,
              reduceResourceRequest, getSchedulerResourceTypes());
        int preemptionReduceNumForPreemptionLimit =
            ResourceCalculatorUtils.divideAndCeilContainers(
              Resources.multiply(resourceLimit, maxReducePreemptionLimit),
              reduceResourceRequest, getSchedulerResourceTypes());
        int preemptionReduceNumForAllMaps =
            ResourceCalculatorUtils.divideAndCeilContainers(
              Resources.multiply(mapResourceRequest, hangingMapRequests),
              reduceResourceRequest, getSchedulerResourceTypes());
        int toPreempt =
            Math.min(Math.max(preemptionReduceNumForOneMap,
              preemptionReduceNumForPreemptionLimit),
              preemptionReduceNumForAllMaps);

        LOG.info("Going to preempt " + toPreempt
            + " due to lack of space for maps");
        assignedRequests.preemptReduce(toPreempt);
      }
    }
  }
}
 
Example 15
Source File: TestFSAppAttempt.java    From hadoop with Apache License 2.0 4 votes vote down vote up
@Test
public void testHeadroom() {
  final FairScheduler mockScheduler = Mockito.mock(FairScheduler.class);
  Mockito.when(mockScheduler.getClock()).thenReturn(scheduler.getClock());

  final FSLeafQueue mockQueue = Mockito.mock(FSLeafQueue.class);

  final Resource queueMaxResources = Resource.newInstance(5 * 1024, 3, 3);
  final Resource queueFairShare = Resources.createResource(4096, 2, 2);
  final Resource queueUsage = Resource.newInstance(2048, 2, 2);

  final Resource queueStarvation =
      Resources.subtract(queueFairShare, queueUsage);
  final Resource queueMaxResourcesAvailable =
      Resources.subtract(queueMaxResources, queueUsage);

  final Resource clusterResource = Resources.createResource(8192, 8, 8);
  final Resource clusterUsage = Resources.createResource(2048, 2, 2);
  final Resource clusterAvailable =
      Resources.subtract(clusterResource, clusterUsage);

  final QueueMetrics fakeRootQueueMetrics = Mockito.mock(QueueMetrics.class);

  Mockito.when(mockQueue.getMaxShare()).thenReturn(queueMaxResources);
  Mockito.when(mockQueue.getFairShare()).thenReturn(queueFairShare);
  Mockito.when(mockQueue.getResourceUsage()).thenReturn(queueUsage);
  Mockito.when(mockScheduler.getClusterResource()).thenReturn
      (clusterResource);
  Mockito.when(fakeRootQueueMetrics.getAllocatedResources()).thenReturn
      (clusterUsage);
  Mockito.when(mockScheduler.getRootQueueMetrics()).thenReturn
      (fakeRootQueueMetrics);

  ApplicationAttemptId applicationAttemptId = createAppAttemptId(1, 1);
  RMContext rmContext = resourceManager.getRMContext();
  FSAppAttempt schedulerApp =
      new FSAppAttempt(mockScheduler, applicationAttemptId, "user1", mockQueue ,
          null, rmContext);

  // Min of Memory and CPU across cluster and queue is used in
  // DominantResourceFairnessPolicy
  Mockito.when(mockQueue.getPolicy()).thenReturn(SchedulingPolicy
      .getInstance(DominantResourceFairnessPolicy.class));
  verifyHeadroom(schedulerApp,
      min(queueStarvation.getMemory(),
          clusterAvailable.getMemory(),
          queueMaxResourcesAvailable.getMemory()),
      min(queueStarvation.getVirtualCores(),
          clusterAvailable.getVirtualCores(),
          queueMaxResourcesAvailable.getVirtualCores())
  );

  // Fair and Fifo ignore CPU of queue, so use cluster available CPU
  Mockito.when(mockQueue.getPolicy()).thenReturn(SchedulingPolicy
      .getInstance(FairSharePolicy.class));
  verifyHeadroom(schedulerApp,
      min(queueStarvation.getMemory(),
          clusterAvailable.getMemory(),
          queueMaxResourcesAvailable.getMemory()),
      Math.min(
          clusterAvailable.getVirtualCores(),
          queueMaxResourcesAvailable.getVirtualCores())
  );

  Mockito.when(mockQueue.getPolicy()).thenReturn(SchedulingPolicy
      .getInstance(FifoPolicy.class));
  verifyHeadroom(schedulerApp,
      min(queueStarvation.getMemory(),
          clusterAvailable.getMemory(),
          queueMaxResourcesAvailable.getMemory()),
      Math.min(
          clusterAvailable.getVirtualCores(),
          queueMaxResourcesAvailable.getVirtualCores())
  );
}
 
Example 16
Source File: YarnTaskSchedulerService.java    From incubator-tez with Apache License 2.0 4 votes vote down vote up
@Override
public void onContainersCompleted(List<ContainerStatus> statuses) {
  if (isStopped.get()) {
    return;
  }
  Map<Object, ContainerStatus> appContainerStatus =
                      new HashMap<Object, ContainerStatus>(statuses.size());
  synchronized (this) {
    for(ContainerStatus containerStatus : statuses) {
      ContainerId completedId = containerStatus.getContainerId();
      HeldContainer delayedContainer = heldContainers.get(completedId);

      Object task = releasedContainers.remove(completedId);
      if(task != null){
        if (delayedContainer != null) {
          LOG.warn("Held container should be null since releasedContainer is not");
        }
        // TODO later we may want to check if exit code matched expectation
        // e.g. successful container should not come back fail exit code after
        // being released
        // completion of a container we had released earlier
        // an allocated container completed. notify app
        LOG.info("Released container completed:" + completedId +
                 " last allocated to task: " + task);
        appContainerStatus.put(task, containerStatus);
        continue;
      }

      // not found in released containers. check currently allocated containers
      // no need to release this container as the RM has already completed it
      task = unAssignContainer(completedId, false);
      if (delayedContainer != null) {
        heldContainers.remove(completedId);
        Resources.subtract(allocatedResources, delayedContainer.getContainer().getResource());
      } else {
        LOG.warn("Held container expected to be not null for a non-AM-released container");
      }
      if(task != null) {
        // completion of a container we have allocated currently
        // an allocated container completed. notify app
        LOG.info("Allocated container completed:" + completedId +
                 " last allocated to task: " + task);
        appContainerStatus.put(task, containerStatus);
        continue;
      }

      // container neither allocated nor released
      LOG.info("Ignoring unknown container: " + containerStatus.getContainerId());
    }
  }

  // upcall to app must be outside locks
  for (Entry<Object, ContainerStatus> entry : appContainerStatus.entrySet()) {
    appClientDelegate.containerCompleted(entry.getKey(), entry.getValue());
  }
}
 
Example 17
Source File: ProportionalCapacityPreemptionPolicy.java    From hadoop with Apache License 2.0 4 votes vote down vote up
/**
 * Given a set of queues compute the fix-point distribution of unassigned
 * resources among them. As pending request of a queue are exhausted, the
 * queue is removed from the set and remaining capacity redistributed among
 * remaining queues. The distribution is weighted based on guaranteed
 * capacity, unless asked to ignoreGuarantee, in which case resources are
 * distributed uniformly.
 */
private void computeFixpointAllocation(ResourceCalculator rc,
    Resource tot_guarant, Collection<TempQueue> qAlloc, Resource unassigned, 
    boolean ignoreGuarantee) {
  // Prior to assigning the unused resources, process each queue as follows:
  // If current > guaranteed, idealAssigned = guaranteed + untouchable extra
  // Else idealAssigned = current;
  // Subtract idealAssigned resources from unassigned.
  // If the queue has all of its needs met (that is, if 
  // idealAssigned >= current + pending), remove it from consideration.
  // Sort queues from most under-guaranteed to most over-guaranteed.
  TQComparator tqComparator = new TQComparator(rc, tot_guarant);
  PriorityQueue<TempQueue> orderedByNeed =
                               new PriorityQueue<TempQueue>(10,tqComparator);
  for (Iterator<TempQueue> i = qAlloc.iterator(); i.hasNext();) {
    TempQueue q = i.next();
    if (Resources.greaterThan(rc, tot_guarant, q.current, q.guaranteed)) {
      q.idealAssigned = Resources.add(q.guaranteed, q.untouchableExtra);
    } else {
      q.idealAssigned = Resources.clone(q.current);
    }
    Resources.subtractFrom(unassigned, q.idealAssigned);
    // If idealAssigned < (current + pending), q needs more resources, so
    // add it to the list of underserved queues, ordered by need.
    Resource curPlusPend = Resources.add(q.current, q.pending);
    if (Resources.lessThan(rc, tot_guarant, q.idealAssigned, curPlusPend)) {
      orderedByNeed.add(q);
    }
  }

  //assign all cluster resources until no more demand, or no resources are left
  while (!orderedByNeed.isEmpty()
     && Resources.greaterThan(rc,tot_guarant, unassigned,Resources.none())) {
    Resource wQassigned = Resource.newInstance(0, 0, 0);
    // we compute normalizedGuarantees capacity based on currently active
    // queues
    resetCapacity(rc, unassigned, orderedByNeed, ignoreGuarantee);

    // For each underserved queue (or set of queues if multiple are equally
    // underserved), offer its share of the unassigned resources based on its
    // normalized guarantee. After the offer, if the queue is not satisfied,
    // place it back in the ordered list of queues, recalculating its place
    // in the order of most under-guaranteed to most over-guaranteed. In this
    // way, the most underserved queue(s) are always given resources first.
    Collection<TempQueue> underserved =
        getMostUnderservedQueues(orderedByNeed, tqComparator);
    for (Iterator<TempQueue> i = underserved.iterator(); i.hasNext();) {
      TempQueue sub = i.next();
      Resource wQavail = Resources.multiplyAndNormalizeUp(rc,
          unassigned, sub.normalizedGuarantee, Resource.newInstance(1, 1, 0));
      Resource wQidle = sub.offer(wQavail, rc, tot_guarant);
      Resource wQdone = Resources.subtract(wQavail, wQidle);

      if (Resources.greaterThan(rc, tot_guarant,
            wQdone, Resources.none())) {
        // The queue is still asking for more. Put it back in the priority
        // queue, recalculating its order based on need.
        orderedByNeed.add(sub);
      }
      Resources.addTo(wQassigned, wQdone);
    }
    Resources.subtractFrom(unassigned, wQassigned);
  }
}
 
Example 18
Source File: LeafQueue.java    From hadoop with Apache License 2.0 4 votes vote down vote up
@Private
protected synchronized boolean canAssignToUser(Resource clusterResource,
    String userName, Resource limit, FiCaSchedulerApp application,
    Set<String> requestLabels, ResourceLimits currentResoureLimits) {
  User user = getUser(userName);
  
  String label = CommonNodeLabelsManager.NO_LABEL;
  if (requestLabels != null && !requestLabels.isEmpty()) {
    label = requestLabels.iterator().next();
  }

  // Note: We aren't considering the current request since there is a fixed
  // overhead of the AM, but it's a > check, not a >= check, so...
  if (Resources
      .greaterThan(resourceCalculator, clusterResource,
          user.getUsed(label),
          limit)) {
    // if enabled, check to see if could we potentially use this node instead
    // of a reserved node if the application has reserved containers
    if (this.reservationsContinueLooking
        && label.equals(CommonNodeLabelsManager.NO_LABEL)) {
      if (Resources.lessThanOrEqual(
          resourceCalculator,
          clusterResource,
          Resources.subtract(user.getUsed(), application.getCurrentReservation()),
          limit)) {

        if (LOG.isDebugEnabled()) {
          LOG.debug("User " + userName + " in queue " + getQueueName()
              + " will exceed limit based on reservations - " + " consumed: "
              + user.getUsed() + " reserved: "
              + application.getCurrentReservation() + " limit: " + limit);
        }
        Resource amountNeededToUnreserve = Resources.subtract(user.getUsed(label), limit);
        // we can only acquire a new container if we unreserve first since we ignored the
        // user limit. Choose the max of user limit or what was previously set by max
        // capacity.
        currentResoureLimits.setAmountNeededUnreserve(Resources.max(resourceCalculator,
            clusterResource, currentResoureLimits.getAmountNeededUnreserve(),
            amountNeededToUnreserve));
        return true;
      }
    }
    if (LOG.isDebugEnabled()) {
      LOG.debug("User " + userName + " in queue " + getQueueName()
          + " will exceed limit - " + " consumed: "
          + user.getUsed() + " limit: " + limit);
    }
    return false;
  }
  return true;
}
 
Example 19
Source File: AbstractCSQueue.java    From hadoop with Apache License 2.0 4 votes vote down vote up
synchronized boolean canAssignToThisQueue(Resource clusterResource,
    Set<String> nodeLabels, ResourceLimits currentResourceLimits,
    Resource nowRequired, Resource resourceCouldBeUnreserved) {
  // Get label of this queue can access, it's (nodeLabel AND queueLabel)
  Set<String> labelCanAccess;
  if (null == nodeLabels || nodeLabels.isEmpty()) {
    labelCanAccess = new HashSet<String>();
    // Any queue can always access any node without label
    labelCanAccess.add(RMNodeLabelsManager.NO_LABEL);
  } else {
    labelCanAccess = new HashSet<String>(
        accessibleLabels.contains(CommonNodeLabelsManager.ANY) ? nodeLabels
            : Sets.intersection(accessibleLabels, nodeLabels));
  }
  
  for (String label : labelCanAccess) {
    // New total resource = used + required
    Resource newTotalResource =
        Resources.add(queueUsage.getUsed(label), nowRequired);

    Resource currentLimitResource =
        getCurrentLimitResource(label, clusterResource, currentResourceLimits);

    if (Resources.greaterThan(resourceCalculator, clusterResource,
        newTotalResource, currentLimitResource)) {

      // if reservation continous looking enabled, check to see if could we
      // potentially use this node instead of a reserved node if the application
      // has reserved containers.
      // TODO, now only consider reservation cases when the node has no label
      if (this.reservationsContinueLooking
          && label.equals(RMNodeLabelsManager.NO_LABEL)
          && Resources.greaterThan(resourceCalculator, clusterResource,
          resourceCouldBeUnreserved, Resources.none())) {
        // resource-without-reserved = used - reserved
        Resource newTotalWithoutReservedResource =
            Resources.subtract(newTotalResource, resourceCouldBeUnreserved);

        // when total-used-without-reserved-resource < currentLimit, we still
        // have chance to allocate on this node by unreserving some containers
        if (Resources.lessThan(resourceCalculator, clusterResource,
            newTotalWithoutReservedResource, currentLimitResource)) {
          if (LOG.isDebugEnabled()) {
            LOG.debug("try to use reserved: " + getQueueName()
                + " usedResources: " + queueUsage.getUsed()
                + ", clusterResources: " + clusterResource
                + ", reservedResources: " + resourceCouldBeUnreserved
                + ", capacity-without-reserved: "
                + newTotalWithoutReservedResource + ", maxLimitCapacity: "
                + currentLimitResource);
          }
          currentResourceLimits.setAmountNeededUnreserve(Resources.subtract(newTotalResource,
              currentLimitResource));
          return true;
        }
      }
      if (LOG.isDebugEnabled()) {
        LOG.debug(getQueueName()
            + "Check assign to queue, label=" + label
            + " usedResources: " + queueUsage.getUsed(label)
            + " clusterResources: " + clusterResource
            + " currentUsedCapacity "
            + Resources.divide(resourceCalculator, clusterResource,
            queueUsage.getUsed(label),
            labelManager.getResourceByLabel(label, clusterResource))
            + " max-capacity: "
            + queueCapacities.getAbsoluteMaximumCapacity(label)
            + ")");
      }
      return false;
    }
    return true;
  }
  
  // Actually, this will not happen, since labelCanAccess will be always
  // non-empty
  return false;
}
 
Example 20
Source File: AbstractCSQueue.java    From big-c with Apache License 2.0 4 votes vote down vote up
synchronized boolean canAssignToThisQueue(Resource clusterResource,
    Set<String> nodeLabels, ResourceLimits currentResourceLimits,
    Resource nowRequired, Resource resourceCouldBeUnreserved) {
  // Get label of this queue can access, it's (nodeLabel AND queueLabel)
  Set<String> labelCanAccess;
  if (null == nodeLabels || nodeLabels.isEmpty()) {
    labelCanAccess = new HashSet<String>();
    // Any queue can always access any node without label
    labelCanAccess.add(RMNodeLabelsManager.NO_LABEL);
  } else {
    labelCanAccess = new HashSet<String>(
        accessibleLabels.contains(CommonNodeLabelsManager.ANY) ? nodeLabels
            : Sets.intersection(accessibleLabels, nodeLabels));
  }
  
  for (String label : labelCanAccess) {
    // New total resource = used + required
    Resource newTotalResource =
        Resources.add(queueUsage.getUsed(label), nowRequired);

    Resource currentLimitResource =
        getCurrentLimitResource(label, clusterResource, currentResourceLimits);

    //current resource is much more than currentlimit
    if (Resources.greaterThan(resourceCalculator, clusterResource,
        newTotalResource, currentLimitResource)) {

      // if reservation continous looking enabled, check to see if could we
      // potentially use this node instead of a reserved node if the application
      // has reserved containers.
      // TODO, now only consider reservation cases when the node has no label
      if (this.reservationsContinueLooking
          && label.equals(RMNodeLabelsManager.NO_LABEL)
          && Resources.greaterThan(resourceCalculator, clusterResource,
          resourceCouldBeUnreserved, Resources.none())) {
        // resource-without-reserved = used - reserved
        Resource newTotalWithoutReservedResource =
            Resources.subtract(newTotalResource, resourceCouldBeUnreserved);

        // when total-used-without-reserved-resource < currentLimit, we still
        // have chance to allocate on this node by unreserving some containers
        if (Resources.lessThan(resourceCalculator, clusterResource,
            newTotalWithoutReservedResource, currentLimitResource)) {
          if (LOG.isDebugEnabled()) {
            LOG.debug("try to use reserved: " + getQueueName()
                + " usedResources: " + queueUsage.getUsed()
                + ", clusterResources: " + clusterResource
                + ", reservedResources: " + resourceCouldBeUnreserved
                + ", capacity-without-reserved: "
                + newTotalWithoutReservedResource + ", maxLimitCapacity: "
                + currentLimitResource);
          }
          currentResourceLimits.setAmountNeededUnreserve(Resources.subtract(newTotalResource,
              currentLimitResource));
          return true;
        }
      }
      if (LOG.isDebugEnabled()) {
        LOG.debug(getQueueName()
            + "Check assign to queue, label=" + label
            + " usedResources: " + queueUsage.getUsed(label)
            + " clusterResources: " + clusterResource
            + " currentUsedCapacity "
            + Resources.divide(resourceCalculator, clusterResource,
            queueUsage.getUsed(label),
            labelManager.getResourceByLabel(label, clusterResource))
            + " max-capacity: "
            + queueCapacities.getAbsoluteMaximumCapacity(label)
            + ")");
      }
      return false;
    }
    return true;
  }
  
  // Actually, this will not happen, since labelCanAccess will be always
  // non-empty
  return false;
}