Java Code Examples for org.apache.hadoop.yarn.util.resource.Resources#subtract()
The following examples show how to use
org.apache.hadoop.yarn.util.resource.Resources#subtract() .
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: AbstractSchedulerPlanFollower.java From hadoop with Apache License 2.0 | 6 votes |
private Resource getUnallocatedReservedResources( ReservationAllocation reservation) { Resource resResource; Resource reservationResource = planFollower .getReservationQueueResourceIfExists (plan, reservation.getReservationId()); if (reservationResource != null) { resResource = Resources.subtract( reservation.getResourcesAtTime(now), reservationResource); } else { resResource = reservation.getResourcesAtTime(now); } return resResource; }
Example 2
Source File: AbstractYarnScheduler.java From big-c with Apache License 2.0 | 6 votes |
public synchronized void adjusctContianerResouce(ContainerId contianerId, Resource resource){ RMContainer container = getRMContainer(contianerId); //get original resource Resource allocatedResource = container.getAllocatedResource(); Resource marginalResource=Resources.subtract(allocatedResource, resource); if(marginalResource.getMemory() > 0 ){ LOG.info("increase memory on continaer"+contianerId.getContainerId()+"by"+marginalResource.getMemory()); }else{ LOG.info("decrease memory on continaer"+contianerId.getContainerId()+"by"+marginalResource.getMemory()); } if(marginalResource.getVirtualCores() > 0){ LOG.info("increase cores on contianer"+contianerId.getContainerId()+"by"+marginalResource.getVirtualCores()); }else{ LOG.info("dncrease cores on contianer"+contianerId.getContainerId()+"by"+marginalResource.getVirtualCores()); } SchedulerNode schedulerNode = getSchedulerNode(container.getAllocatedNode()); //now inform the schedulerNode to adjust resurce assumption schedulerNode.addAvailableResource(marginalResource); //now inform RMContinaer to adjuct its allocatedResource container.handle(new RMContainerResourceUpdateEvent(contianerId, RMContainerEventType.RESOURCE_UPDATE, resource)); }
Example 3
Source File: FSAppAttempt.java From hadoop with Apache License 2.0 | 5 votes |
/** * Headroom depends on resources in the cluster, current usage of the * queue, queue's fair-share and queue's max-resources. */ @Override public Resource getHeadroom() { final FSQueue queue = (FSQueue) this.queue; SchedulingPolicy policy = queue.getPolicy(); Resource queueFairShare = queue.getFairShare(); Resource queueUsage = queue.getResourceUsage(); Resource clusterResource = this.scheduler.getClusterResource(); Resource clusterUsage = this.scheduler.getRootQueueMetrics() .getAllocatedResources(); Resource clusterAvailableResources = Resources.subtract(clusterResource, clusterUsage); Resource queueMaxAvailableResources = Resources.subtract(queue.getMaxShare(), queueUsage); Resource maxAvailableResource = Resources.componentwiseMin( clusterAvailableResources, queueMaxAvailableResources); Resource headroom = policy.getHeadroom(queueFairShare, queueUsage, maxAvailableResource); if (LOG.isDebugEnabled()) { LOG.debug("Headroom calculation for " + this.getName() + ":" + "Min(" + "(queueFairShare=" + queueFairShare + " - queueUsage=" + queueUsage + ")," + " maxAvailableResource=" + maxAvailableResource + "Headroom=" + headroom); } return headroom; }
Example 4
Source File: ParentQueue.java From hadoop with Apache License 2.0 | 5 votes |
private ResourceLimits getResourceLimitsOfChild(CSQueue child, Resource clusterResource, ResourceLimits parentLimits) { // Set resource-limit of a given child, child.limit = // min(my.limit - my.used + child.used, child.max) // Parent available resource = parent-limit - parent-used-resource Resource parentMaxAvailableResource = Resources.subtract(parentLimits.getLimit(), getUsedResources()); // Child's limit = parent-available-resource + child-used Resource childLimit = Resources.add(parentMaxAvailableResource, child.getUsedResources()); // Get child's max resource Resource childConfiguredMaxResource = Resources.multiplyAndNormalizeDown(resourceCalculator, labelManager .getResourceByLabel(RMNodeLabelsManager.NO_LABEL, clusterResource), child.getAbsoluteMaximumCapacity(), minimumAllocation); // Child's limit should be capped by child configured max resource childLimit = Resources.min(resourceCalculator, clusterResource, childLimit, childConfiguredMaxResource); // Normalize before return childLimit = Resources.roundDown(resourceCalculator, childLimit, minimumAllocation); return new ResourceLimits(childLimit); }
Example 5
Source File: ParentQueue.java From big-c with Apache License 2.0 | 5 votes |
private ResourceLimits getResourceLimitsOfChild(CSQueue child, Resource clusterResource, ResourceLimits parentLimits) { // Set resource-limit of a given child, child.limit = // min(my.limit - my.used + child.used, child.max)//why ? // my.limit-my-used+child.used 等于在父队列中除去别的子对列所用资源后剩下的资源,有可能别的子对列所用资源超过了他们自身的限制 // Parent available resource = parent-limit - parent-used-resource Resource parentMaxAvailableResource = Resources.subtract(parentLimits.getLimit(), getUsedResources()); // Child's limit = parent-available-resource + child-used Resource childLimit = Resources.add(parentMaxAvailableResource, child.getUsedResources()); // Get child's max resource Resource childConfiguredMaxResource = Resources.multiplyAndNormalizeDown(resourceCalculator, labelManager .getResourceByLabel(RMNodeLabelsManager.NO_LABEL, clusterResource), child.getAbsoluteMaximumCapacity(), minimumAllocation); // Child's limit should be capped by child configured max resource childLimit = Resources.min(resourceCalculator, clusterResource, childLimit, childConfiguredMaxResource); // Normalize before return childLimit = Resources.roundDown(resourceCalculator, childLimit, minimumAllocation); return new ResourceLimits(childLimit); }
Example 6
Source File: FSAppAttempt.java From big-c with Apache License 2.0 | 5 votes |
/** * Headroom depends on resources in the cluster, current usage of the * queue, queue's fair-share and queue's max-resources. */ @Override public Resource getHeadroom() { final FSQueue queue = (FSQueue) this.queue; SchedulingPolicy policy = queue.getPolicy(); Resource queueFairShare = queue.getFairShare(); Resource queueUsage = queue.getResourceUsage(); Resource clusterResource = this.scheduler.getClusterResource(); Resource clusterUsage = this.scheduler.getRootQueueMetrics() .getAllocatedResources(); Resource clusterAvailableResources = Resources.subtract(clusterResource, clusterUsage); Resource queueMaxAvailableResources = Resources.subtract(queue.getMaxShare(), queueUsage); Resource maxAvailableResource = Resources.componentwiseMin( clusterAvailableResources, queueMaxAvailableResources); Resource headroom = policy.getHeadroom(queueFairShare, queueUsage, maxAvailableResource); if (LOG.isDebugEnabled()) { LOG.debug("Headroom calculation for " + this.getName() + ":" + "Min(" + "(queueFairShare=" + queueFairShare + " - queueUsage=" + queueUsage + ")," + " maxAvailableResource=" + maxAvailableResource + "Headroom=" + headroom); } return headroom; }
Example 7
Source File: RMContainerImpl.java From big-c with Apache License 2.0 | 5 votes |
public Resource getCurrentUsedResource(){ if(isSuspending){ return Resources.subtract(container.getResource(), preempted); }else{ return container.getResource(); } }
Example 8
Source File: TestWorkPreservingRMRestart.java From hadoop with Apache License 2.0 | 5 votes |
private void checkCSQueue(MockRM rm, SchedulerApplication<SchedulerApplicationAttempt> app, Resource clusterResource, Resource queueResource, Resource usedResource, int numContainers) throws Exception { checkCSLeafQueue(rm, app, clusterResource, queueResource, usedResource, numContainers); LeafQueue queue = (LeafQueue) app.getQueue(); Resource availableResources = Resources.subtract(queueResource, usedResource); // ************ check app headroom **************** SchedulerApplicationAttempt schedulerAttempt = app.getCurrentAppAttempt(); assertEquals(availableResources, schedulerAttempt.getHeadroom()); // ************* check Queue metrics ************ QueueMetrics queueMetrics = queue.getMetrics(); assertMetrics(queueMetrics, 1, 0, 1, 0, 2, availableResources.getMemory(), availableResources.getVirtualCores(), usedResource.getMemory(), usedResource.getVirtualCores()); // ************ check user metrics *********** QueueMetrics userMetrics = queueMetrics.getUserMetrics(app.getUser()); assertMetrics(userMetrics, 1, 0, 1, 0, 2, availableResources.getMemory(), availableResources.getVirtualCores(), usedResource.getMemory(), usedResource.getVirtualCores()); }
Example 9
Source File: ProportionalCapacityPreemptionPolicy.java From big-c with Apache License 2.0 | 4 votes |
Resource offer(Resource avail, ResourceCalculator rc, Resource clusterResource) { Resource absMaxCapIdealAssignedDelta = Resources.componentwiseMax( Resources.subtract(maxCapacity, idealAssigned), Resource.newInstance(0, 0)); // remain = avail - min(avail, (max - assigned), (current + pending - assigned)) // we have bug here. in some case: //(current + pending - assigned).core > avail.core //(current + pending - assigned).memo < avail.memo //so we get least cores of the three and least memory of the three Resource possibleAccepted = Resources.mins(rc, clusterResource, absMaxCapIdealAssignedDelta, Resources.mins(rc, clusterResource, avail, Resources.subtract( Resources.add(current, pending), idealAssigned))); //final allocation resource Resource finalAccepted = Resources.clone(possibleAccepted); //in extrame case where avail cores are more less than the available memory, it may preempt mroe memory //Max: 1310720 320 //avail: 542634 26 //Delta: 734280 60 //Pending: 525312 120 //current: 576512 260 //ideal: 576512 260 //then the accepted will be (525312,26) in which the memory is far more beyond the requirement if(isSuspended){ if(dominantResource == Resources.CPU && !Resources.equals(pending,Resources.none())){ //pending must be either none() or resource(int ,int) if(avail.getVirtualCores() == 0){ //if the dominant resource is cpu, we will stop allocation even we have memory finalAccepted.setMemory(0); //but if we still have more available memory, we can allocate, to avoid preemption //we set memory to current usage int gapMemory = current.getMemory() - idealAssigned.getMemory(); if(gapMemory > 0 && possibleAccepted.getMemory() > gapMemory){ finalAccepted.setMemory(gapMemory); LOG.info("gap memory: "+gapMemory); } }else{ double memoryRatio = pending.getMemory()*1.0/pending.getVirtualCores(); int ratioedMemory = (int)(memoryRatio*possibleAccepted.getVirtualCores()); finalAccepted.setMemory(ratioedMemory < possibleAccepted.getMemory() ? ratioedMemory:possibleAccepted.getMemory()); } LOG.info("queue: "+queueName+" cpu dominant "); if(finalAccepted.getMemory() < possibleAccepted.getMemory()){ LOG.info("previous memory: "+possibleAccepted.getMemory()+" final memory: "+finalAccepted.getMemory()); } }else if(dominantResource == Resources.MEMORY && !Resources.equals(pending, Resources.none())){ if(avail.getMemory() == 0){ finalAccepted.setVirtualCores(0); int gapCores = current.getVirtualCores() - idealAssigned.getVirtualCores(); if(gapCores > 0 && possibleAccepted.getVirtualCores() > gapCores){ finalAccepted.setVirtualCores(gapCores); LOG.info("gap cores: "+gapCores); } }else{ double cpuRatio = pending.getVirtualCores()*1.0/pending.getMemory(); int ratioedcpu = (int)(cpuRatio*possibleAccepted.getMemory()); finalAccepted.setVirtualCores(ratioedcpu < possibleAccepted.getMemory() ? ratioedcpu:possibleAccepted.getMemory()); } LOG.info("queue: "+queueName+" memory dominant "); }else{ LOG.info("queue: "+queueName+" empty "); } } LOG.info("queueName: "+queueName); LOG.info("beforeideal: "+idealAssigned); Resource remain = Resources.subtract(avail, finalAccepted); Resources.addTo(idealAssigned, finalAccepted); LOG.info("avaul: "+avail); LOG.info("absMaxDelta: "+absMaxCapIdealAssignedDelta); LOG.info("max: "+maxCapacity); LOG.info("current: "+current); LOG.info("pending: "+pending); LOG.info("acceped: "+finalAccepted); LOG.info("ideal: "+idealAssigned); return remain; }
Example 10
Source File: TestFSAppAttempt.java From big-c with Apache License 2.0 | 4 votes |
@Test public void testHeadroom() { final FairScheduler mockScheduler = Mockito.mock(FairScheduler.class); Mockito.when(mockScheduler.getClock()).thenReturn(scheduler.getClock()); final FSLeafQueue mockQueue = Mockito.mock(FSLeafQueue.class); final Resource queueMaxResources = Resource.newInstance(5 * 1024, 3); final Resource queueFairShare = Resources.createResource(4096, 2); final Resource queueUsage = Resource.newInstance(2048, 2); final Resource queueStarvation = Resources.subtract(queueFairShare, queueUsage); final Resource queueMaxResourcesAvailable = Resources.subtract(queueMaxResources, queueUsage); final Resource clusterResource = Resources.createResource(8192, 8); final Resource clusterUsage = Resources.createResource(2048, 2); final Resource clusterAvailable = Resources.subtract(clusterResource, clusterUsage); final QueueMetrics fakeRootQueueMetrics = Mockito.mock(QueueMetrics.class); Mockito.when(mockQueue.getMaxShare()).thenReturn(queueMaxResources); Mockito.when(mockQueue.getFairShare()).thenReturn(queueFairShare); Mockito.when(mockQueue.getResourceUsage()).thenReturn(queueUsage); Mockito.when(mockScheduler.getClusterResource()).thenReturn (clusterResource); Mockito.when(fakeRootQueueMetrics.getAllocatedResources()).thenReturn (clusterUsage); Mockito.when(mockScheduler.getRootQueueMetrics()).thenReturn (fakeRootQueueMetrics); ApplicationAttemptId applicationAttemptId = createAppAttemptId(1, 1); RMContext rmContext = resourceManager.getRMContext(); FSAppAttempt schedulerApp = new FSAppAttempt(mockScheduler, applicationAttemptId, "user1", mockQueue , null, rmContext); // Min of Memory and CPU across cluster and queue is used in // DominantResourceFairnessPolicy Mockito.when(mockQueue.getPolicy()).thenReturn(SchedulingPolicy .getInstance(DominantResourceFairnessPolicy.class)); verifyHeadroom(schedulerApp, min(queueStarvation.getMemory(), clusterAvailable.getMemory(), queueMaxResourcesAvailable.getMemory()), min(queueStarvation.getVirtualCores(), clusterAvailable.getVirtualCores(), queueMaxResourcesAvailable.getVirtualCores()) ); // Fair and Fifo ignore CPU of queue, so use cluster available CPU Mockito.when(mockQueue.getPolicy()).thenReturn(SchedulingPolicy .getInstance(FairSharePolicy.class)); verifyHeadroom(schedulerApp, min(queueStarvation.getMemory(), clusterAvailable.getMemory(), queueMaxResourcesAvailable.getMemory()), Math.min( clusterAvailable.getVirtualCores(), queueMaxResourcesAvailable.getVirtualCores()) ); Mockito.when(mockQueue.getPolicy()).thenReturn(SchedulingPolicy .getInstance(FifoPolicy.class)); verifyHeadroom(schedulerApp, min(queueStarvation.getMemory(), clusterAvailable.getMemory(), queueMaxResourcesAvailable.getMemory()), Math.min( clusterAvailable.getVirtualCores(), queueMaxResourcesAvailable.getVirtualCores()) ); }
Example 11
Source File: RMContainerAllocator.java From big-c with Apache License 2.0 | 4 votes |
@Private @VisibleForTesting void preemptReducesIfNeeded() { if (reduceResourceRequest.equals(Resources.none())) { return; // no reduces } //check if reduces have taken over the whole cluster and there are //unassigned maps if (scheduledRequests.maps.size() > 0) { Resource resourceLimit = getResourceLimit(); Resource availableResourceForMap = Resources.subtract( resourceLimit, Resources.multiply(reduceResourceRequest, assignedRequests.reduces.size() - assignedRequests.preemptionWaitingReduces.size())); // availableMemForMap must be sufficient to run at least 1 map if (ResourceCalculatorUtils.computeAvailableContainers(availableResourceForMap, mapResourceRequest, getSchedulerResourceTypes()) <= 0) { // to make sure new containers are given to maps and not reduces // ramp down all scheduled reduces if any // (since reduces are scheduled at higher priority than maps) LOG.info("Ramping down all scheduled reduces:" + scheduledRequests.reduces.size()); for (ContainerRequest req : scheduledRequests.reduces.values()) { pendingReduces.add(req); } scheduledRequests.reduces.clear(); //do further checking to find the number of map requests that were //hanging around for a while int hangingMapRequests = getNumOfHangingRequests(scheduledRequests.maps); if (hangingMapRequests > 0) { // preempt for making space for at least one map int preemptionReduceNumForOneMap = ResourceCalculatorUtils.divideAndCeilContainers(mapResourceRequest, reduceResourceRequest, getSchedulerResourceTypes()); int preemptionReduceNumForPreemptionLimit = ResourceCalculatorUtils.divideAndCeilContainers( Resources.multiply(resourceLimit, maxReducePreemptionLimit), reduceResourceRequest, getSchedulerResourceTypes()); int preemptionReduceNumForAllMaps = ResourceCalculatorUtils.divideAndCeilContainers( Resources.multiply(mapResourceRequest, hangingMapRequests), reduceResourceRequest, getSchedulerResourceTypes()); int toPreempt = Math.min(Math.max(preemptionReduceNumForOneMap, preemptionReduceNumForPreemptionLimit), preemptionReduceNumForAllMaps); LOG.info("Going to preempt " + toPreempt + " due to lack of space for maps"); assignedRequests.preemptReduce(toPreempt); } } } }
Example 12
Source File: CSQueueUtils.java From big-c with Apache License 2.0 | 4 votes |
@Lock(CSQueue.class) public static void updateQueueStatistics( final ResourceCalculator calculator, final CSQueue childQueue, final CSQueue parentQueue, final Resource clusterResource, final Resource minimumAllocation) { Resource queueLimit = Resources.none(); Resource usedResources = childQueue.getUsedResources(); float absoluteUsedCapacity = 0.0f; float usedCapacity = 0.0f; if (Resources.greaterThan( calculator, clusterResource, clusterResource, Resources.none())) { queueLimit = Resources.multiply(clusterResource, childQueue.getAbsoluteCapacity()); //absoluteUsedCapacity = // Resources.divide(calculator, clusterResource, // usedResources, clusterResource); absoluteUsedCapacity = (float) ((usedResources.getVirtualCores()*1.0)/(clusterResource.getVirtualCores()*1.0)); usedCapacity = Resources.equals(queueLimit, Resources.none()) ? 0 : Resources.divide(calculator, clusterResource, usedResources, queueLimit); } childQueue.setUsedCapacity(usedCapacity); childQueue.setAbsoluteUsedCapacity(absoluteUsedCapacity); Resource available = Resources.subtract(queueLimit, usedResources); childQueue.getMetrics().setAvailableResourcesToQueue( Resources.max( calculator, clusterResource, available, Resources.none() ) ); }
Example 13
Source File: RLESparseResourceAllocation.java From big-c with Apache License 2.0 | 4 votes |
/** * Removes a resource for the specified interval * * @param reservationInterval the interval for which the resource is to be * removed * @param capacity the resource to be removed * @return true if removal is successful, false otherwise */ public boolean removeInterval(ReservationInterval reservationInterval, ReservationRequest capacity) { Resource totCap = Resources.multiply(capacity.getCapability(), (float) capacity.getNumContainers()); if (totCap.equals(ZERO_RESOURCE)) { return true; } writeLock.lock(); try { long startKey = reservationInterval.getStartTime(); long endKey = reservationInterval.getEndTime(); // update the start key NavigableMap<Long, Resource> ticks = cumulativeCapacity.headMap(endKey, false); // Decrease all the capacities of overlapping intervals SortedMap<Long, Resource> overlapSet = ticks.tailMap(startKey); if (overlapSet != null && !overlapSet.isEmpty()) { Resource updatedCapacity = Resource.newInstance(0, 0); long currentKey = -1; for (Iterator<Entry<Long, Resource>> overlapEntries = overlapSet.entrySet().iterator(); overlapEntries.hasNext();) { Entry<Long, Resource> entry = overlapEntries.next(); currentKey = entry.getKey(); updatedCapacity = Resources.subtract(entry.getValue(), totCap); // update each entry between start and end key cumulativeCapacity.put(currentKey, updatedCapacity); } // Remove the first overlap entry if it is same as previous after // updation Long firstKey = overlapSet.firstKey(); if (isSameAsPrevious(firstKey, overlapSet.get(firstKey))) { cumulativeCapacity.remove(firstKey); } // Remove the next entry if it is same as end entry after updation if ((currentKey != -1) && (isSameAsNext(currentKey, updatedCapacity))) { cumulativeCapacity.remove(cumulativeCapacity.higherKey(currentKey)); } } return true; } finally { writeLock.unlock(); } }
Example 14
Source File: RMContainerAllocator.java From hadoop with Apache License 2.0 | 4 votes |
@Private @VisibleForTesting void preemptReducesIfNeeded() { if (reduceResourceRequest.equals(Resources.none())) { return; // no reduces } //check if reduces have taken over the whole cluster and there are //unassigned maps if (scheduledRequests.maps.size() > 0) { Resource resourceLimit = getResourceLimit(); Resource availableResourceForMap = Resources.subtract( resourceLimit, Resources.multiply(reduceResourceRequest, assignedRequests.reduces.size() - assignedRequests.preemptionWaitingReduces.size())); // availableMemForMap must be sufficient to run at least 1 map if (ResourceCalculatorUtils.computeAvailableContainers(availableResourceForMap, mapResourceRequest, getSchedulerResourceTypes()) <= 0) { // to make sure new containers are given to maps and not reduces // ramp down all scheduled reduces if any // (since reduces are scheduled at higher priority than maps) LOG.info("Ramping down all scheduled reduces:" + scheduledRequests.reduces.size()); for (ContainerRequest req : scheduledRequests.reduces.values()) { pendingReduces.add(req); } scheduledRequests.reduces.clear(); //do further checking to find the number of map requests that were //hanging around for a while int hangingMapRequests = getNumOfHangingRequests(scheduledRequests.maps); if (hangingMapRequests > 0) { // preempt for making space for at least one map int preemptionReduceNumForOneMap = ResourceCalculatorUtils.divideAndCeilContainers(mapResourceRequest, reduceResourceRequest, getSchedulerResourceTypes()); int preemptionReduceNumForPreemptionLimit = ResourceCalculatorUtils.divideAndCeilContainers( Resources.multiply(resourceLimit, maxReducePreemptionLimit), reduceResourceRequest, getSchedulerResourceTypes()); int preemptionReduceNumForAllMaps = ResourceCalculatorUtils.divideAndCeilContainers( Resources.multiply(mapResourceRequest, hangingMapRequests), reduceResourceRequest, getSchedulerResourceTypes()); int toPreempt = Math.min(Math.max(preemptionReduceNumForOneMap, preemptionReduceNumForPreemptionLimit), preemptionReduceNumForAllMaps); LOG.info("Going to preempt " + toPreempt + " due to lack of space for maps"); assignedRequests.preemptReduce(toPreempt); } } } }
Example 15
Source File: TestFSAppAttempt.java From hadoop with Apache License 2.0 | 4 votes |
@Test public void testHeadroom() { final FairScheduler mockScheduler = Mockito.mock(FairScheduler.class); Mockito.when(mockScheduler.getClock()).thenReturn(scheduler.getClock()); final FSLeafQueue mockQueue = Mockito.mock(FSLeafQueue.class); final Resource queueMaxResources = Resource.newInstance(5 * 1024, 3, 3); final Resource queueFairShare = Resources.createResource(4096, 2, 2); final Resource queueUsage = Resource.newInstance(2048, 2, 2); final Resource queueStarvation = Resources.subtract(queueFairShare, queueUsage); final Resource queueMaxResourcesAvailable = Resources.subtract(queueMaxResources, queueUsage); final Resource clusterResource = Resources.createResource(8192, 8, 8); final Resource clusterUsage = Resources.createResource(2048, 2, 2); final Resource clusterAvailable = Resources.subtract(clusterResource, clusterUsage); final QueueMetrics fakeRootQueueMetrics = Mockito.mock(QueueMetrics.class); Mockito.when(mockQueue.getMaxShare()).thenReturn(queueMaxResources); Mockito.when(mockQueue.getFairShare()).thenReturn(queueFairShare); Mockito.when(mockQueue.getResourceUsage()).thenReturn(queueUsage); Mockito.when(mockScheduler.getClusterResource()).thenReturn (clusterResource); Mockito.when(fakeRootQueueMetrics.getAllocatedResources()).thenReturn (clusterUsage); Mockito.when(mockScheduler.getRootQueueMetrics()).thenReturn (fakeRootQueueMetrics); ApplicationAttemptId applicationAttemptId = createAppAttemptId(1, 1); RMContext rmContext = resourceManager.getRMContext(); FSAppAttempt schedulerApp = new FSAppAttempt(mockScheduler, applicationAttemptId, "user1", mockQueue , null, rmContext); // Min of Memory and CPU across cluster and queue is used in // DominantResourceFairnessPolicy Mockito.when(mockQueue.getPolicy()).thenReturn(SchedulingPolicy .getInstance(DominantResourceFairnessPolicy.class)); verifyHeadroom(schedulerApp, min(queueStarvation.getMemory(), clusterAvailable.getMemory(), queueMaxResourcesAvailable.getMemory()), min(queueStarvation.getVirtualCores(), clusterAvailable.getVirtualCores(), queueMaxResourcesAvailable.getVirtualCores()) ); // Fair and Fifo ignore CPU of queue, so use cluster available CPU Mockito.when(mockQueue.getPolicy()).thenReturn(SchedulingPolicy .getInstance(FairSharePolicy.class)); verifyHeadroom(schedulerApp, min(queueStarvation.getMemory(), clusterAvailable.getMemory(), queueMaxResourcesAvailable.getMemory()), Math.min( clusterAvailable.getVirtualCores(), queueMaxResourcesAvailable.getVirtualCores()) ); Mockito.when(mockQueue.getPolicy()).thenReturn(SchedulingPolicy .getInstance(FifoPolicy.class)); verifyHeadroom(schedulerApp, min(queueStarvation.getMemory(), clusterAvailable.getMemory(), queueMaxResourcesAvailable.getMemory()), Math.min( clusterAvailable.getVirtualCores(), queueMaxResourcesAvailable.getVirtualCores()) ); }
Example 16
Source File: YarnTaskSchedulerService.java From incubator-tez with Apache License 2.0 | 4 votes |
@Override public void onContainersCompleted(List<ContainerStatus> statuses) { if (isStopped.get()) { return; } Map<Object, ContainerStatus> appContainerStatus = new HashMap<Object, ContainerStatus>(statuses.size()); synchronized (this) { for(ContainerStatus containerStatus : statuses) { ContainerId completedId = containerStatus.getContainerId(); HeldContainer delayedContainer = heldContainers.get(completedId); Object task = releasedContainers.remove(completedId); if(task != null){ if (delayedContainer != null) { LOG.warn("Held container should be null since releasedContainer is not"); } // TODO later we may want to check if exit code matched expectation // e.g. successful container should not come back fail exit code after // being released // completion of a container we had released earlier // an allocated container completed. notify app LOG.info("Released container completed:" + completedId + " last allocated to task: " + task); appContainerStatus.put(task, containerStatus); continue; } // not found in released containers. check currently allocated containers // no need to release this container as the RM has already completed it task = unAssignContainer(completedId, false); if (delayedContainer != null) { heldContainers.remove(completedId); Resources.subtract(allocatedResources, delayedContainer.getContainer().getResource()); } else { LOG.warn("Held container expected to be not null for a non-AM-released container"); } if(task != null) { // completion of a container we have allocated currently // an allocated container completed. notify app LOG.info("Allocated container completed:" + completedId + " last allocated to task: " + task); appContainerStatus.put(task, containerStatus); continue; } // container neither allocated nor released LOG.info("Ignoring unknown container: " + containerStatus.getContainerId()); } } // upcall to app must be outside locks for (Entry<Object, ContainerStatus> entry : appContainerStatus.entrySet()) { appClientDelegate.containerCompleted(entry.getKey(), entry.getValue()); } }
Example 17
Source File: ProportionalCapacityPreemptionPolicy.java From hadoop with Apache License 2.0 | 4 votes |
/** * Given a set of queues compute the fix-point distribution of unassigned * resources among them. As pending request of a queue are exhausted, the * queue is removed from the set and remaining capacity redistributed among * remaining queues. The distribution is weighted based on guaranteed * capacity, unless asked to ignoreGuarantee, in which case resources are * distributed uniformly. */ private void computeFixpointAllocation(ResourceCalculator rc, Resource tot_guarant, Collection<TempQueue> qAlloc, Resource unassigned, boolean ignoreGuarantee) { // Prior to assigning the unused resources, process each queue as follows: // If current > guaranteed, idealAssigned = guaranteed + untouchable extra // Else idealAssigned = current; // Subtract idealAssigned resources from unassigned. // If the queue has all of its needs met (that is, if // idealAssigned >= current + pending), remove it from consideration. // Sort queues from most under-guaranteed to most over-guaranteed. TQComparator tqComparator = new TQComparator(rc, tot_guarant); PriorityQueue<TempQueue> orderedByNeed = new PriorityQueue<TempQueue>(10,tqComparator); for (Iterator<TempQueue> i = qAlloc.iterator(); i.hasNext();) { TempQueue q = i.next(); if (Resources.greaterThan(rc, tot_guarant, q.current, q.guaranteed)) { q.idealAssigned = Resources.add(q.guaranteed, q.untouchableExtra); } else { q.idealAssigned = Resources.clone(q.current); } Resources.subtractFrom(unassigned, q.idealAssigned); // If idealAssigned < (current + pending), q needs more resources, so // add it to the list of underserved queues, ordered by need. Resource curPlusPend = Resources.add(q.current, q.pending); if (Resources.lessThan(rc, tot_guarant, q.idealAssigned, curPlusPend)) { orderedByNeed.add(q); } } //assign all cluster resources until no more demand, or no resources are left while (!orderedByNeed.isEmpty() && Resources.greaterThan(rc,tot_guarant, unassigned,Resources.none())) { Resource wQassigned = Resource.newInstance(0, 0, 0); // we compute normalizedGuarantees capacity based on currently active // queues resetCapacity(rc, unassigned, orderedByNeed, ignoreGuarantee); // For each underserved queue (or set of queues if multiple are equally // underserved), offer its share of the unassigned resources based on its // normalized guarantee. After the offer, if the queue is not satisfied, // place it back in the ordered list of queues, recalculating its place // in the order of most under-guaranteed to most over-guaranteed. In this // way, the most underserved queue(s) are always given resources first. Collection<TempQueue> underserved = getMostUnderservedQueues(orderedByNeed, tqComparator); for (Iterator<TempQueue> i = underserved.iterator(); i.hasNext();) { TempQueue sub = i.next(); Resource wQavail = Resources.multiplyAndNormalizeUp(rc, unassigned, sub.normalizedGuarantee, Resource.newInstance(1, 1, 0)); Resource wQidle = sub.offer(wQavail, rc, tot_guarant); Resource wQdone = Resources.subtract(wQavail, wQidle); if (Resources.greaterThan(rc, tot_guarant, wQdone, Resources.none())) { // The queue is still asking for more. Put it back in the priority // queue, recalculating its order based on need. orderedByNeed.add(sub); } Resources.addTo(wQassigned, wQdone); } Resources.subtractFrom(unassigned, wQassigned); } }
Example 18
Source File: LeafQueue.java From hadoop with Apache License 2.0 | 4 votes |
@Private protected synchronized boolean canAssignToUser(Resource clusterResource, String userName, Resource limit, FiCaSchedulerApp application, Set<String> requestLabels, ResourceLimits currentResoureLimits) { User user = getUser(userName); String label = CommonNodeLabelsManager.NO_LABEL; if (requestLabels != null && !requestLabels.isEmpty()) { label = requestLabels.iterator().next(); } // Note: We aren't considering the current request since there is a fixed // overhead of the AM, but it's a > check, not a >= check, so... if (Resources .greaterThan(resourceCalculator, clusterResource, user.getUsed(label), limit)) { // if enabled, check to see if could we potentially use this node instead // of a reserved node if the application has reserved containers if (this.reservationsContinueLooking && label.equals(CommonNodeLabelsManager.NO_LABEL)) { if (Resources.lessThanOrEqual( resourceCalculator, clusterResource, Resources.subtract(user.getUsed(), application.getCurrentReservation()), limit)) { if (LOG.isDebugEnabled()) { LOG.debug("User " + userName + " in queue " + getQueueName() + " will exceed limit based on reservations - " + " consumed: " + user.getUsed() + " reserved: " + application.getCurrentReservation() + " limit: " + limit); } Resource amountNeededToUnreserve = Resources.subtract(user.getUsed(label), limit); // we can only acquire a new container if we unreserve first since we ignored the // user limit. Choose the max of user limit or what was previously set by max // capacity. currentResoureLimits.setAmountNeededUnreserve(Resources.max(resourceCalculator, clusterResource, currentResoureLimits.getAmountNeededUnreserve(), amountNeededToUnreserve)); return true; } } if (LOG.isDebugEnabled()) { LOG.debug("User " + userName + " in queue " + getQueueName() + " will exceed limit - " + " consumed: " + user.getUsed() + " limit: " + limit); } return false; } return true; }
Example 19
Source File: AbstractCSQueue.java From hadoop with Apache License 2.0 | 4 votes |
synchronized boolean canAssignToThisQueue(Resource clusterResource, Set<String> nodeLabels, ResourceLimits currentResourceLimits, Resource nowRequired, Resource resourceCouldBeUnreserved) { // Get label of this queue can access, it's (nodeLabel AND queueLabel) Set<String> labelCanAccess; if (null == nodeLabels || nodeLabels.isEmpty()) { labelCanAccess = new HashSet<String>(); // Any queue can always access any node without label labelCanAccess.add(RMNodeLabelsManager.NO_LABEL); } else { labelCanAccess = new HashSet<String>( accessibleLabels.contains(CommonNodeLabelsManager.ANY) ? nodeLabels : Sets.intersection(accessibleLabels, nodeLabels)); } for (String label : labelCanAccess) { // New total resource = used + required Resource newTotalResource = Resources.add(queueUsage.getUsed(label), nowRequired); Resource currentLimitResource = getCurrentLimitResource(label, clusterResource, currentResourceLimits); if (Resources.greaterThan(resourceCalculator, clusterResource, newTotalResource, currentLimitResource)) { // if reservation continous looking enabled, check to see if could we // potentially use this node instead of a reserved node if the application // has reserved containers. // TODO, now only consider reservation cases when the node has no label if (this.reservationsContinueLooking && label.equals(RMNodeLabelsManager.NO_LABEL) && Resources.greaterThan(resourceCalculator, clusterResource, resourceCouldBeUnreserved, Resources.none())) { // resource-without-reserved = used - reserved Resource newTotalWithoutReservedResource = Resources.subtract(newTotalResource, resourceCouldBeUnreserved); // when total-used-without-reserved-resource < currentLimit, we still // have chance to allocate on this node by unreserving some containers if (Resources.lessThan(resourceCalculator, clusterResource, newTotalWithoutReservedResource, currentLimitResource)) { if (LOG.isDebugEnabled()) { LOG.debug("try to use reserved: " + getQueueName() + " usedResources: " + queueUsage.getUsed() + ", clusterResources: " + clusterResource + ", reservedResources: " + resourceCouldBeUnreserved + ", capacity-without-reserved: " + newTotalWithoutReservedResource + ", maxLimitCapacity: " + currentLimitResource); } currentResourceLimits.setAmountNeededUnreserve(Resources.subtract(newTotalResource, currentLimitResource)); return true; } } if (LOG.isDebugEnabled()) { LOG.debug(getQueueName() + "Check assign to queue, label=" + label + " usedResources: " + queueUsage.getUsed(label) + " clusterResources: " + clusterResource + " currentUsedCapacity " + Resources.divide(resourceCalculator, clusterResource, queueUsage.getUsed(label), labelManager.getResourceByLabel(label, clusterResource)) + " max-capacity: " + queueCapacities.getAbsoluteMaximumCapacity(label) + ")"); } return false; } return true; } // Actually, this will not happen, since labelCanAccess will be always // non-empty return false; }
Example 20
Source File: AbstractCSQueue.java From big-c with Apache License 2.0 | 4 votes |
synchronized boolean canAssignToThisQueue(Resource clusterResource, Set<String> nodeLabels, ResourceLimits currentResourceLimits, Resource nowRequired, Resource resourceCouldBeUnreserved) { // Get label of this queue can access, it's (nodeLabel AND queueLabel) Set<String> labelCanAccess; if (null == nodeLabels || nodeLabels.isEmpty()) { labelCanAccess = new HashSet<String>(); // Any queue can always access any node without label labelCanAccess.add(RMNodeLabelsManager.NO_LABEL); } else { labelCanAccess = new HashSet<String>( accessibleLabels.contains(CommonNodeLabelsManager.ANY) ? nodeLabels : Sets.intersection(accessibleLabels, nodeLabels)); } for (String label : labelCanAccess) { // New total resource = used + required Resource newTotalResource = Resources.add(queueUsage.getUsed(label), nowRequired); Resource currentLimitResource = getCurrentLimitResource(label, clusterResource, currentResourceLimits); //current resource is much more than currentlimit if (Resources.greaterThan(resourceCalculator, clusterResource, newTotalResource, currentLimitResource)) { // if reservation continous looking enabled, check to see if could we // potentially use this node instead of a reserved node if the application // has reserved containers. // TODO, now only consider reservation cases when the node has no label if (this.reservationsContinueLooking && label.equals(RMNodeLabelsManager.NO_LABEL) && Resources.greaterThan(resourceCalculator, clusterResource, resourceCouldBeUnreserved, Resources.none())) { // resource-without-reserved = used - reserved Resource newTotalWithoutReservedResource = Resources.subtract(newTotalResource, resourceCouldBeUnreserved); // when total-used-without-reserved-resource < currentLimit, we still // have chance to allocate on this node by unreserving some containers if (Resources.lessThan(resourceCalculator, clusterResource, newTotalWithoutReservedResource, currentLimitResource)) { if (LOG.isDebugEnabled()) { LOG.debug("try to use reserved: " + getQueueName() + " usedResources: " + queueUsage.getUsed() + ", clusterResources: " + clusterResource + ", reservedResources: " + resourceCouldBeUnreserved + ", capacity-without-reserved: " + newTotalWithoutReservedResource + ", maxLimitCapacity: " + currentLimitResource); } currentResourceLimits.setAmountNeededUnreserve(Resources.subtract(newTotalResource, currentLimitResource)); return true; } } if (LOG.isDebugEnabled()) { LOG.debug(getQueueName() + "Check assign to queue, label=" + label + " usedResources: " + queueUsage.getUsed(label) + " clusterResources: " + clusterResource + " currentUsedCapacity " + Resources.divide(resourceCalculator, clusterResource, queueUsage.getUsed(label), labelManager.getResourceByLabel(label, clusterResource)) + " max-capacity: " + queueCapacities.getAbsoluteMaximumCapacity(label) + ")"); } return false; } return true; } // Actually, this will not happen, since labelCanAccess will be always // non-empty return false; }