Java Code Examples for org.apache.hadoop.yarn.util.resource.Resources#max()

public synchronized Resource getAMResourceLimitPerPartition(
    String nodePartition) {
  /*
   * For non-labeled partition, get the max value from resources currently
   * available to the queue and the absolute resources guaranteed for the
   * partition in the queue. For labeled partition, consider only the absolute
   * resources guaranteed. Multiply this value (based on labeled/
   * non-labeled), * with per-partition am-resource-percent to get the max am
   * resource limit for this queue and partition.
   */
  Resource queuePartitionResource = Resources.multiplyAndNormalizeUp(
      resourceCalculator,
      labelManager.getResourceByLabel(nodePartition, lastClusterResource),
      queueCapacities.getAbsoluteCapacity(nodePartition), minimumAllocation);

  Resource queueCurrentLimit = Resources.none();
  // For non-labeled partition, we need to consider the current queue
  // usage limit.
  if (nodePartition.equals(RMNodeLabelsManager.NO_LABEL)) {
    synchronized (queueResourceLimitsInfo) {
      queueCurrentLimit = queueResourceLimitsInfo.getQueueCurrentLimit();
    }
  }

  float amResourcePercent = queueCapacities
      .getMaxAMResourcePercentage(nodePartition);

  // Current usable resource for this queue and partition is the max of
  // queueCurrentLimit and queuePartitionResource.
  Resource queuePartitionUsableResource = Resources.max(resourceCalculator,
      lastClusterResource, queueCurrentLimit, queuePartitionResource);

  Resource amResouceLimit = Resources.multiplyAndNormalizeUp(
      resourceCalculator, queuePartitionUsableResource, amResourcePercent,
      minimumAllocation);

   //YARN-3494
   //metrics.setAMResouceLimit(amResouceLimit);
   return amResouceLimit;
}
 

public synchronized Resource getAMResourceLimit() {
   /* 
    * The limit to the amount of resources which can be consumed by
    * application masters for applications running in the queue
    * is calculated by taking the greater of the max resources currently
    * available to the queue (see absoluteMaxAvailCapacity) and the absolute
    * resources guaranteed for the queue and multiplying it by the am
    * resource percent.
    *
    * This is to allow a queue to grow its (proportional) application 
    * master resource use up to its max capacity when other queues are 
    * idle but to scale back down to it's guaranteed capacity as they 
    * become busy.
    *
    */
   Resource queueCurrentLimit;
   synchronized (queueResourceLimitsInfo) {
     queueCurrentLimit = queueResourceLimitsInfo.getQueueCurrentLimit();
   }
   
   //该队列的最大资源限制
   Resource queueCap = Resources.max(resourceCalculator, lastClusterResource,
     absoluteCapacityResource, queueCurrentLimit);
   
   return Resources.multiplyAndNormalizeUp( 
        resourceCalculator,
        queueCap, 
        maxAMResourcePerQueuePercent, minimumAllocation);
}
 

private void validateReservationDefinition(ReservationId reservationId,
    ReservationDefinition contract, Plan plan, String auditConstant)
    throws YarnException {
  String message = "";
  // check if deadline is in the past
  if (contract == null) {
    message =
        "Missing reservation definition."
            + " Please try again by specifying a reservation definition.";
    RMAuditLogger.logFailure("UNKNOWN", auditConstant,
        "validate reservation input definition", "ClientRMService", message);
    throw RPCUtil.getRemoteException(message);
  }
  if (contract.getDeadline() <= clock.getTime()) {
    message =
        "The specified deadline: " + contract.getDeadline()
            + " is the past. Please try again with deadline in the future.";
    RMAuditLogger.logFailure("UNKNOWN", auditConstant,
        "validate reservation input definition", "ClientRMService", message);
    throw RPCUtil.getRemoteException(message);
  }
  // Check if at least one RR has been specified
  ReservationRequests resReqs = contract.getReservationRequests();
  if (resReqs == null) {
    message =
        "No resources have been specified to reserve."
            + "Please try again by specifying the resources to reserve.";
    RMAuditLogger.logFailure("UNKNOWN", auditConstant,
        "validate reservation input definition", "ClientRMService", message);
    throw RPCUtil.getRemoteException(message);
  }
  List<ReservationRequest> resReq = resReqs.getReservationResources();
  if (resReq == null || resReq.isEmpty()) {
    message =
        "No resources have been specified to reserve."
            + " Please try again by specifying the resources to reserve.";
    RMAuditLogger.logFailure("UNKNOWN", auditConstant,
        "validate reservation input definition", "ClientRMService", message);
    throw RPCUtil.getRemoteException(message);
  }
  // compute minimum duration and max gang size
  long minDuration = 0;
  Resource maxGangSize = Resource.newInstance(0, 0, 0);
  ReservationRequestInterpreter type =
      contract.getReservationRequests().getInterpreter();
  for (ReservationRequest rr : resReq) {
    if (type == ReservationRequestInterpreter.R_ALL
        || type == ReservationRequestInterpreter.R_ANY) {
      minDuration = Math.max(minDuration, rr.getDuration());
    } else {
      minDuration += rr.getDuration();
    }
    maxGangSize =
        Resources.max(plan.getResourceCalculator(), plan.getTotalCapacity(),
            maxGangSize,
            Resources.multiply(rr.getCapability(), rr.getConcurrency()));
  }
  // verify the allocation is possible (skip for ANY)
  if (contract.getDeadline() - contract.getArrival() < minDuration
      && type != ReservationRequestInterpreter.R_ANY) {
    message =
        "The time difference ("
            + (contract.getDeadline() - contract.getArrival())
            + ") between arrival (" + contract.getArrival() + ") "
            + "and deadline (" + contract.getDeadline() + ") must "
            + " be greater or equal to the minimum resource duration ("
            + minDuration + ")";
    RMAuditLogger.logFailure("UNKNOWN", auditConstant,
        "validate reservation input definition", "ClientRMService", message);
    throw RPCUtil.getRemoteException(message);
  }
  // check that the largest gang does not exceed the inventory available
  // capacity (skip for ANY)
  if (Resources.greaterThan(plan.getResourceCalculator(),
      plan.getTotalCapacity(), maxGangSize, plan.getTotalCapacity())
      && type != ReservationRequestInterpreter.R_ANY) {
    message =
        "The size of the largest gang in the reservation refinition ("
            + maxGangSize + ") exceed the capacity available ("
            + plan.getTotalCapacity() + " )";
    RMAuditLogger.logFailure("UNKNOWN", auditConstant,
        "validate reservation input definition", "ClientRMService", message);
    throw RPCUtil.getRemoteException(message);
  }
}
 

@Lock(NoLock.class)
private Resource computeUserLimit(FiCaSchedulerApp application,
    Resource clusterResource, Resource required, User user,
    Set<String> requestedLabels) {
  // What is our current capacity? 
  // * It is equal to the max(required, queue-capacity) if
  //   we're running below capacity. The 'max' ensures that jobs in queues
  //   with miniscule capacity (< 1 slot) make progress
  // * If we're running over capacity, then its
  //   (usedResources + required) (which extra resources we are allocating)
  Resource queueCapacity = Resource.newInstance(0, 0, 0);
  if (requestedLabels != null && !requestedLabels.isEmpty()) {
    // if we have multiple labels to request, we will choose to use the first
    // label
    String firstLabel = requestedLabels.iterator().next();
    queueCapacity =
        Resources
            .max(resourceCalculator, clusterResource, queueCapacity,
                Resources.multiplyAndNormalizeUp(resourceCalculator,
                    labelManager.getResourceByLabel(firstLabel,
                        clusterResource),
                    queueCapacities.getAbsoluteCapacity(firstLabel),
                    minimumAllocation));
  } else {
    // else there's no label on request, just to use absolute capacity as
    // capacity for nodes without label
    queueCapacity =
        Resources.multiplyAndNormalizeUp(resourceCalculator, labelManager
              .getResourceByLabel(CommonNodeLabelsManager.NO_LABEL, clusterResource),
            queueCapacities.getAbsoluteCapacity(), minimumAllocation);
  }

  // Allow progress for queues with miniscule capacity
  queueCapacity =
      Resources.max(
          resourceCalculator, clusterResource, 
          queueCapacity, 
          required);

  Resource currentCapacity =
      Resources.lessThan(resourceCalculator, clusterResource, 
          queueUsage.getUsed(), queueCapacity) ?
          queueCapacity : Resources.add(queueUsage.getUsed(), required);
  
  // Never allow a single user to take more than the 
  // queue's configured capacity * user-limit-factor.
  // Also, the queue's configured capacity should be higher than 
  // queue-hard-limit * ulMin
  
  final int activeUsers = activeUsersManager.getNumActiveUsers();  
  		
  Resource limit =
      Resources.roundUp(
          resourceCalculator, 
          Resources.min(
              resourceCalculator, clusterResource,   
              Resources.max(
                  resourceCalculator, clusterResource, 
                  Resources.divideAndCeil(
                      resourceCalculator, currentCapacity, activeUsers),
                  Resources.divideAndCeil(
                      resourceCalculator, 
                      Resources.multiplyAndRoundDown(
                          currentCapacity, userLimit), 
                      100)
                  ), 
              Resources.multiplyAndRoundDown(queueCapacity, userLimitFactor)
              ), 
          minimumAllocation);

  if (LOG.isDebugEnabled()) {
    String userName = application.getUser();
    LOG.debug("User limit computation for " + userName + 
        " in queue " + getQueueName() +
        " userLimit=" + userLimit +
        " userLimitFactor=" + userLimitFactor +
        " required: " + required + 
        " consumed: " + user.getUsed() + 
        " limit: " + limit +
        " queueCapacity: " + queueCapacity + 
        " qconsumed: " + queueUsage.getUsed() +
        " currentCapacity: " + currentCapacity +
        " activeUsers: " + activeUsers +
        " clusterCapacity: " + clusterResource
    );
  }
  user.setUserResourceLimit(limit);
  return limit;
}
 

private void validateReservationDefinition(ReservationId reservationId,
    ReservationDefinition contract, Plan plan, String auditConstant)
    throws YarnException {
  String message = "";
  // check if deadline is in the past
  if (contract == null) {
    message =
        "Missing reservation definition."
            + " Please try again by specifying a reservation definition.";
    RMAuditLogger.logFailure("UNKNOWN", auditConstant,
        "validate reservation input definition", "ClientRMService", message);
    throw RPCUtil.getRemoteException(message);
  }
  if (contract.getDeadline() <= clock.getTime()) {
    message =
        "The specified deadline: " + contract.getDeadline()
            + " is the past. Please try again with deadline in the future.";
    RMAuditLogger.logFailure("UNKNOWN", auditConstant,
        "validate reservation input definition", "ClientRMService", message);
    throw RPCUtil.getRemoteException(message);
  }
  // Check if at least one RR has been specified
  ReservationRequests resReqs = contract.getReservationRequests();
  if (resReqs == null) {
    message =
        "No resources have been specified to reserve."
            + "Please try again by specifying the resources to reserve.";
    RMAuditLogger.logFailure("UNKNOWN", auditConstant,
        "validate reservation input definition", "ClientRMService", message);
    throw RPCUtil.getRemoteException(message);
  }
  List<ReservationRequest> resReq = resReqs.getReservationResources();
  if (resReq == null || resReq.isEmpty()) {
    message =
        "No resources have been specified to reserve."
            + " Please try again by specifying the resources to reserve.";
    RMAuditLogger.logFailure("UNKNOWN", auditConstant,
        "validate reservation input definition", "ClientRMService", message);
    throw RPCUtil.getRemoteException(message);
  }
  // compute minimum duration and max gang size
  long minDuration = 0;
  Resource maxGangSize = Resource.newInstance(0, 0);
  ReservationRequestInterpreter type =
      contract.getReservationRequests().getInterpreter();
  for (ReservationRequest rr : resReq) {
    if (type == ReservationRequestInterpreter.R_ALL
        || type == ReservationRequestInterpreter.R_ANY) {
      minDuration = Math.max(minDuration, rr.getDuration());
    } else {
      minDuration += rr.getDuration();
    }
    maxGangSize =
        Resources.max(plan.getResourceCalculator(), plan.getTotalCapacity(),
            maxGangSize,
            Resources.multiply(rr.getCapability(), rr.getConcurrency()));
  }
  // verify the allocation is possible (skip for ANY)
  if (contract.getDeadline() - contract.getArrival() < minDuration
      && type != ReservationRequestInterpreter.R_ANY) {
    message =
        "The time difference ("
            + (contract.getDeadline() - contract.getArrival())
            + ") between arrival (" + contract.getArrival() + ") "
            + "and deadline (" + contract.getDeadline() + ") must "
            + " be greater or equal to the minimum resource duration ("
            + minDuration + ")";
    RMAuditLogger.logFailure("UNKNOWN", auditConstant,
        "validate reservation input definition", "ClientRMService", message);
    throw RPCUtil.getRemoteException(message);
  }
  // check that the largest gang does not exceed the inventory available
  // capacity (skip for ANY)
  if (Resources.greaterThan(plan.getResourceCalculator(),
      plan.getTotalCapacity(), maxGangSize, plan.getTotalCapacity())
      && type != ReservationRequestInterpreter.R_ANY) {
    message =
        "The size of the largest gang in the reservation refinition ("
            + maxGangSize + ") exceed the capacity available ("
            + plan.getTotalCapacity() + " )";
    RMAuditLogger.logFailure("UNKNOWN", auditConstant,
        "validate reservation input definition", "ClientRMService", message);
    throw RPCUtil.getRemoteException(message);
  }
}
 

@Lock(NoLock.class)
private Resource computeUserLimit(FiCaSchedulerApp application,
    Resource clusterResource, Resource required, User user,
    Set<String> requestedLabels) {
  // What is our current capacity? 
  // * It is equal to the max(required, queue-capacity) if
  //   we're running below capacity. The 'max' ensures that jobs in queues
  //   with miniscule capacity (< 1 slot) make progress
  // * If we're running over capacity, then its
  //   (usedResources + required) (which extra resources we are allocating)
  Resource queueCapacity = Resource.newInstance(0, 0);
  if (requestedLabels != null && !requestedLabels.isEmpty()) {
    // if we have multiple labels to request, we will choose to use the first
    // label
    String firstLabel = requestedLabels.iterator().next();
    queueCapacity =
        Resources
            .max(resourceCalculator, clusterResource, queueCapacity,
                Resources.multiplyAndNormalizeUp(resourceCalculator,
                    labelManager.getResourceByLabel(firstLabel,
                        clusterResource),
                    queueCapacities.getAbsoluteCapacity(firstLabel),
                    minimumAllocation));
  } else {
    // else there's no label on request, just to use absolute capacity as
    // capacity for nodes without label
    queueCapacity =
        Resources.multiplyAndNormalizeUp(resourceCalculator, labelManager
              .getResourceByLabel(CommonNodeLabelsManager.NO_LABEL, clusterResource),
            queueCapacities.getAbsoluteCapacity(), minimumAllocation);
  }

  // Allow progress for queues with miniscule capacity,至少这个queue每次能够分配一个container。可以稍微的超出每个queue的limit
  queueCapacity =
      Resources.max(
          resourceCalculator, clusterResource, 
          queueCapacity, 
          required);

  Resource currentCapacity =
      Resources.lessThan(resourceCalculator, clusterResource, 
          queueUsage.getUsed(), queueCapacity) ?
          queueCapacity : Resources.add(queueUsage.getUsed(), required);
  
  // Never allow a single user to take more than the 
  // queue's configured capacity * user-limit-factor.
  // Also, the queue's configured capacity should be higher than 
  // queue-hard-limit * ulMin
  
  final int activeUsers = activeUsersManager.getNumActiveUsers();  
  
  //UserCapacity_limit  =currentCapacity*userLimit     //can not be less than this if multiple user share cluster,如果用户数太多，导致每个人的share
                                                       //小于这个数值则不再为新提交的job分配资源，而是把它queue起来
  //UserCapacity_Average=currentCapacity/activeUSers   //如果多用户提交则平局一下
  
  //UserCapacity        =max{UserCapacity_limit, UserCapacity_Average}
  
  //FactorCapacity      =queueCapacity * userLimitFactor //可以让一个用户拿到多余一个queue容量的资源，default1,如果这个cluster idle的话可以多分配
  
  //Final_UserCapacity  =min{UserCapacity, FactorCapacity }
  		
  Resource limit =
      Resources.roundUp(
          resourceCalculator, 
          Resources.min(
              resourceCalculator, clusterResource,   
              Resources.max(
                  resourceCalculator, clusterResource, 
                  Resources.divideAndCeil(
                      resourceCalculator, currentCapacity, activeUsers),
                  Resources.divideAndCeil(
                      resourceCalculator, 
                      Resources.multiplyAndRoundDown(
                          currentCapacity, userLimit), 
                      100)
                  ), 
              Resources.multiplyAndRoundDown(queueCapacity, userLimitFactor)
              ), 
          minimumAllocation);

  
    String userName = application.getUser();
    /*
    LOG.info("User limit computation for " + userName + 
        " in queue " + getQueueName() +
        " userLimit=" + userLimit +
        " userLimitFactor=" + userLimitFactor +
        " required: " + required + 
        " consumed: " + user.getUsed() + 
        " limit: " + limit +
        " queueCapacity: " + queueCapacity + 
        " qconsumed: " + queueUsage.getUsed() +
        " currentCapacity: " + currentCapacity +
        " activeUsers: " + activeUsers +
        " clusterCapacity: " + clusterResource
    );
    */
  
  user.setUserResourceLimit(limit);
  return limit;
}