Java Code Examples for org.apache.hadoop.yarn.api.records.ResourceRequest#getResourceName()

private void validateLabelsRequests(ResourceRequest resourceRequest,
    boolean isReduce) {
  switch (resourceRequest.getResourceName()) {
  case "map":
  case "reduce":
  case NetworkTopology.DEFAULT_RACK:
    Assert.assertNull(resourceRequest.getNodeLabelExpression());
    break;
  case "*":
    Assert.assertEquals(isReduce ? "ReduceNodes" : "MapNodes",
        resourceRequest.getNodeLabelExpression());
    break;
  default:
    Assert.fail("Invalid resource location "
        + resourceRequest.getResourceName());
  }
}
 

/**
 * Utility method to validate a resource request, by insuring that the
 * requested memory/vcore/gcore is non-negative and not greater than max
 * 
 * @throws InvalidResourceRequestException when there is invalid request
 */
private static void validateResourceRequest(ResourceRequest resReq,
    Resource maximumResource, QueueInfo queueInfo, RMContext rmContext)
    throws InvalidResourceRequestException {
  if (resReq.getCapability().getMemory() < 0 ||
      resReq.getCapability().getMemory() > maximumResource.getMemory()) {
    throw new InvalidResourceRequestException("Invalid resource request"
        + ", requested memory < 0"
        + ", or requested memory > max configured"
        + ", requestedMemory=" + resReq.getCapability().getMemory()
        + ", maxMemory=" + maximumResource.getMemory());
  }
  if (resReq.getCapability().getVirtualCores() < 0 ||
      resReq.getCapability().getVirtualCores() >
      maximumResource.getVirtualCores()) {
    throw new InvalidResourceRequestException("Invalid resource request"
        + ", requested virtual cores < 0"
        + ", or requested virtual cores > max configured"
        + ", requestedVirtualCores="
        + resReq.getCapability().getVirtualCores()
        + ", maxVirtualCores=" + maximumResource.getVirtualCores());
  }
  if (resReq.getCapability().getGpuCores() < 0 ||
      resReq.getCapability().getGpuCores() >
      maximumResource.getGpuCores()) {
    throw new InvalidResourceRequestException("Invalid resource request"
        + ", requested gpu cores < 0"
        + ", or requested gpu cores > max configured"
        + ", requestedGpuCores="
        + resReq.getCapability().getGpuCores()
        + ", maxGpuCores=" + maximumResource.getGpuCores());
  }
  String labelExp = resReq.getNodeLabelExpression();

  // we don't allow specify label expression other than resourceName=ANY now
  if (!ResourceRequest.ANY.equals(resReq.getResourceName())
      && labelExp != null && !labelExp.trim().isEmpty()) {
    throw new InvalidResourceRequestException(
        "Invailid resource request, queue=" + queueInfo.getQueueName()
            + " specified node label expression in a "
            + "resource request has resource name = "
            + resReq.getResourceName());
  }
  
  // we don't allow specify label expression with more than one node labels now
  if (labelExp != null && labelExp.contains("&&")) {
    throw new InvalidResourceRequestException(
        "Invailid resource request, queue=" + queueInfo.getQueueName()
            + " specified more than one node label "
            + "in a node label expression, node label expression = "
            + labelExp);
  }
  
  if (labelExp != null && !labelExp.trim().isEmpty() && queueInfo != null) {
    if (!checkQueueLabelExpression(queueInfo.getAccessibleNodeLabels(),
        labelExp, rmContext)) {
      throw new InvalidResourceRequestException("Invalid resource request"
          + ", queue="
          + queueInfo.getQueueName()
          + " doesn't have permission to access all labels "
          + "in resource request. labelExpression of resource request="
          + labelExp
          + ". Queue labels="
          + (queueInfo.getAccessibleNodeLabels() == null ? "" : StringUtils.join(queueInfo
              .getAccessibleNodeLabels().iterator(), ',')));
    }
  }
}
 

/**还没有弄明白
 * The ApplicationMaster is updating resource requirements for the
 * application, by asking for more resources and releasing resources acquired
 * by the application.
 * 如果是recover的话表明该资源被请求过，现在只不过是被抢占了，所以需要RM重新分配资源
 *
 * @param requests resources to be acquired
 * @param recoverPreemptedRequest recover Resource Request on preemption
 */
synchronized public void updateResourceRequests(
    List<ResourceRequest> requests, boolean recoverPreemptedRequest) {
  QueueMetrics metrics = queue.getMetrics();
  
  // Update resource requests
  for (ResourceRequest request : requests) {
    Priority priority = request.getPriority();
    String resourceName = request.getResourceName();
    boolean updatePendingResources = false;
    ResourceRequest lastRequest = null;

    if (resourceName.equals(ResourceRequest.ANY)) {
      if (LOG.isDebugEnabled()) {
        LOG.debug("update:" + " application=" + applicationId + " request="
            + request);
      }
      updatePendingResources = true;
      
      // Premature optimization?
      // Assumes that we won't see more than one priority request updated
      // in one call, reasonable assumption... however, it's totally safe
      // to activate same application more than once.
      // Thus we don't need another loop ala the one in decrementOutstanding()  
      // which is needed during deactivate.
      if (request.getNumContainers() > 0) {
        activeUsersManager.activateApplication(user, applicationId);
      }
    }

    Map<String, ResourceRequest> asks = this.requests.get(priority);

    if (asks == null) {                      //在该优先级还没有请求，所以新建一个请求
      asks = new ConcurrentHashMap<String, ResourceRequest>();
      this.requests.put(priority, asks);
      this.priorities.add(priority);
    }
    lastRequest = asks.get(resourceName);    //上一次的请求

    if (recoverPreemptedRequest && lastRequest != null) {
      // Increment the number of containers to 1, as it is recovering a
      // single container.
      request.setNumContainers(lastRequest.getNumContainers() + 1);
    }

    asks.put(resourceName, request);
    if (updatePendingResources) {
      
      // Similarly, deactivate application?
      if (request.getNumContainers() <= 0) {
        LOG.info("checking for deactivate of application :"
            + this.applicationId);
        checkForDeactivation();
      }
      
      int lastRequestContainers = lastRequest != null ? lastRequest
          .getNumContainers() : 0;
      Resource lastRequestCapability = lastRequest != null ? lastRequest
          .getCapability() : Resources.none();
      metrics.incrPendingResources(user, request.getNumContainers(),
          request.getCapability());
      metrics.decrPendingResources(user, lastRequestContainers,
          lastRequestCapability);
    }
  }
}
 

/**
 * Utility method to validate a resource request, by insuring that the
 * requested memory/vcore is non-negative and not greater than max
 * 
 * @throws InvalidResourceRequestException when there is invalid request
 */
private static void validateResourceRequest(ResourceRequest resReq,
    Resource maximumResource, QueueInfo queueInfo, RMContext rmContext)
    throws InvalidResourceRequestException {
  if (resReq.getCapability().getMemory() < 0 ||
      resReq.getCapability().getMemory() > maximumResource.getMemory()) {
    throw new InvalidResourceRequestException("Invalid resource request"
        + ", requested memory < 0"
        + ", or requested memory > max configured"
        + ", requestedMemory=" + resReq.getCapability().getMemory()
        + ", maxMemory=" + maximumResource.getMemory());
  }
  if (resReq.getCapability().getVirtualCores() < 0 ||
      resReq.getCapability().getVirtualCores() >
      maximumResource.getVirtualCores()) {
    throw new InvalidResourceRequestException("Invalid resource request"
        + ", requested virtual cores < 0"
        + ", or requested virtual cores > max configured"
        + ", requestedVirtualCores="
        + resReq.getCapability().getVirtualCores()
        + ", maxVirtualCores=" + maximumResource.getVirtualCores());
  }
  String labelExp = resReq.getNodeLabelExpression();

  // we don't allow specify label expression other than resourceName=ANY now
  if (!ResourceRequest.ANY.equals(resReq.getResourceName())
      && labelExp != null && !labelExp.trim().isEmpty()) {
    throw new InvalidResourceRequestException(
        "Invailid resource request, queue=" + queueInfo.getQueueName()
            + " specified node label expression in a "
            + "resource request has resource name = "
            + resReq.getResourceName());
  }
  
  // we don't allow specify label expression with more than one node labels now
  if (labelExp != null && labelExp.contains("&&")) {
    throw new InvalidResourceRequestException(
        "Invailid resource request, queue=" + queueInfo.getQueueName()
            + " specified more than one node label "
            + "in a node label expression, node label expression = "
            + labelExp);
  }
  
  if (labelExp != null && !labelExp.trim().isEmpty() && queueInfo != null) {
    if (!checkQueueLabelExpression(queueInfo.getAccessibleNodeLabels(),
        labelExp, rmContext)) {
      throw new InvalidResourceRequestException("Invalid resource request"
          + ", queue="
          + queueInfo.getQueueName()
          + " doesn't have permission to access all labels "
          + "in resource request. labelExpression of resource request="
          + labelExp
          + ". Queue labels="
          + (queueInfo.getAccessibleNodeLabels() == null ? "" : StringUtils.join(queueInfo
              .getAccessibleNodeLabels().iterator(), ',')));
    }
  }
}