Java Code Examples for org.apache.hadoop.yarn.api.records.ResourceRequest#getNumContainers()

private Resource getTotalResource(List<ResourceRequest> requests) {
  Resource totalResource = Resource.newInstance(0, 0, 0);
  if (requests == null) {
    return totalResource;
  }
  for (ResourceRequest request : requests) {
    if (request.getNumContainers() == 0) {
      continue;
    }
    if (request.getResourceName().equals(ResourceRequest.ANY)) {
      Resources.addTo(
        totalResource,
        Resources.multiply(request.getCapability(),
          request.getNumContainers()));
    }
  }
  return totalResource;
}
 

private int assignRackLocalContainers(FiCaSchedulerNode node, 
    FiCaSchedulerApp application, Priority priority) {
  int assignedContainers = 0;
  ResourceRequest request = 
    application.getResourceRequest(priority, node.getRMNode().getRackName());
  if (request != null) {
    // Don't allocate on this rack if the application doens't need containers
    ResourceRequest offSwitchRequest =
        application.getResourceRequest(priority, ResourceRequest.ANY);
    if (offSwitchRequest.getNumContainers() <= 0) {
      return 0;
    }
    
    int assignableContainers = 
      Math.min(
          getMaxAllocatableContainers(application, priority, node, 
              NodeType.RACK_LOCAL), 
              request.getNumContainers());
    assignedContainers = 
      assignContainer(node, application, priority, 
          assignableContainers, request, NodeType.RACK_LOCAL);
  }
  return assignedContainers;
}
 

private int assignNodeLocalContainers(FiCaSchedulerNode node, 
    FiCaSchedulerApp application, Priority priority) {
  int assignedContainers = 0;
  ResourceRequest request = 
    application.getResourceRequest(priority, node.getNodeName());
  if (request != null) {
    // Don't allocate on this node if we don't need containers on this rack
    ResourceRequest rackRequest =
        application.getResourceRequest(priority, 
            node.getRMNode().getRackName());
    if (rackRequest == null || rackRequest.getNumContainers() <= 0) {
      return 0;
    }
    
    int assignableContainers = 
      Math.min(
          getMaxAllocatableContainers(application, priority, node, 
              NodeType.NODE_LOCAL), 
              request.getNumContainers());
    assignedContainers = 
      assignContainer(node, application, priority, 
          assignableContainers, request, NodeType.NODE_LOCAL);
  }
  return assignedContainers;
}
 

private int assignNodeLocalContainers(FiCaSchedulerNode node, 
    FiCaSchedulerApp application, Priority priority) {
  int assignedContainers = 0;
  ResourceRequest request = 
    application.getResourceRequest(priority, node.getNodeName());
  if (request != null) {
    // Don't allocate on this node if we don't need containers on this rack
    ResourceRequest rackRequest =
        application.getResourceRequest(priority, 
            node.getRMNode().getRackName());
    if (rackRequest == null || rackRequest.getNumContainers() <= 0) {
      return 0;
    }
    
    int assignableContainers = 
      Math.min(
          getMaxAllocatableContainers(application, priority, node, 
              NodeType.NODE_LOCAL), 
              request.getNumContainers());
    assignedContainers = 
      assignContainer(node, application, priority, 
          assignableContainers, request, NodeType.NODE_LOCAL);
  }
  return assignedContainers;
}
 

/**
 * Whether this app has containers requests that could be satisfied on the
 * given node, if the node had full space.
 */
public boolean hasContainerForNode(Priority prio, FSSchedulerNode node) {
  ResourceRequest anyRequest = getResourceRequest(prio, ResourceRequest.ANY);
  ResourceRequest rackRequest = getResourceRequest(prio, node.getRackName());
  ResourceRequest nodeRequest = getResourceRequest(prio, node.getNodeName());

  return
      // There must be outstanding requests at the given priority:
      anyRequest != null && anyRequest.getNumContainers() > 0 &&
          // If locality relaxation is turned off at *-level, there must be a
          // non-zero request for the node's rack:
          (anyRequest.getRelaxLocality() ||
              (rackRequest != null && rackRequest.getNumContainers() > 0)) &&
          // If locality relaxation is turned off at rack-level, there must be a
          // non-zero request at the node:
          (rackRequest == null || rackRequest.getRelaxLocality() ||
              (nodeRequest != null && nodeRequest.getNumContainers() > 0)) &&
          // The requested container must be able to fit on the node:
          Resources.lessThanOrEqual(RESOURCE_CALCULATOR, null,
              anyRequest.getCapability(), node.getRMNode().getTotalCapability());
}
 

@Override
public AllocateResponse allocate(AllocateRequest request)
    throws YarnException, IOException {
  lastAsk = request.getAskList();
  for (ResourceRequest req : lastAsk) {
    if (ResourceRequest.ANY.equals(req.getResourceName())) {
      Priority priority = req.getPriority();
      if (priority.equals(RMContainerAllocator.PRIORITY_MAP)) {
        lastAnyAskMap = req.getNumContainers();
      } else if (priority.equals(RMContainerAllocator.PRIORITY_REDUCE)){
        lastAnyAskReduce = req.getNumContainers();
      }
    }
  }
  AllocateResponse response =  AllocateResponse.newInstance(
      request.getResponseId(),
      containersToComplete, containersToAllocate,
      Collections.<NodeReport>emptyList(),
      Resource.newInstance(512000, 1024, 1024), null, 10, null,
      Collections.<NMToken>emptyList());
  containersToComplete.clear();
  containersToAllocate.clear();
  return response;
}
 

synchronized private void decrementOutstanding(
    ResourceRequest offSwitchRequest) {
  int numOffSwitchContainers = offSwitchRequest.getNumContainers() - 1;

  // Do not remove ANY
  offSwitchRequest.setNumContainers(numOffSwitchContainers);
  
  // Do we have any outstanding requests?
  // If there is nothing, we need to deactivate this application
  if (numOffSwitchContainers == 0) {
    checkForDeactivation();
  }
  
  queue.decPendingResource(offSwitchRequest.getNodeLabelExpression(),
      offSwitchRequest.getCapability());
}
 

/**
 * Whether this app has containers requests that could be satisfied on the
 * given node, if the node had full space.
 */
public boolean hasContainerForNode(Priority prio, FSSchedulerNode node) {
  ResourceRequest anyRequest = getResourceRequest(prio, ResourceRequest.ANY);
  ResourceRequest rackRequest = getResourceRequest(prio, node.getRackName());
  ResourceRequest nodeRequest = getResourceRequest(prio, node.getNodeName());

  return
      // There must be outstanding requests at the given priority:
      anyRequest != null && anyRequest.getNumContainers() > 0 &&
          // If locality relaxation is turned off at *-level, there must be a
          // non-zero request for the node's rack:
          (anyRequest.getRelaxLocality() ||
              (rackRequest != null && rackRequest.getNumContainers() > 0)) &&
          // If locality relaxation is turned off at rack-level, there must be a
          // non-zero request at the node:
          (rackRequest == null || rackRequest.getRelaxLocality() ||
              (nodeRequest != null && nodeRequest.getNumContainers() > 0)) &&
          // The requested container must be able to fit on the node:
          Resources.lessThanOrEqual(RESOURCE_CALCULATOR, null,
              anyRequest.getCapability(), node.getRMNode().getTotalCapability());
}
 

protected void setRequestLimit(Priority priority, Resource capability,
    int limit) {
  if (limit < 0) {
    limit = Integer.MAX_VALUE;
  }
  ResourceRequest newReqLimit = ResourceRequest.newInstance(priority,
      ResourceRequest.ANY, capability, limit);
  ResourceRequest oldReqLimit = requestLimits.put(newReqLimit, newReqLimit);
  if (oldReqLimit == null || oldReqLimit.getNumContainers() < limit) {
    requestLimitsToUpdate.add(newReqLimit);
  }
}
 

private int getMaxAllocatableContainers(FiCaSchedulerApp application,
    Priority priority, FiCaSchedulerNode node, NodeType type) {
  int maxContainers = 0;
  
  ResourceRequest offSwitchRequest = 
    application.getResourceRequest(priority, ResourceRequest.ANY);
  if (offSwitchRequest != null) {
    maxContainers = offSwitchRequest.getNumContainers();
  }

  if (type == NodeType.OFF_SWITCH) {
    return maxContainers;
  }

  if (type == NodeType.RACK_LOCAL) {
    ResourceRequest rackLocalRequest = 
      application.getResourceRequest(priority, node.getRMNode().getRackName());
    if (rackLocalRequest == null) {
      return maxContainers;
    }

    maxContainers = Math.min(maxContainers, rackLocalRequest.getNumContainers());
  }

  if (type == NodeType.NODE_LOCAL) {
    ResourceRequest nodeLocalRequest = 
      application.getResourceRequest(priority, node.getRMNode().getNodeAddress());
    if (nodeLocalRequest != null) {
      maxContainers = Math.min(maxContainers, nodeLocalRequest.getNumContainers());
    }
  }

  return maxContainers;
}
 

protected void setRequestLimit(Priority priority, Resource capability,
    int limit) {
  if (limit < 0) {
    limit = Integer.MAX_VALUE;
  }
  ResourceRequest newReqLimit = ResourceRequest.newInstance(priority,
      ResourceRequest.ANY, capability, limit);
  ResourceRequest oldReqLimit = requestLimits.put(newReqLimit, newReqLimit);
  if (oldReqLimit == null || oldReqLimit.getNumContainers() < limit) {
    requestLimitsToUpdate.add(newReqLimit);
  }
}
 

private void applyRequestLimits() {
  Iterator<ResourceRequest> iter = requestLimits.values().iterator();
  while (iter.hasNext()) {
    ResourceRequest reqLimit = iter.next();
    int limit = reqLimit.getNumContainers();
    Map<String, Map<Resource, ResourceRequest>> remoteRequests =
        remoteRequestsTable.get(reqLimit.getPriority());
    Map<Resource, ResourceRequest> reqMap = (remoteRequests != null)
        ? remoteRequests.get(ResourceRequest.ANY) : null;
    ResourceRequest req = (reqMap != null)
        ? reqMap.get(reqLimit.getCapability()) : null;
    if (req == null) {
      continue;
    }
    // update an existing ask or send a new one if updating
    if (ask.remove(req) || requestLimitsToUpdate.contains(req)) {
      ResourceRequest newReq = req.getNumContainers() > limit
          ? reqLimit : req;
      ask.add(newReq);
      LOG.info("Applying ask limit of " + newReq.getNumContainers()
          + " for priority:" + reqLimit.getPriority()
          + " and capability:" + reqLimit.getCapability());
    }
    if (limit == Integer.MAX_VALUE) {
      iter.remove();
    }
  }
  requestLimitsToUpdate.clear();
}
 

private void decResourceRequest(String resourceName, Priority priority,
    ResourceRequest request) {
  request.setNumContainers(request.getNumContainers() - 1);
  if (request.getNumContainers() == 0) {
    requests.get(priority).remove(resourceName);
  }
}
 

private void applyRequestLimits() {
  Iterator<ResourceRequest> iter = requestLimits.values().iterator();
  while (iter.hasNext()) {
    ResourceRequest reqLimit = iter.next();
    int limit = reqLimit.getNumContainers();
    Map<String, Map<Resource, ResourceRequest>> remoteRequests =
        remoteRequestsTable.get(reqLimit.getPriority());
    Map<Resource, ResourceRequest> reqMap = (remoteRequests != null)
        ? remoteRequests.get(ResourceRequest.ANY) : null;
    ResourceRequest req = (reqMap != null)
        ? reqMap.get(reqLimit.getCapability()) : null;
    if (req == null) {
      continue;
    }
    // update an existing ask or send a new one if updating
    if (ask.remove(req) || requestLimitsToUpdate.contains(req)) {
      ResourceRequest newReq = req.getNumContainers() > limit
          ? reqLimit : req;
      ask.add(newReq);
      LOG.info("Applying ask limit of " + newReq.getNumContainers()
          + " for priority:" + reqLimit.getPriority()
          + " and capability:" + reqLimit.getCapability());
    }
    if (limit == Integer.MAX_VALUE) {
      iter.remove();
    }
  }
  requestLimitsToUpdate.clear();
}
 

synchronized private void decrementOutstanding(
    ResourceRequest offSwitchRequest) {
  int numOffSwitchContainers = offSwitchRequest.getNumContainers() - 1;

  // Do not remove ANY
  offSwitchRequest.setNumContainers(numOffSwitchContainers);
  
  // Do we have any outstanding requests?
  // If there is nothing, we need to deactivate this application
  if (numOffSwitchContainers == 0) {
    checkForDeactivation();
  }
}
 

private int getMaxAllocatableContainers(FiCaSchedulerApp application,
    Priority priority, FiCaSchedulerNode node, NodeType type) {
  int maxContainers = 0;
  
  ResourceRequest offSwitchRequest = 
    application.getResourceRequest(priority, ResourceRequest.ANY);
  if (offSwitchRequest != null) {
    maxContainers = offSwitchRequest.getNumContainers();
  }

  if (type == NodeType.OFF_SWITCH) {
    return maxContainers;
  }

  if (type == NodeType.RACK_LOCAL) {
    ResourceRequest rackLocalRequest = 
      application.getResourceRequest(priority, node.getRMNode().getRackName());
    if (rackLocalRequest == null) {
      return maxContainers;
    }

    maxContainers = Math.min(maxContainers, rackLocalRequest.getNumContainers());
  }

  if (type == NodeType.NODE_LOCAL) {
    ResourceRequest nodeLocalRequest = 
      application.getResourceRequest(priority, node.getRMNode().getNodeAddress());
    if (nodeLocalRequest != null) {
      maxContainers = Math.min(maxContainers, nodeLocalRequest.getNumContainers());
    }
  }

  return maxContainers;
}
 

private void decResourceRequest(Priority priority, String resourceName,
    Resource capability) {
  Map<String, Map<Resource, ResourceRequest>> remoteRequests =
    this.remoteRequestsTable.get(priority);
  Map<Resource, ResourceRequest> reqMap = remoteRequests.get(resourceName);
  if (reqMap == null) {
    // as we modify the resource requests by filtering out blacklisted hosts 
    // when they are added, this value may be null when being 
    // decremented
    if (LOG.isDebugEnabled()) {
      LOG.debug("Not decrementing resource as " + resourceName
          + " is not present in request table");
    }
    return;
  }
  ResourceRequest remoteRequest = reqMap.get(capability);

  if (LOG.isDebugEnabled()) {
    LOG.debug("BEFORE decResourceRequest:" + " applicationId="
        + applicationId.getId() + " priority=" + priority.getPriority()
        + " resourceName=" + resourceName + " numContainers="
        + remoteRequest.getNumContainers() + " #asks=" + ask.size());
  }

  if(remoteRequest.getNumContainers() > 0) {
    // based on blacklisting comments above we can end up decrementing more 
    // than requested. so guard for that.
    remoteRequest.setNumContainers(remoteRequest.getNumContainers() -1);
  }
  
  if (remoteRequest.getNumContainers() == 0) {
    reqMap.remove(capability);
    if (reqMap.size() == 0) {
      remoteRequests.remove(resourceName);
    }
    if (remoteRequests.size() == 0) {
      remoteRequestsTable.remove(priority);
    }
  }

  // send the updated resource request to RM
  // send 0 container count requests also to cancel previous requests
  addResourceRequestToAsk(remoteRequest);

  if (LOG.isDebugEnabled()) {
    LOG.info("AFTER decResourceRequest:" + " applicationId="
        + applicationId.getId() + " priority=" + priority.getPriority()
        + " resourceName=" + resourceName + " numContainers="
        + remoteRequest.getNumContainers() + " #asks=" + ask.size());
  }
}
 

private void createResourceRequestsTable(Block html) {
  AppInfo app =
      new AppInfo(rm, rm.getRMContext().getRMApps()
        .get(this.appAttemptId.getApplicationId()), true,
        WebAppUtils.getHttpSchemePrefix(conf));

  List<ResourceRequest> resourceRequests = app.getResourceRequests();
  if (resourceRequests == null || resourceRequests.isEmpty()) {
    return;
  }

  DIV<Hamlet> div = html.div(_INFO_WRAP);
  TABLE<DIV<Hamlet>> table =
      div.h3("Total Outstanding Resource Requests: "
        + getTotalResource(resourceRequests)).table(
            "#ResourceRequests");

  table.tr().
    th(_TH, "Priority").
    th(_TH, "ResourceName").
    th(_TH, "Capability").
    th(_TH, "NumContainers").
    th(_TH, "RelaxLocality").
    th(_TH, "NodeLabelExpression").
  _();

  boolean odd = false;
  for (ResourceRequest request : resourceRequests) {
    if (request.getNumContainers() == 0) {
      continue;
    }
    table.tr((odd = !odd) ? _ODD : _EVEN)
      .td(String.valueOf(request.getPriority()))
      .td(request.getResourceName())
      .td(String.valueOf(request.getCapability()))
      .td(String.valueOf(request.getNumContainers()))
      .td(String.valueOf(request.getRelaxLocality()))
      .td(request.getNodeLabelExpression() == null ? "N/A" : request
          .getNodeLabelExpression())._();
  }
  table._();
  div._();
}
 

/**还没有弄明白
 * The ApplicationMaster is updating resource requirements for the
 * application, by asking for more resources and releasing resources acquired
 * by the application.
 * 如果是recover的话表明该资源被请求过，现在只不过是被抢占了，所以需要RM重新分配资源
 *
 * @param requests resources to be acquired
 * @param recoverPreemptedRequest recover Resource Request on preemption
 */
synchronized public void updateResourceRequests(
    List<ResourceRequest> requests, boolean recoverPreemptedRequest) {
  QueueMetrics metrics = queue.getMetrics();
  
  // Update resource requests
  for (ResourceRequest request : requests) {
    Priority priority = request.getPriority();
    String resourceName = request.getResourceName();
    boolean updatePendingResources = false;
    ResourceRequest lastRequest = null;

    if (resourceName.equals(ResourceRequest.ANY)) {
      if (LOG.isDebugEnabled()) {
        LOG.debug("update:" + " application=" + applicationId + " request="
            + request);
      }
      updatePendingResources = true;
      
      // Premature optimization?
      // Assumes that we won't see more than one priority request updated
      // in one call, reasonable assumption... however, it's totally safe
      // to activate same application more than once.
      // Thus we don't need another loop ala the one in decrementOutstanding()  
      // which is needed during deactivate.
      if (request.getNumContainers() > 0) {
        activeUsersManager.activateApplication(user, applicationId);
      }
    }

    Map<String, ResourceRequest> asks = this.requests.get(priority);

    if (asks == null) {                      //在该优先级还没有请求，所以新建一个请求
      asks = new ConcurrentHashMap<String, ResourceRequest>();
      this.requests.put(priority, asks);
      this.priorities.add(priority);
    }
    lastRequest = asks.get(resourceName);    //上一次的请求

    if (recoverPreemptedRequest && lastRequest != null) {
      // Increment the number of containers to 1, as it is recovering a
      // single container.
      request.setNumContainers(lastRequest.getNumContainers() + 1);
    }

    asks.put(resourceName, request);
    if (updatePendingResources) {
      
      // Similarly, deactivate application?
      if (request.getNumContainers() <= 0) {
        LOG.info("checking for deactivate of application :"
            + this.applicationId);
        checkForDeactivation();
      }
      
      int lastRequestContainers = lastRequest != null ? lastRequest
          .getNumContainers() : 0;
      Resource lastRequestCapability = lastRequest != null ? lastRequest
          .getCapability() : Resources.none();
      metrics.incrPendingResources(user, request.getNumContainers(),
          request.getCapability());
      metrics.decrPendingResources(user, lastRequestContainers,
          lastRequestCapability);
    }
  }
}
 

private void createResourceRequestsTable(Block html) {
  AppInfo app =
      new AppInfo(rm, rm.getRMContext().getRMApps()
        .get(this.appAttemptId.getApplicationId()), true,
        WebAppUtils.getHttpSchemePrefix(conf));

  List<ResourceRequest> resourceRequests = app.getResourceRequests();
  if (resourceRequests == null || resourceRequests.isEmpty()) {
    return;
  }

  DIV<Hamlet> div = html.div(_INFO_WRAP);
  TABLE<DIV<Hamlet>> table =
      div.h3("Total Outstanding Resource Requests: "
        + getTotalResource(resourceRequests)).table(
            "#ResourceRequests");

  table.tr().
    th(_TH, "Priority").
    th(_TH, "ResourceName").
    th(_TH, "Capability").
    th(_TH, "NumContainers").
    th(_TH, "RelaxLocality").
    th(_TH, "NodeLabelExpression").
  _();

  boolean odd = false;
  for (ResourceRequest request : resourceRequests) {
    if (request.getNumContainers() == 0) {
      continue;
    }
    table.tr((odd = !odd) ? _ODD : _EVEN)
      .td(String.valueOf(request.getPriority()))
      .td(request.getResourceName())
      .td(String.valueOf(request.getCapability()))
      .td(String.valueOf(request.getNumContainers()))
      .td(String.valueOf(request.getRelaxLocality()))
      .td(request.getNodeLabelExpression() == null ? "N/A" : request
          .getNodeLabelExpression())._();
  }
  table._();
  div._();
}