Java Code Examples for org.apache.hadoop.yarn.util.resource.Resources#none()

private synchronized CSAssignment assignReservedContainer(
    FiCaSchedulerApp application, FiCaSchedulerNode node,
    RMContainer rmContainer, Resource clusterResource) {
  // Do we still need this reservation?
  Priority priority = rmContainer.getReservedPriority();
  if (application.getTotalRequiredResources(priority) == 0) {
    // Release
    return new CSAssignment(application, rmContainer);
  }

  // Try to assign if we have sufficient resources
  assignContainersOnNode(clusterResource, node, application, priority, 
      rmContainer, new ResourceLimits(Resources.none()));
  
  // Doesn't matter... since it's already charged for at time of reservation
  // "re-reservation" is *free*
  return new CSAssignment(Resources.none(), NodeType.NODE_LOCAL);
}
 

@Override
public Resource assignContainer(FSSchedulerNode node) {
  Resource assigned = Resources.none();

  // If this queue is over its limit, reject
  if (!assignContainerPreCheck(node)) {
    return assigned;
  }

  Collections.sort(childQueues, policy.getComparator());
  for (FSQueue child : childQueues) {
    assigned = child.assignContainer(node);
    if (!Resources.equals(assigned, Resources.none())) {
      break;
    }
  }
  return assigned;
}
 

public Resource getQueueResource(String queueName, Set<String> queueLabels,
    Resource clusterResource) {
  try {
    readLock.lock();
    if (queueLabels.contains(ANY)) {
      return clusterResource;
    }
    Queue q = queueCollections.get(queueName);
    if (null == q) {
      return Resources.none();
    }
    return q.resource;
  } finally {
    readLock.unlock();
  }
}
 

/**
 * Called when this application already has an existing reservation on the
 * given node.  Sees whether we can turn the reservation into an allocation.
 * Also checks whether the application needs the reservation anymore, and
 * releases it if not.
 *
 * @param node
 *     Node that the application has an existing reservation on
 */
public Resource assignReservedContainer(FSSchedulerNode node) {
  RMContainer rmContainer = node.getReservedContainer();
  Priority priority = rmContainer.getReservedPriority();

  // Make sure the application still needs requests at this priority
  if (getTotalRequiredResources(priority) == 0) {
    unreserve(priority, node);
    return Resources.none();
  }

  // Fail early if the reserved container won't fit.
  // Note that we have an assumption here that there's only one container size
  // per priority.
  if (!Resources.fitsIn(node.getReservedContainer().getReservedResource(),
      node.getAvailableResource())) {
    return Resources.none();
  }

  return assignContainer(node, true);
}
 

private Schedulable createSchedulable(int memUsage, int cpuUsage,
    ResourceWeights weights, int minMemShare, int minCpuShare) {
  Resource usage = BuilderUtils.newResource(memUsage, cpuUsage);
  Resource minShare = BuilderUtils.newResource(minMemShare, minCpuShare);
  return new FakeSchedulable(minShare,
      Resources.createResource(Integer.MAX_VALUE, Integer.MAX_VALUE),
      weights, Resources.none(), usage, 0l);
}
 

@Override
public Resource assignContainer(FSSchedulerNode node) {
  Resource assigned = Resources.none();
  if (LOG.isDebugEnabled()) {
    LOG.debug("Node " + node.getNodeName() + " offered to queue: " +
        getName());
  }

  if (!assignContainerPreCheck(node)) {
    return assigned;
  }

  Comparator<Schedulable> comparator = policy.getComparator();
  writeLock.lock();
  try {
    Collections.sort(runnableApps, comparator);
  } finally {
    writeLock.unlock();
  }
  // Release write lock here for better performance and avoiding deadlocks.
  // runnableApps can be in unsorted state because of this section,
  // but we can accept it in practice since the probability is low.
  readLock.lock();
  try {
    for (FSAppAttempt sched : runnableApps) {
      if (SchedulerAppUtils.isBlacklisted(sched, node, LOG)) {
        continue;
      }

      assigned = sched.assignContainer(node);
      if (!assigned.equals(Resources.none())) {
        break;
      }
    }
  } finally {
    readLock.unlock();
  }
  return assigned;
}
 

private Resource _get(String label, ResourceType type) {
  try {
    readLock.lock();
    UsageByLabel usage = usages.get(label);
    if (null == usage) {
      return Resources.none();
    }
    return normalize(usage.resArr[type.idx]);
  } finally {
    readLock.unlock();
  }
}
 

private Resource assignOffSwitchContainers(Resource clusterResource,
    ResourceRequest offSwitchResourceRequest, FiCaSchedulerNode node,
    FiCaSchedulerApp application, Priority priority,
    RMContainer reservedContainer, MutableObject allocatedContainer,
    ResourceLimits currentResoureLimits) {
  if (canAssign(application, priority, node, NodeType.OFF_SWITCH,
      reservedContainer)) {
    return assignContainer(clusterResource, node, application, priority,
        offSwitchResourceRequest, NodeType.OFF_SWITCH, reservedContainer,
        allocatedContainer, currentResoureLimits);
  }
  
  return Resources.none();
}
 

@Lock(CSQueue.class)
public static void updateQueueStatistics(
    final ResourceCalculator calculator,
    final CSQueue childQueue, final CSQueue parentQueue, 
    final Resource clusterResource, final Resource minimumAllocation) {
  Resource queueLimit = Resources.none();
  Resource usedResources = childQueue.getUsedResources();
  
  float absoluteUsedCapacity = 0.0f;
  float usedCapacity = 0.0f;

  if (Resources.greaterThan(
      calculator, clusterResource, clusterResource, Resources.none())) {
    queueLimit = 
        Resources.multiply(clusterResource, childQueue.getAbsoluteCapacity());
    absoluteUsedCapacity = 
        Resources.divide(calculator, clusterResource, 
            usedResources, clusterResource);
    usedCapacity = 
        Resources.equals(queueLimit, Resources.none()) ? 0 :
        Resources.divide(calculator, clusterResource, 
            usedResources, queueLimit);
  }

  childQueue.setUsedCapacity(usedCapacity);
  childQueue.setAbsoluteUsedCapacity(absoluteUsedCapacity);
  
  Resource available = Resources.subtract(queueLimit, usedResources);
  childQueue.getMetrics().setAvailableResourcesToQueue(
      Resources.max(
          calculator, 
          clusterResource, 
          available, 
          Resources.none()
          )
      );
 }
 

private Resource _get(String label, ResourceType type) {
  try {
    readLock.lock();
    UsageByLabel usage = usages.get(label);
    if (null == usage) {
      return Resources.none();
    }
    return normalize(usage.resArr[type.idx]);
  } finally {
    readLock.unlock();
  }
}
 

@Private
public Resource getResourceLimit() {
  Resource headRoom = getAvailableResources();
  if (headRoom == null) {
    headRoom = Resources.none();
  }
  Resource assignedMapResource =
      Resources.multiply(mapResourceRequest, assignedRequests.maps.size());
  Resource assignedReduceResource =
      Resources.multiply(reduceResourceRequest,
        assignedRequests.reduces.size());
  return Resources.add(headRoom,
    Resources.add(assignedMapResource, assignedReduceResource));
}
 

public Resource getResourceByLabel(String label, Resource clusterResource) {
  label = normalizeLabel(label);
  try {
    readLock.lock();
    if (null == labelCollections.get(label)) {
      return Resources.none();
    }
    return labelCollections.get(label).getResource();
  } finally {
    readLock.unlock();
  }
}
 

private Resource assignRackLocalContainers(Resource clusterResource,
    ResourceRequest rackLocalResourceRequest, FiCaSchedulerNode node,
    FiCaSchedulerApp application, Priority priority,
    RMContainer reservedContainer, MutableObject allocatedContainer,
    ResourceLimits currentResoureLimits) {
  if (canAssign(application, priority, node, NodeType.RACK_LOCAL,
      reservedContainer)) {
    return assignContainer(clusterResource, node, application, priority,
        rackLocalResourceRequest, NodeType.RACK_LOCAL, reservedContainer,
        allocatedContainer, currentResoureLimits);
  }
  
  return Resources.none();
}
 

private Resource assignNodeLocalContainers(Resource clusterResource,
    ResourceRequest nodeLocalResourceRequest, FiCaSchedulerNode node,
    FiCaSchedulerApp application, Priority priority,
    RMContainer reservedContainer, MutableObject allocatedContainer,
    ResourceLimits currentResoureLimits) {
  if (canAssign(application, priority, node, NodeType.NODE_LOCAL, 
      reservedContainer)) {
    return assignContainer(clusterResource, node, application, priority,
        nodeLocalResourceRequest, NodeType.NODE_LOCAL, reservedContainer,
        allocatedContainer, currentResoureLimits);
  }
  
  return Resources.none();
}
 

public Resource getResourceByLabel(String label, Resource clusterResource) {
  label = normalizeLabel(label);
  try {
    readLock.lock();
    if (null == labelCollections.get(label)) {
      return Resources.none();
    }
    return labelCollections.get(label).getResource();
  } finally {
    readLock.unlock();
  }
}
 

private static Resource normalize(Resource res) {
  if (res == null) {
    return Resources.none();
  }
  return res;
}
 

/**
 * Get the minimum resource allocation for the given queue.
 * @return the cap set on this queue, or 0 if not set.
 */
public Resource getMinResources(String queue) {
  Resource minQueueResource = minQueueResources.get(queue);
  return (minQueueResource == null) ? Resources.none() : minQueueResource;
}
 

private static Resource normalize(Resource res) {
  if (res == null) {
    return Resources.none();
  }
  return res;
}
 

private CSAssignment assignContainersOnNode(Resource clusterResource,
    FiCaSchedulerNode node, FiCaSchedulerApp application, Priority priority,
    RMContainer reservedContainer, ResourceLimits currentResoureLimits) {
  Resource assigned = Resources.none();

  NodeType requestType = null;
  MutableObject allocatedContainer = new MutableObject();
  // Data-local
  ResourceRequest nodeLocalResourceRequest =
      application.getResourceRequest(priority, node.getNodeName());
  if (nodeLocalResourceRequest != null) {
    requestType = NodeType.NODE_LOCAL;
    assigned =
        assignNodeLocalContainers(clusterResource, nodeLocalResourceRequest, 
          node, application, priority, reservedContainer,
          allocatedContainer, currentResoureLimits);
    if (Resources.greaterThan(resourceCalculator, clusterResource,
        assigned, Resources.none())) {

      //update locality statistics
      if (allocatedContainer.getValue() != null) {
        application.incNumAllocatedContainers(NodeType.NODE_LOCAL,
          requestType);
      }
      return new CSAssignment(assigned, NodeType.NODE_LOCAL);
    }
  }

  // Rack-local
  ResourceRequest rackLocalResourceRequest =
      application.getResourceRequest(priority, node.getRackName());
  if (rackLocalResourceRequest != null) {
    if (!rackLocalResourceRequest.getRelaxLocality()) {
      return SKIP_ASSIGNMENT;
    }

    if (requestType != NodeType.NODE_LOCAL) {
      requestType = NodeType.RACK_LOCAL;
    }

    assigned = 
        assignRackLocalContainers(clusterResource, rackLocalResourceRequest, 
          node, application, priority, reservedContainer,
          allocatedContainer, currentResoureLimits);
    if (Resources.greaterThan(resourceCalculator, clusterResource,
        assigned, Resources.none())) {

      //update locality statistics
      if (allocatedContainer.getValue() != null) {
        application.incNumAllocatedContainers(NodeType.RACK_LOCAL,
          requestType);
      }
      return new CSAssignment(assigned, NodeType.RACK_LOCAL);
    }
  }
  
  // Off-switch
  ResourceRequest offSwitchResourceRequest =
      application.getResourceRequest(priority, ResourceRequest.ANY);
  if (offSwitchResourceRequest != null) {
    if (!offSwitchResourceRequest.getRelaxLocality()) {
      return SKIP_ASSIGNMENT;
    }
    if (requestType != NodeType.NODE_LOCAL
        && requestType != NodeType.RACK_LOCAL) {
      requestType = NodeType.OFF_SWITCH;
    }

    assigned =
        assignOffSwitchContainers(clusterResource, offSwitchResourceRequest,
          node, application, priority, reservedContainer,
          allocatedContainer, currentResoureLimits);

    // update locality statistics
    if (allocatedContainer.getValue() != null) {
      application.incNumAllocatedContainers(NodeType.OFF_SWITCH, requestType);
    }
    return new CSAssignment(assigned, NodeType.OFF_SWITCH);
  }
  
  return SKIP_ASSIGNMENT;
}
 

/**
 * Assign a container to this node to facilitate {@code request}. If node does
 * not have enough memory, create a reservation. This is called once we are
 * sure the particular request should be facilitated by this node.
 *
 * @param node
 *     The node to try placing the container on.
 * @param request
 *     The ResourceRequest we're trying to satisfy.
 * @param type
 *     The locality of the assignment.
 * @param reserved
 *     Whether there's already a container reserved for this app on the node.
 * @return
 *     If an assignment was made, returns the resources allocated to the
 *     container.  If a reservation was made, returns
 *     FairScheduler.CONTAINER_RESERVED.  If no assignment or reservation was
 *     made, returns an empty resource.
 */
private Resource assignContainer(
    FSSchedulerNode node, ResourceRequest request, NodeType type,
    boolean reserved) {

  // How much does this request need?
  Resource capability = request.getCapability();

  // How much does the node have?
  Resource available = node.getAvailableResource();

  Container container = null;
  if (reserved) {
    container = node.getReservedContainer().getContainer();
  } else {
    container = createContainer(node, capability, request.getPriority());
  }

  // Can we allocate a container on this node?
  if (Resources.fitsIn(capability, available)) {
    // Inform the application of the new container for this request
    RMContainer allocatedContainer =
        allocate(type, node, request.getPriority(), request, container);
    if (allocatedContainer == null) {
      // Did the application need this resource?
      if (reserved) {
        unreserve(request.getPriority(), node);
      }
      return Resources.none();
    }

    // If we had previously made a reservation, delete it
    if (reserved) {
      unreserve(request.getPriority(), node);
    }

    // Inform the node
    node.allocateContainer(allocatedContainer);

    // If this container is used to run AM, update the leaf queue's AM usage
    if (getLiveContainers().size() == 1 && !getUnmanagedAM()) {
      getQueue().addAMResourceUsage(container.getResource());
      setAmRunning(true);
    }

    return container.getResource();
  } else {
    if (!FairScheduler.fitsInMaxShare(getQueue(), capability)) {
      return Resources.none();
    }

    // The desired container won't fit here, so reserve
    reserve(request.getPriority(), node, container, reserved);

    return FairScheduler.CONTAINER_RESERVED;
  }
}