Java Code Examples for org.apache.hadoop.yarn.api.records.Container#getResource()

The following examples show how to use org.apache.hadoop.yarn.api.records.Container#getResource() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: NMSimulator.java    From hadoop with Apache License 2.0 6 votes vote down vote up 
/**
 * launch a new container with the given life time
 */
public void addNewContainer(Container container, long lifeTimeMS) {
  LOG.debug(MessageFormat.format("NodeManager {0} launches a new " +
          "container ({1}).", node.getNodeID(), container.getId()));
  if (lifeTimeMS != -1) {
    // normal container
    ContainerSimulator cs = new ContainerSimulator(container.getId(),
            container.getResource(), lifeTimeMS + System.currentTimeMillis(),
            lifeTimeMS);
    containerQueue.add(cs);
    runningContainers.put(cs.getId(), cs);
  } else {
    // AM container
    // -1 means AMContainer
    synchronized(amContainerList) {
      amContainerList.add(container.getId());
    }
  }
}
Example 2
Source File: NMSimulator.java    From big-c with Apache License 2.0 6 votes vote down vote up 
/**
 * launch a new container with the given life time
 */
public void addNewContainer(Container container, long lifeTimeMS) {
  LOG.debug(MessageFormat.format("NodeManager {0} launches a new " +
          "container ({1}).", node.getNodeID(), container.getId()));
  if (lifeTimeMS != -1) {
    // normal container
    ContainerSimulator cs = new ContainerSimulator(container.getId(),
            container.getResource(), lifeTimeMS + System.currentTimeMillis(),
            lifeTimeMS);
    containerQueue.add(cs);
    runningContainers.put(cs.getId(), cs);
  } else {
    // AM container
    // -1 means AMContainer
    synchronized(amContainerList) {
      amContainerList.add(container.getId());
    }
  }
}
Example 3
Source File: YarnTaskSchedulerService.java    From incubator-tez with Apache License 2.0 6 votes vote down vote up 
private CookieContainerRequest getMatchingRequestWithPriority(
    Container container,
    String location) {
  Priority priority = container.getPriority();
  Resource capability = container.getResource();
  List<? extends Collection<CookieContainerRequest>> requestsList =
      amRmClient.getMatchingRequests(priority, location, capability);

  if (!requestsList.isEmpty()) {
    // pick first one
    for (Collection<CookieContainerRequest> requests : requestsList) {
      for (CookieContainerRequest cookieContainerRequest : requests) {
        if (canAssignTaskToContainer(cookieContainerRequest, container)) {
          return cookieContainerRequest;
        }
      }
    }
  }

  return null;
}
Example 4
Source File: YarnTaskSchedulerService.java    From tez with Apache License 2.0 6 votes vote down vote up 
private CookieContainerRequest getMatchingRequestWithPriority(
    Container container,
    String location) {
  Priority priority = container.getPriority();
  Resource capability = container.getResource();
  List<? extends Collection<CookieContainerRequest>> requestsList =
      amRmClient.getMatchingRequests(priority, location, capability);

  if (!requestsList.isEmpty()) {
    // pick first one
    for (Collection<CookieContainerRequest> requests : requestsList) {
      for (CookieContainerRequest cookieContainerRequest : requests) {
        if (canAssignTaskToContainer(cookieContainerRequest, container)) {
          return cookieContainerRequest;
        }
      }
    }
  }

  return null;
}
Example 5
Source File: FSAppAttempt.java    From hadoop with Apache License 2.0 4 votes vote down vote up 
/**
 * Assign a container to this node to facilitate {@code request}. If node does
 * not have enough memory, create a reservation. This is called once we are
 * sure the particular request should be facilitated by this node.
 *
 * @param node
 *     The node to try placing the container on.
 * @param request
 *     The ResourceRequest we're trying to satisfy.
 * @param type
 *     The locality of the assignment.
 * @param reserved
 *     Whether there's already a container reserved for this app on the node.
 * @return
 *     If an assignment was made, returns the resources allocated to the
 *     container.  If a reservation was made, returns
 *     FairScheduler.CONTAINER_RESERVED.  If no assignment or reservation was
 *     made, returns an empty resource.
 */
private Resource assignContainer(
    FSSchedulerNode node, ResourceRequest request, NodeType type,
    boolean reserved) {

  // How much does this request need?
  Resource capability = request.getCapability();

  // How much does the node have?
  Resource available = node.getAvailableResource();

  Container container = null;
  if (reserved) {
    container = node.getReservedContainer().getContainer();
  } else {
    container = createContainer(node, capability, request.getPriority());
  }

  // Can we allocate a container on this node?
  if (Resources.fitsIn(capability, available)) {
    // Inform the application of the new container for this request
    RMContainer allocatedContainer =
        allocate(type, node, request.getPriority(), request, container);
    if (allocatedContainer == null) {
      // Did the application need this resource?
      if (reserved) {
        unreserve(request.getPriority(), node);
      }
      return Resources.none();
    }

    // If we had previously made a reservation, delete it
    if (reserved) {
      unreserve(request.getPriority(), node);
    }

    // Inform the node
    node.allocateContainer(allocatedContainer);

    // If this container is used to run AM, update the leaf queue's AM usage
    if (getLiveContainers().size() == 1 && !getUnmanagedAM()) {
      getQueue().addAMResourceUsage(container.getResource());
      setAmRunning(true);
    }

    return container.getResource();
  } else {
    if (!FairScheduler.fitsInMaxShare(getQueue(), capability)) {
      return Resources.none();
    }

    // The desired container won't fit here, so reserve
    reserve(request.getPriority(), node, container, reserved);

    return FairScheduler.CONTAINER_RESERVED;
  }
}
Example 6
Source File: FSAppAttempt.java    From big-c with Apache License 2.0 4 votes vote down vote up 
/**
 * Assign a container to this node to facilitate {@code request}. If node does
 * not have enough memory, create a reservation. This is called once we are
 * sure the particular request should be facilitated by this node.
 *
 * @param node
 *     The node to try placing the container on.
 * @param request
 *     The ResourceRequest we're trying to satisfy.
 * @param type
 *     The locality of the assignment.
 * @param reserved
 *     Whether there's already a container reserved for this app on the node.
 * @return
 *     If an assignment was made, returns the resources allocated to the
 *     container.  If a reservation was made, returns
 *     FairScheduler.CONTAINER_RESERVED.  If no assignment or reservation was
 *     made, returns an empty resource.
 */
private Resource assignContainer(
    FSSchedulerNode node, ResourceRequest request, NodeType type,
    boolean reserved) {

  // How much does this request need?
  Resource capability = request.getCapability();

  // How much does the node have?
  Resource available = node.getAvailableResource();

  Container container = null;
  if (reserved) {
    container = node.getReservedContainer().getContainer();
  } else {
    container = createContainer(node, capability, request.getPriority());
  }

  // Can we allocate a container on this node?
  if (Resources.fitsIn(capability, available)) {
    // Inform the application of the new container for this request
    RMContainer allocatedContainer =
        allocate(type, node, request.getPriority(), request, container);
    if (allocatedContainer == null) {
      // Did the application need this resource?
      if (reserved) {
        unreserve(request.getPriority(), node);
      }
      return Resources.none();
    }

    // If we had previously made a reservation, delete it
    if (reserved) {
      unreserve(request.getPriority(), node);
    }

    // Inform the node
    node.allocateContainer(allocatedContainer);

    // If this container is used to run AM, update the leaf queue's AM usage
    if (getLiveContainers().size() == 1 && !getUnmanagedAM()) {
      getQueue().addAMResourceUsage(container.getResource());
      setAmRunning(true);
    }

    return container.getResource();
  } else {
    if (!FairScheduler.fitsInMaxShare(getQueue(), capability)) {
      return Resources.none();
    }

    // The desired container won't fit here, so reserve
    reserve(request.getPriority(), node, container, reserved);

    return FairScheduler.CONTAINER_RESERVED;
  }
}
Example 7
Source File: LeafQueue.java    From big-c with Apache License 2.0 4 votes vote down vote up 
@Override
public void completedContainer(Resource clusterResource, 
    FiCaSchedulerApp application, FiCaSchedulerNode node, RMContainer rmContainer, 
    ContainerStatus containerStatus, RMContainerEventType event, CSQueue childQueue,
    boolean sortQueues) {
  if (application != null) {

    boolean removed = false;

    // Careful! Locking order is important!
    synchronized (this) {

      Container container = rmContainer.getContainer();
      
      Resource toRelease = null;

      // Inform the application & the node
      // Note: It's safe to assume that all state changes to RMContainer
      // happen under scheduler's lock... 
      // So, this is, in effect, a transaction across application & node
      if (rmContainer.getState() == RMContainerState.RESERVED) {
        removed = unreserve(application, rmContainer.getReservedPriority(),
            node, rmContainer);
        toRelease = container.getResource();
      } else {
        //for container suspend event	       
        if (event == RMContainerEventType.SUSPEND){
         removed =
      	application.containerSuspend(rmContainer, containerStatus, event);
      	//suspend and resume in fifo order
          if(!suspendedApps.contains(application.getApplicationAttemptId())){
          	LOG.info(application.getApplicationAttemptId()+"into suspending list");
          	suspendedApps.add(application.getApplicationAttemptId());
          }
          //we suspend the container on this node
      	node.suspendContainer(container,rmContainer.getLastPreemptedResource());
      	toRelease = rmContainer.getLastPreemptedResource();
      	
        }else{
      	toRelease = rmContainer.getCurrentUsedResource();
      	removed =
      	            application.containerCompleted(rmContainer, containerStatus, event);
      	            node.releaseContainer(container,toRelease);
      	          //for container suspend event
      	 //in case of completing a suspended container 
        }
      }
      // Book-keeping
      if (removed) {
         //更新资源试用情况
        if(event == RMContainerEventType.SUSPEND){
        releaseResource(clusterResource, application,
      		  toRelease, node.getLabels(),true);
        }else{
        releaseResource(clusterResource, application,
          		  toRelease, node.getLabels(),false);  
        }
        LOG.info("completedContainer" +
            " container=" + container +
            " queue=" + this +
            " cluster=" + clusterResource);
      }
    }

    if (removed) {
      // Inform the parent queue _outside_ of the leaf-queue lock
      getParent().completedContainer(clusterResource, application, node,
        rmContainer, null, event, this, sortQueues);
    }
  }
}
Example 8
Source File: YarnTaskSchedulerService.java    From incubator-tez with Apache License 2.0 4 votes vote down vote up 
private CookieContainerRequest getMatchingRequestWithoutPriority(
    Container container,
    String location,
    boolean considerContainerAffinity) {
  Resource capability = container.getResource();
  List<? extends Collection<CookieContainerRequest>> pRequestsList =
    amRmClient.getMatchingRequestsForTopPriority(location, capability);
  if (considerContainerAffinity && 
      !priorityHasAffinity.contains(amRmClient.getTopPriority())) {
    considerContainerAffinity = false;
  }
  if (pRequestsList == null || pRequestsList.isEmpty()) {
    return null;
  }
  CookieContainerRequest firstMatch = null;
  for (Collection<CookieContainerRequest> requests : pRequestsList) {
    for (CookieContainerRequest cookieContainerRequest : requests) {
      if (firstMatch == null || // we dont have a match. So look for one 
          // we have a match but are looking for a better container level match.
          // skip the expensive canAssignTaskToContainer() if the request is 
          // not affinitized to the container
          container.getId().equals(cookieContainerRequest.getAffinitizedContainer())
          ) {
        if (canAssignTaskToContainer(cookieContainerRequest, container)) {
          // request matched to container
          if (!considerContainerAffinity) {
            return cookieContainerRequest;
          }
          ContainerId affCId = cookieContainerRequest.getAffinitizedContainer();
          boolean canMatchTaskWithAffinity = true;
          if (affCId == null || 
              !heldContainers.containsKey(affCId) ||
              inUseContainers.contains(affCId)) {
            // affinity not specified
            // affinitized container is no longer held
            // affinitized container is in use
            canMatchTaskWithAffinity = false;
          }
          if (canMatchTaskWithAffinity) {
            if (container.getId().equals(
                cookieContainerRequest.getAffinitizedContainer())) {
              // container level match
              if (LOG.isDebugEnabled()) {
                LOG.debug("Matching with affinity for request: "
                    + cookieContainerRequest + " container: " + affCId);
              }
              return cookieContainerRequest;
            }
            if (LOG.isDebugEnabled()) {
              LOG.debug("Skipping request for container " + container.getId()
                  + " due to affinity. Request: " + cookieContainerRequest
                  + " affContainer: " + affCId);
            }
          } else {
            firstMatch = cookieContainerRequest;
          }
        }
      }
    }
  }
  
  return firstMatch;
}
Example 9
Source File: YarnTaskSchedulerService.java    From tez with Apache License 2.0 4 votes vote down vote up 
private CookieContainerRequest getMatchingRequestWithoutPriority(
    Container container,
    String location,
    boolean considerContainerAffinity) {
  Resource capability = container.getResource();
  List<? extends Collection<CookieContainerRequest>> pRequestsList =
    amRmClient.getMatchingRequestsForTopPriority(location, capability);
  if (considerContainerAffinity && 
      !priorityHasAffinity.contains(amRmClient.getTopPriority())) {
    considerContainerAffinity = false;
  }
  if (pRequestsList == null || pRequestsList.isEmpty()) {
    return null;
  }
  CookieContainerRequest firstMatch = null;
  for (Collection<CookieContainerRequest> requests : pRequestsList) {
    for (CookieContainerRequest cookieContainerRequest : requests) {
      if (firstMatch == null || // we dont have a match. So look for one 
          // we have a match but are looking for a better container level match.
          // skip the expensive canAssignTaskToContainer() if the request is 
          // not affinitized to the container
          container.getId().equals(cookieContainerRequest.getAffinitizedContainer())
          ) {
        if (canAssignTaskToContainer(cookieContainerRequest, container)) {
          // request matched to container
          if (!considerContainerAffinity) {
            return cookieContainerRequest;
          }
          ContainerId affCId = cookieContainerRequest.getAffinitizedContainer();
          boolean canMatchTaskWithAffinity = true;
          if (affCId == null || 
              !heldContainers.containsKey(affCId) ||
              inUseContainers.contains(affCId)) {
            // affinity not specified
            // affinitized container is no longer held
            // affinitized container is in use
            canMatchTaskWithAffinity = false;
          }
          if (canMatchTaskWithAffinity) {
            if (container.getId().equals(
                cookieContainerRequest.getAffinitizedContainer())) {
              // container level match
              if (LOG.isDebugEnabled()) {
                LOG.debug("Matching with affinity for request: "
                    + cookieContainerRequest + " container: " + affCId);
              }
              return cookieContainerRequest;
            }
            if (LOG.isDebugEnabled()) {
              LOG.debug("Skipping request for container " + container.getId()
                  + " due to affinity. Request: " + cookieContainerRequest
                  + " affContainer: " + affCId);
            }
          } else {
            firstMatch = cookieContainerRequest;
          }
        }
      }
    }
  }
  
  return firstMatch;
}