Java Code Examples for org.apache.hadoop.yarn.api.records.ResourceRequest#newInstance()

/**
 * This method produces an Allocation that includes the current view
 * of the resources that will be allocated to and preempted from this
 * application.
 *
 * @param rc
 * @param clusterResource
 * @param minimumAllocation
 * @return an allocation
 */
public synchronized Allocation getAllocation(ResourceCalculator rc,
    Resource clusterResource, Resource minimumAllocation) {

  Set<ContainerId> currentContPreemption = Collections.unmodifiableSet(
      new HashSet<ContainerId>(containersToPreempt));
  containersToPreempt.clear();
  Resource tot = Resource.newInstance(0, 0, 0);
  for(ContainerId c : currentContPreemption){
    Resources.addTo(tot,
        liveContainers.get(c).getContainer().getResource());
  }
  int numCont = (int) Math.ceil(
      Resources.divide(rc, clusterResource, tot, minimumAllocation));
  ResourceRequest rr = ResourceRequest.newInstance(
      Priority.UNDEFINED, ResourceRequest.ANY,
      minimumAllocation, numCont);
  ContainersAndNMTokensAllocation allocation =
      pullNewlyAllocatedContainersAndNMTokens();
  Resource headroom = getHeadroom();
  setApplicationHeadroomForMetrics(headroom);
  return new Allocation(allocation.getContainerList(), headroom, null,
    currentContPreemption, Collections.singletonList(rr),
    allocation.getNMTokenList());
}
 

@Override
public synchronized Allocation allocate(
    ApplicationAttemptId applicationAttemptId, List<ResourceRequest> ask,
    List<ContainerId> release, 
    List<String> blacklistAdditions, List<String> blacklistRemovals) {
  List<ResourceRequest> askCopy = new ArrayList<ResourceRequest>();
  for (ResourceRequest req : ask) {
    ResourceRequest reqCopy = ResourceRequest.newInstance(req
        .getPriority(), req.getResourceName(), req.getCapability(), req
        .getNumContainers(), req.getRelaxLocality());
    askCopy.add(reqCopy);
  }
  lastAsk = ask;
  lastRelease = release;
  lastBlacklistAdditions = blacklistAdditions;
  lastBlacklistRemovals = blacklistRemovals;
  return super.allocate(
      applicationAttemptId, askCopy, release, 
      blacklistAdditions, blacklistRemovals);
}
 

@Override
public synchronized Allocation allocate(
    ApplicationAttemptId applicationAttemptId, List<ResourceRequest> ask,
    List<ContainerId> release, List<String> blacklistAdditions,
    List<String> blacklistRemovals) {
  List<ResourceRequest> askCopy = new ArrayList<ResourceRequest>();
  for (ResourceRequest req : ask) {
    ResourceRequest reqCopy =
        ResourceRequest.newInstance(req.getPriority(),
            req.getResourceName(), req.getCapability(),
            req.getNumContainers(), req.getRelaxLocality());
    askCopy.add(reqCopy);
  }
  lastAsk = ask;
  lastRelease = release;
  lastBlacklistAdditions = blacklistAdditions;
  lastBlacklistRemovals = blacklistRemovals;
  return super.allocate(applicationAttemptId, askCopy, release,
      blacklistAdditions, blacklistRemovals);
}
 

/**
 * This method produces an Allocation that includes the current view
 * of the resources that will be allocated to and preempted from this
 * application.
 *
 * @param rc
 * @param clusterResource
 * @param minimumAllocation
 * @return an allocation
 */
public synchronized Allocation getAllocation(ResourceCalculator rc,
    Resource clusterResource, Resource minimumAllocation) {

  Set<ContainerId> currentContPreemption = Collections.unmodifiableSet(
      new HashSet<ContainerId>(containersToPreempt));
  containersToPreempt.clear();
  Resource tot = Resource.newInstance(0, 0);
  for(ContainerId c : currentContPreemption){
    Resources.addTo(tot,
        liveContainers.get(c).getContainer().getResource());
  }
  int numCont = (int) Math.ceil(
      Resources.divide(rc, clusterResource, tot, minimumAllocation));
  ResourceRequest rr = ResourceRequest.newInstance(
      Priority.UNDEFINED, ResourceRequest.ANY,
      minimumAllocation, numCont);
  ContainersAndNMTokensAllocation allocation =
      pullNewlyAllocatedContainersAndNMTokens();
  Resource headroom = getHeadroom();
  setApplicationHeadroomForMetrics(headroom);
  return new Allocation(allocation.getContainerList(), headroom, null,
    currentContPreemption, Collections.singletonList(rr),
    allocation.getNMTokenList());
}
 

@Override
public synchronized Allocation allocate(
    ApplicationAttemptId applicationAttemptId, List<ResourceRequest> ask,
    List<ContainerId> release, 
    List<String> blacklistAdditions, List<String> blacklistRemovals) {
  List<ResourceRequest> askCopy = new ArrayList<ResourceRequest>();
  for (ResourceRequest req : ask) {
    ResourceRequest reqCopy = ResourceRequest.newInstance(req
        .getPriority(), req.getResourceName(), req.getCapability(), req
        .getNumContainers(), req.getRelaxLocality());
    askCopy.add(reqCopy);
  }
  SecurityUtil.setTokenServiceUseIp(false);
  lastAsk = ask;
  lastRelease = release;
  lastBlacklistAdditions = blacklistAdditions;
  lastBlacklistRemovals = blacklistRemovals;
  return super.allocate(
      applicationAttemptId, askCopy, release, 
      blacklistAdditions, blacklistRemovals);
}
 

@Override
public synchronized Allocation allocate(
    ApplicationAttemptId applicationAttemptId, List<ResourceRequest> ask,
    List<ContainerId> release, List<String> blacklistAdditions,
    List<String> blacklistRemovals) {
  List<ResourceRequest> askCopy = new ArrayList<ResourceRequest>();
  for (ResourceRequest req : ask) {
    ResourceRequest reqCopy =
        ResourceRequest.newInstance(req.getPriority(),
            req.getResourceName(), req.getCapability(),
            req.getNumContainers(), req.getRelaxLocality());
    askCopy.add(reqCopy);
  }
  lastAsk = ask;
  lastRelease = release;
  lastBlacklistAdditions = blacklistAdditions;
  lastBlacklistRemovals = blacklistRemovals;
  return super.allocate(applicationAttemptId, askCopy, release,
      blacklistAdditions, blacklistRemovals);
}
 

protected void setRequestLimit(Priority priority, Resource capability,
    int limit) {
  if (limit < 0) {
    limit = Integer.MAX_VALUE;
  }
  ResourceRequest newReqLimit = ResourceRequest.newInstance(priority,
      ResourceRequest.ANY, capability, limit);
  ResourceRequest oldReqLimit = requestLimits.put(newReqLimit, newReqLimit);
  if (oldReqLimit == null || oldReqLimit.getNumContainers() < limit) {
    requestLimitsToUpdate.add(newReqLimit);
  }
}
 

ResourceRequestInfo(Priority priority, String resourceName,
    Resource capability, boolean relaxLocality) {
  remoteRequest = ResourceRequest.newInstance(priority, resourceName,
      capability, 0);
  remoteRequest.setRelaxLocality(relaxLocality);
  containerRequests = new LinkedHashSet<T>();
}
 

ResourceRequestInfo(Priority priority, String resourceName,
    Resource capability, boolean relaxLocality) {
  remoteRequest = ResourceRequest.newInstance(priority, resourceName,
      capability, 0);
  remoteRequest.setRelaxLocality(relaxLocality);
  containerRequests = new LinkedHashSet<T>();
}
 

protected void setRequestLimit(Priority priority, Resource capability,
    int limit) {
  if (limit < 0) {
    limit = Integer.MAX_VALUE;
  }
  ResourceRequest newReqLimit = ResourceRequest.newInstance(priority,
      ResourceRequest.ANY, capability, limit);
  ResourceRequest oldReqLimit = requestLimits.put(newReqLimit, newReqLimit);
  if (oldReqLimit == null || oldReqLimit.getNumContainers() < limit) {
    requestLimitsToUpdate.add(newReqLimit);
  }
}
 

/**
 * Simple test Resource request.
 * Test hashCode, equals and compare.
 */
@Test
public void testResourceRequest() {

  Resource resource = recordFactory.newRecordInstance(Resource.class);
  Priority priority = recordFactory.newRecordInstance(Priority.class);

  ResourceRequest original = ResourceRequest.newInstance(priority, "localhost", resource, 2) ;

  ResourceRequest copy = ResourceRequest.newInstance(priority, "localhost", resource, 2);

  assertTrue(original.equals(copy));
  assertEquals(0, original.compareTo(copy));
  assertTrue(original.hashCode() == copy.hashCode());

  copy.setNumContainers(1);

  assertFalse(original.equals(copy));
  assertNotSame(0, original.compareTo(copy));
  assertFalse(original.hashCode() == copy.hashCode());

}
 

protected void containerFailedOnHost(String hostName) {
  if (!nodeBlacklistingEnabled) {
    return;
  }
  if (blacklistedNodes.contains(hostName)) {
    if (LOG.isDebugEnabled()) {
      LOG.debug("Host " + hostName + " is already blacklisted.");
    }
    return; //already blacklisted
  }
  Integer failures = nodeFailures.remove(hostName);
  failures = failures == null ? Integer.valueOf(0) : failures;
  failures++;
  LOG.info(failures + " failures on node " + hostName);
  if (failures >= maxTaskFailuresPerNode) {
    blacklistedNodes.add(hostName);
    if (!ignoreBlacklisting.get()) {
      blacklistAdditions.add(hostName);
    }
    //Even if blacklisting is ignored, continue to remove the host from
    // the request table. The RM may have additional nodes it can allocate on.
    LOG.info("Blacklisted host " + hostName);

    //remove all the requests corresponding to this hostname
    for (Map<String, Map<Resource, ResourceRequest>> remoteRequests 
        : remoteRequestsTable.values()){
      //remove from host if no pending allocations
      boolean foundAll = true;
      Map<Resource, ResourceRequest> reqMap = remoteRequests.get(hostName);
      if (reqMap != null) {
        for (ResourceRequest req : reqMap.values()) {
          if (!ask.remove(req)) {
            foundAll = false;
            // if ask already sent to RM, we can try and overwrite it if possible.
            // send a new ask to RM with numContainers
            // specified for the blacklisted host to be 0.
            ResourceRequest zeroedRequest =
                ResourceRequest.newInstance(req.getPriority(),
                  req.getResourceName(), req.getCapability(),
                  req.getNumContainers(), req.getRelaxLocality());

            zeroedRequest.setNumContainers(0);
            // to be sent to RM on next heartbeat
            addResourceRequestToAsk(zeroedRequest);
          }
        }
        // if all requests were still in ask queue
        // we can remove this request
        if (foundAll) {
          remoteRequests.remove(hostName);
        }
      }
      // TODO handling of rack blacklisting
      // Removing from rack should be dependent on no. of failures within the rack 
      // Blacklisting a rack on the basis of a single node's blacklisting 
      // may be overly aggressive. 
      // Node failures could be co-related with other failures on the same rack 
      // but we probably need a better approach at trying to decide how and when 
      // to blacklist a rack
    }
  } else {
    nodeFailures.put(hostName, failures);
  }
}
 

public ResourceRequest cloneResourceRequest(ResourceRequest request) {
  ResourceRequest newRequest = ResourceRequest.newInstance(
      request.getPriority(), request.getResourceName(),
      request.getCapability(), 1, request.getRelaxLocality());
  return newRequest;
}
 

@Test
public void testMove() {
  final String user = "user1";
  Queue parentQueue = createQueue("parent", null);
  Queue oldQueue = createQueue("old", parentQueue);
  Queue newQueue = createQueue("new", parentQueue);
  QueueMetrics parentMetrics = parentQueue.getMetrics();
  QueueMetrics oldMetrics = oldQueue.getMetrics();
  QueueMetrics newMetrics = newQueue.getMetrics();

  ApplicationAttemptId appAttId = createAppAttemptId(0, 0);
  RMContext rmContext = mock(RMContext.class);
  when(rmContext.getEpoch()).thenReturn(3L);
  SchedulerApplicationAttempt app = new SchedulerApplicationAttempt(appAttId,
      user, oldQueue, oldQueue.getActiveUsersManager(), rmContext);
  oldMetrics.submitApp(user);
  
  // confirm that containerId is calculated based on epoch.
  assertEquals(0x30000000001L, app.getNewContainerId());
  
  // Resource request
  Resource requestedResource = Resource.newInstance(1536, 2, 2);
  Priority requestedPriority = Priority.newInstance(2);
  ResourceRequest request = ResourceRequest.newInstance(requestedPriority,
      ResourceRequest.ANY, requestedResource, 3);
  app.updateResourceRequests(Arrays.asList(request));

  // Allocated container
  RMContainer container1 = createRMContainer(appAttId, 1, requestedResource);
  app.liveContainers.put(container1.getContainerId(), container1);
  SchedulerNode node = createNode();
  app.appSchedulingInfo.allocate(NodeType.OFF_SWITCH, node, requestedPriority,
      request, container1.getContainer());
  
  // Reserved container
  Priority prio1 = Priority.newInstance(1);
  Resource reservedResource = Resource.newInstance(2048, 3, 3);
  RMContainer container2 = createReservedRMContainer(appAttId, 1, reservedResource,
      node.getNodeID(), prio1);
  Map<NodeId, RMContainer> reservations = new HashMap<NodeId, RMContainer>();
  reservations.put(node.getNodeID(), container2);
  app.reservedContainers.put(prio1, reservations);
  oldMetrics.reserveResource(user, reservedResource);
  
  checkQueueMetrics(oldMetrics, 1, 1, 1536, 2, 2, 2048, 3, 3, 3072, 4, 4);
  checkQueueMetrics(newMetrics, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0);
  checkQueueMetrics(parentMetrics, 1, 1, 1536, 2, 2, 2048, 3, 3, 3072, 4, 4);
  
  app.move(newQueue);
  
  checkQueueMetrics(oldMetrics, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0);
  checkQueueMetrics(newMetrics, 1, 1, 1536, 2, 2, 2048, 3, 3, 3072, 4, 4);
  checkQueueMetrics(parentMetrics, 1, 1, 1536, 2, 2, 2048, 3, 3, 3072, 4, 4);
}
 

@SuppressWarnings("unchecked")
@Test
public void testScheduleTransitionReplaceAMContainerRequestWithDefaults() {
  YarnScheduler mockScheduler = mock(YarnScheduler.class);
  when(
      mockScheduler.allocate(any(ApplicationAttemptId.class),
          any(List.class), any(List.class), any(List.class), any(List.class)))
      .thenAnswer(new Answer<Allocation>() {

        @SuppressWarnings("rawtypes")
        @Override
        public Allocation answer(InvocationOnMock invocation)
            throws Throwable {
          ResourceRequest rr =
              (ResourceRequest) ((List) invocation.getArguments()[1]).get(0);
          
          // capacity shouldn't changed
          assertEquals(Resource.newInstance(3333, 1), rr.getCapability());
          assertEquals("label-expression", rr.getNodeLabelExpression());
          
          // priority, #container, relax-locality will be changed
          assertEquals(RMAppAttemptImpl.AM_CONTAINER_PRIORITY, rr.getPriority());
          assertEquals(1, rr.getNumContainers());
          assertEquals(ResourceRequest.ANY, rr.getResourceName());

          // just return an empty allocation
          List l = new ArrayList();
          Set s = new HashSet();
          return new Allocation(l, Resources.none(), s, s, l);
        }
      });
  
  // create an attempt.
  applicationAttempt =
      new RMAppAttemptImpl(applicationAttempt.getAppAttemptId(),
          spyRMContext, scheduler, masterService, submissionContext,
          new Configuration(), true, ResourceRequest.newInstance(
              Priority.UNDEFINED, "host1", Resource.newInstance(3333, 1), 3,
              false, "label-expression"));
  new RMAppAttemptImpl.ScheduleTransition().transition(
      (RMAppAttemptImpl) applicationAttempt, null);
}
 

@Test
public void testMove() {
  final String user = "user1";
  Queue parentQueue = createQueue("parent", null);
  Queue oldQueue = createQueue("old", parentQueue);
  Queue newQueue = createQueue("new", parentQueue);
  QueueMetrics parentMetrics = parentQueue.getMetrics();
  QueueMetrics oldMetrics = oldQueue.getMetrics();
  QueueMetrics newMetrics = newQueue.getMetrics();

  ApplicationAttemptId appAttId = createAppAttemptId(0, 0);
  RMContext rmContext = mock(RMContext.class);
  when(rmContext.getEpoch()).thenReturn(3L);
  SchedulerApplicationAttempt app = new SchedulerApplicationAttempt(appAttId,
      user, oldQueue, oldQueue.getActiveUsersManager(), rmContext);
  oldMetrics.submitApp(user);
  
  // confirm that containerId is calculated based on epoch.
  assertEquals(0x30000000001L, app.getNewContainerId());
  
  // Resource request
  Resource requestedResource = Resource.newInstance(1536, 2);
  Priority requestedPriority = Priority.newInstance(2);
  ResourceRequest request = ResourceRequest.newInstance(requestedPriority,
      ResourceRequest.ANY, requestedResource, 3);
  app.updateResourceRequests(Arrays.asList(request));

  // Allocated container
  RMContainer container1 = createRMContainer(appAttId, 1, requestedResource);
  app.liveContainers.put(container1.getContainerId(), container1);
  SchedulerNode node = createNode();
  app.appSchedulingInfo.allocate(NodeType.OFF_SWITCH, node, requestedPriority,
      request, container1.getContainer());
  
  // Reserved container
  Priority prio1 = Priority.newInstance(1);
  Resource reservedResource = Resource.newInstance(2048, 3);
  RMContainer container2 = createReservedRMContainer(appAttId, 1, reservedResource,
      node.getNodeID(), prio1);
  Map<NodeId, RMContainer> reservations = new HashMap<NodeId, RMContainer>();
  reservations.put(node.getNodeID(), container2);
  app.reservedContainers.put(prio1, reservations);
  oldMetrics.reserveResource(user, reservedResource);
  
  checkQueueMetrics(oldMetrics, 1, 1, 1536, 2, 2048, 3, 3072, 4);
  checkQueueMetrics(newMetrics, 0, 0, 0, 0, 0, 0, 0, 0);
  checkQueueMetrics(parentMetrics, 1, 1, 1536, 2, 2048, 3, 3072, 4);
  
  app.move(newQueue);
  
  checkQueueMetrics(oldMetrics, 0, 0, 0, 0, 0, 0, 0, 0);
  checkQueueMetrics(newMetrics, 1, 1, 1536, 2, 2048, 3, 3072, 4);
  checkQueueMetrics(parentMetrics, 1, 1, 1536, 2, 2048, 3, 3072, 4);
}
 

@SuppressWarnings("unchecked")
@Test
public void testScheduleTransitionReplaceAMContainerRequestWithDefaults() {
  YarnScheduler mockScheduler = mock(YarnScheduler.class);
  when(
      mockScheduler.allocate(any(ApplicationAttemptId.class),
          any(List.class), any(List.class), any(List.class), any(List.class)))
      .thenAnswer(new Answer<Allocation>() {

        @SuppressWarnings("rawtypes")
        @Override
        public Allocation answer(InvocationOnMock invocation)
            throws Throwable {
          ResourceRequest rr =
              (ResourceRequest) ((List) invocation.getArguments()[1]).get(0);
          
          // capacity shouldn't changed
          assertEquals(Resource.newInstance(3333, 1, 1), rr.getCapability());
          assertEquals("label-expression", rr.getNodeLabelExpression());
          
          // priority, #container, relax-locality will be changed
          assertEquals(RMAppAttemptImpl.AM_CONTAINER_PRIORITY, rr.getPriority());
          assertEquals(1, rr.getNumContainers());
          assertEquals(ResourceRequest.ANY, rr.getResourceName());

          // just return an empty allocation
          List l = new ArrayList();
          Set s = new HashSet();
          return new Allocation(l, Resources.none(), s, s, l);
        }
      });
  
  // create an attempt.
  applicationAttempt =
      new RMAppAttemptImpl(applicationAttempt.getAppAttemptId(),
          spyRMContext, scheduler, masterService, submissionContext,
          new Configuration(), true, ResourceRequest.newInstance(
              Priority.UNDEFINED, "host1", Resource.newInstance(3333, 1, 1), 3,
              false, "label-expression"));
  new RMAppAttemptImpl.ScheduleTransition().transition(
      (RMAppAttemptImpl) applicationAttempt, null);
}
 

/**
 * Simple test Resource request.
 * Test hashCode, equals and compare.
 */
@Test
public void testResourceRequest() {

  Resource resource = recordFactory.newRecordInstance(Resource.class);
  Priority priority = recordFactory.newRecordInstance(Priority.class);

  ResourceRequest original = ResourceRequest.newInstance(priority, "localhost", resource, 2) ;

  ResourceRequest copy = ResourceRequest.newInstance(priority, "localhost", resource, 2);

  assertTrue(original.equals(copy));
  assertEquals(0, original.compareTo(copy));
  assertTrue(original.hashCode() == copy.hashCode());

  copy.setNumContainers(1);

  assertFalse(original.equals(copy));
  assertNotSame(0, original.compareTo(copy));
  assertFalse(original.hashCode() == copy.hashCode());

}
 

protected void containerFailedOnHost(String hostName) {
  if (!nodeBlacklistingEnabled) {
    return;
  }
  if (blacklistedNodes.contains(hostName)) {
    if (LOG.isDebugEnabled()) {
      LOG.debug("Host " + hostName + " is already blacklisted.");
    }
    return; //already blacklisted
  }
  Integer failures = nodeFailures.remove(hostName);
  failures = failures == null ? Integer.valueOf(0) : failures;
  failures++;
  LOG.info(failures + " failures on node " + hostName);
  if (failures >= maxTaskFailuresPerNode) {
    blacklistedNodes.add(hostName);
    if (!ignoreBlacklisting.get()) {
      blacklistAdditions.add(hostName);
    }
    //Even if blacklisting is ignored, continue to remove the host from
    // the request table. The RM may have additional nodes it can allocate on.
    LOG.info("Blacklisted host " + hostName);

    //remove all the requests corresponding to this hostname
    for (Map<String, Map<Resource, ResourceRequest>> remoteRequests 
        : remoteRequestsTable.values()){
      //remove from host if no pending allocations
      boolean foundAll = true;
      Map<Resource, ResourceRequest> reqMap = remoteRequests.get(hostName);
      if (reqMap != null) {
        for (ResourceRequest req : reqMap.values()) {
          if (!ask.remove(req)) {
            foundAll = false;
            // if ask already sent to RM, we can try and overwrite it if possible.
            // send a new ask to RM with numContainers
            // specified for the blacklisted host to be 0.
            ResourceRequest zeroedRequest =
                ResourceRequest.newInstance(req.getPriority(),
                  req.getResourceName(), req.getCapability(),
                  req.getNumContainers(), req.getRelaxLocality());

            zeroedRequest.setNumContainers(0);
            // to be sent to RM on next heartbeat
            addResourceRequestToAsk(zeroedRequest);
          }
        }
        // if all requests were still in ask queue
        // we can remove this request
        if (foundAll) {
          remoteRequests.remove(hostName);
        }
      }
      // TODO handling of rack blacklisting
      // Removing from rack should be dependent on no. of failures within the rack 
      // Blacklisting a rack on the basis of a single node's blacklisting 
      // may be overly aggressive. 
      // Node failures could be co-related with other failures on the same rack 
      // but we probably need a better approach at trying to decide how and when 
      // to blacklist a rack
    }
  } else {
    nodeFailures.put(hostName, failures);
  }
}
 

public ResourceRequest cloneResourceRequest(ResourceRequest request) {
  ResourceRequest newRequest = ResourceRequest.newInstance(
      request.getPriority(), request.getResourceName(),
      request.getCapability(), 1, request.getRelaxLocality());
  return newRequest;
}