org.apache.hadoop.yarn.api.records.Priority Java Examples

@Test
public void testFillInRacks() {
  AMRMClientImpl<ContainerRequest> client =
      new AMRMClientImpl<ContainerRequest>();
  
  Configuration conf = new Configuration();
  conf.setClass(
      CommonConfigurationKeysPublic.NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY,
      MyResolver.class, DNSToSwitchMapping.class);
  client.init(conf);
 
  Resource capability = Resource.newInstance(1024, 1, 1);
  ContainerRequest request =
      new ContainerRequest(capability, new String[] {"host1", "host2"},
          new String[] {"/rack2"}, Priority.newInstance(1));
  client.addContainerRequest(request);
  verifyResourceRequest(client, request, "host1", true);
  verifyResourceRequest(client, request, "host2", true);
  verifyResourceRequest(client, request, "/rack1", true);
  verifyResourceRequest(client, request, "/rack2", true);
  verifyResourceRequest(client, request, ResourceRequest.ANY, true);
}
 

private CookieContainerRequest getMatchingRequestWithPriority(
    Container container,
    String location) {
  Priority priority = container.getPriority();
  Resource capability = container.getResource();
  List<? extends Collection<CookieContainerRequest>> requestsList =
      amRmClient.getMatchingRequests(priority, location, capability);

  if (!requestsList.isEmpty()) {
    // pick first one
    for (Collection<CookieContainerRequest> requests : requestsList) {
      for (CookieContainerRequest cookieContainerRequest : requests) {
        if (canAssignTaskToContainer(cookieContainerRequest, container)) {
          return cookieContainerRequest;
        }
      }
    }
  }

  return null;
}
 

@Override
public synchronized void allocateTask(
    Object task,
    Resource capability,
    String[] hosts,
    String[] racks,
    Priority priority,
    Object containerSignature,
    Object clientCookie) {

  // XXX Have ContainerContext implement an interface defined by TaskScheduler.
  // TODO check for nulls etc
  // TODO extra memory allocation
  CRCookie cookie = new CRCookie(task, clientCookie, containerSignature);
  CookieContainerRequest request = new CookieContainerRequest(
    capability, hosts, racks, priority, cookie);

  addRequestAndTrigger(task, request, hosts, racks);
}
 

/**
 * Setup the request(s) that will be sent to the RM for the container ask.
 */
public ContainerRequest createContainerRequest(ContainerStartRequest csr, boolean first)
{
  int priority = csr.container.getResourceRequestPriority();
  // check for node locality constraint
  String[] nodes = null;
  String[] racks = null;

  String host = getHost(csr, first);
  Resource capability = Records.newRecord(Resource.class);
  capability.setMemory(csr.container.getRequiredMemoryMB());
  capability.setVirtualCores(csr.container.getRequiredVCores());
  if (host == INVALID_HOST) {
    return null;
  }
  if (host != null) {
    nodes = new String[]{host};
    // in order to request a host, we don't have to set the rack if the locality is false
    /*
     * if(this.nodeToRack.get(host) != null){ racks = new String[] { this.nodeToRack.get(host) }; }
     */
    return new ContainerRequest(capability, nodes, racks, Priority.newInstance(priority), false);
  }
  // For now, only memory is supported so we set memory requirements
  return new ContainerRequest(capability, nodes, racks, Priority.newInstance(priority));
}
 

private synchronized CSAssignment assignReservedContainer(
    FiCaSchedulerApp application, FiCaSchedulerNode node,
    RMContainer rmContainer, Resource clusterResource) {
  // Do we still need this reservation?
  Priority priority = rmContainer.getReservedPriority();
  if (application.getTotalRequiredResources(priority) == 0) {
    // Release
    return new CSAssignment(application, rmContainer);
  }

  // Try to assign if we have sufficient resources
  assignContainersOnNode(clusterResource, node, application, priority, 
      rmContainer, new ResourceLimits(Resources.none()));
  
  // Doesn't matter... since it's already charged for at time of reservation
  // "re-reservation" is *free*
  return new CSAssignment(Resources.none(), NodeType.NODE_LOCAL);
}
 

/**
 * Removes a vertex from a RequestPriorityStats.
 *
 * Removing a vertex is more expensive than adding a vertex. The stats contain bitmasks which only store on/off
 * values rather than reference counts. Therefore we must rebuild the descendants bitmasks from the remaining
 * vertices in the request stats. Once the new descendants mask is computed we then need to rebuild the
 * allowedVertices BitSet for all lower priority request stats in case the removal of this vertex unblocks lower
 * priority requests of a descendant vertex.
 *
 * Rebuilding allowedVertices for the lower priorities involves starting with the allowedVertices mask at the
 * current priority then masking off the descendants at each priority level encountered, accumulating the results.
 * Any descendants of a level will be blocked at all lower levels. See the addVertexToRequestStats documentation
 * for details on how vertices map to the descendants and allowedVertices bit masks.
 */
private void removeVertexFromRequestStats(Priority priority, RequestPriorityStats stats, Integer vertexIndexInt) {
  stats.vertexTaskCount.remove(vertexIndexInt);
  int vertexIndex = vertexIndexInt;
  stats.vertices.clear(vertexIndex);

  // Rebuild the descendants BitSet for the remaining vertices at this priority.
  stats.descendants.clear();
  for (Integer vIndex : stats.vertexTaskCount.keySet()) {
    stats.descendants.or(vertexDescendants.get(vIndex));
  }

  // The allowedVertices for all lower priorities need to be recalculated where the vertex descendants at each
  // level are removed from the list of allowed vertices at all subsequent levels.
  Collection<RequestPriorityStats> tailStats = priorityStats.tailMap(priority, false).values();
  if (!tailStats.isEmpty()) {
    BitSet cumulativeAllowed = new BitSet(vertexDescendants.size());
    cumulativeAllowed.or(stats.allowedVertices);
    cumulativeAllowed.andNot(stats.descendants);
    for (RequestPriorityStats s : tailStats) {
      s.allowedVertices.clear();
      s.allowedVertices.or(cumulativeAllowed);
      cumulativeAllowed.andNot(s.descendants);
    }
  }
}
 

/**
 * Instantiates a {@link ContainerRequest} with the given constraints.
 * 
 * @param capability
 *          The {@link Resource} to be requested for each container.
 * @param nodes
 *          Any hosts to request that the containers are placed on.
 * @param racks
 *          Any racks to request that the containers are placed on. The
 *          racks corresponding to any hosts requested will be automatically
 *          added to this list.
 * @param priority
 *          The priority at which to request the containers. Higher
 *          priorities have lower numerical values.
 * @param relaxLocality
 *          If true, containers for this request may be assigned on hosts
 *          and racks other than the ones explicitly requested.
 * @param nodeLabelsExpression
 *          Set node labels to allocate resource, now we only support
 *          asking for only a single node label
 */
public ContainerRequest(Resource capability, String[] nodes,
    String[] racks, Priority priority, boolean relaxLocality,
    String nodeLabelsExpression) {
  // Validate request
  Preconditions.checkArgument(capability != null,
      "The Resource to be requested for each container " +
          "should not be null ");
  Preconditions.checkArgument(priority != null,
      "The priority at which to request containers should not be null ");
  Preconditions.checkArgument(
          !(!relaxLocality && (racks == null || racks.length == 0) 
              && (nodes == null || nodes.length == 0)),
          "Can't turn off locality relaxation on a " + 
          "request with no location constraints");
  this.capability = capability;
  this.nodes = (nodes != null ? ImmutableList.copyOf(nodes) : null);
  this.racks = (racks != null ? ImmutableList.copyOf(racks) : null);
  this.priority = priority;
  this.relaxLocality = relaxLocality;
  this.nodeLabelsExpression = nodeLabelsExpression;
}
 

private void requestContainer(Optional<String> preferredNode) {
  Priority priority = Records.newRecord(Priority.class);
  priority.setPriority(0);

  Resource capability = Records.newRecord(Resource.class);
  int maxMemoryCapacity = this.maxResourceCapacity.get().getMemory();
  capability.setMemory(this.requestedContainerMemoryMbs <= maxMemoryCapacity ?
      this.requestedContainerMemoryMbs : maxMemoryCapacity);
  int maxCoreCapacity = this.maxResourceCapacity.get().getVirtualCores();
  capability.setVirtualCores(this.requestedContainerCores <= maxCoreCapacity ?
      this.requestedContainerCores : maxCoreCapacity);

  String[] preferredNodes = preferredNode.isPresent() ? new String[] {preferredNode.get()} : null;
  this.amrmClientAsync.addContainerRequest(
      new AMRMClient.ContainerRequest(capability, preferredNodes, null, priority));
}
 

@Override
protected RMContainerTokenSecretManager createContainerTokenSecretManager(
    Configuration conf) {
  return new RMContainerTokenSecretManager(conf) {

    @Override
    public Token createContainerToken(ContainerId containerId,
        NodeId nodeId, String appSubmitter, Resource capability,
        Priority priority, long createTime,
        LogAggregationContext logAggregationContext) {
      numRetries++;
      return super.createContainerToken(containerId, nodeId, appSubmitter,
        capability, priority, createTime, logAggregationContext);
    }
  };
}
 

private int assignRackLocalContainers(FiCaSchedulerNode node, 
    FiCaSchedulerApp application, Priority priority) {
  int assignedContainers = 0;
  ResourceRequest request = 
    application.getResourceRequest(priority, node.getRMNode().getRackName());
  if (request != null) {
    // Don't allocate on this rack if the application doens't need containers
    ResourceRequest offSwitchRequest =
        application.getResourceRequest(priority, ResourceRequest.ANY);
    if (offSwitchRequest.getNumContainers() <= 0) {
      return 0;
    }
    
    int assignableContainers = 
      Math.min(
          getMaxAllocatableContainers(application, priority, node, 
              NodeType.RACK_LOCAL), 
              request.getNumContainers());
    assignedContainers = 
      assignContainer(node, application, priority, 
          assignableContainers, request, NodeType.RACK_LOCAL);
  }
  return assignedContainers;
}
 

/**
 * This method produces an Allocation that includes the current view
 * of the resources that will be allocated to and preempted from this
 * application.
 *
 * @param rc
 * @param clusterResource
 * @param minimumAllocation
 * @return an allocation
 */
public synchronized Allocation getAllocation(ResourceCalculator rc,
    Resource clusterResource, Resource minimumAllocation) {

  Set<ContainerId> currentContPreemption = Collections.unmodifiableSet(
      new HashSet<ContainerId>(containersToPreempt));
  containersToPreempt.clear();
  Resource tot = Resource.newInstance(0, 0, 0);
  for(ContainerId c : currentContPreemption){
    Resources.addTo(tot,
        liveContainers.get(c).getContainer().getResource());
  }
  int numCont = (int) Math.ceil(
      Resources.divide(rc, clusterResource, tot, minimumAllocation));
  ResourceRequest rr = ResourceRequest.newInstance(
      Priority.UNDEFINED, ResourceRequest.ANY,
      minimumAllocation, numCont);
  ContainersAndNMTokensAllocation allocation =
      pullNewlyAllocatedContainersAndNMTokens();
  Resource headroom = getHeadroom();
  setApplicationHeadroomForMetrics(headroom);
  return new Allocation(allocation.getContainerList(), headroom, null,
    currentContPreemption, Collections.singletonList(rr),
    allocation.getNMTokenList());
}
 

@Public
@Unstable
public static ContainerHistoryData newInstance(ContainerId containerId,
    Resource allocatedResource, NodeId assignedNode, Priority priority,
    long startTime, long finishTime, String diagnosticsInfo,
    int containerExitCode, ContainerState containerState) {
  ContainerHistoryData containerHD = new ContainerHistoryData();
  containerHD.setContainerId(containerId);
  containerHD.setAllocatedResource(allocatedResource);
  containerHD.setAssignedNode(assignedNode);
  containerHD.setPriority(priority);
  containerHD.setStartTime(startTime);
  containerHD.setFinishTime(finishTime);
  containerHD.setDiagnosticsInfo(diagnosticsInfo);
  containerHD.setContainerExitStatus(containerExitCode);
  containerHD.setContainerState(containerState);
  return containerHD;
}
 

/**
 * This method produces an Allocation that includes the current view
 * of the resources that will be allocated to and preempted from this
 * application.
 *
 * @param rc
 * @param clusterResource
 * @param minimumAllocation
 * @return an allocation
 */
public synchronized Allocation getAllocation(ResourceCalculator rc,
    Resource clusterResource, Resource minimumAllocation) {

  Set<ContainerId> currentContPreemption = Collections.unmodifiableSet(
      new HashSet<ContainerId>(containersToPreempt));
  containersToPreempt.clear();
  Resource tot = Resource.newInstance(0, 0);
  for(ContainerId c : currentContPreemption){
    Resources.addTo(tot,
        liveContainers.get(c).getContainer().getResource());
  }
  int numCont = (int) Math.ceil(
      Resources.divide(rc, clusterResource, tot, minimumAllocation));
  ResourceRequest rr = ResourceRequest.newInstance(
      Priority.UNDEFINED, ResourceRequest.ANY,
      minimumAllocation, numCont);
  ContainersAndNMTokensAllocation allocation =
      pullNewlyAllocatedContainersAndNMTokens();
  Resource headroom = getHeadroom();
  setApplicationHeadroomForMetrics(headroom);
  return new Allocation(allocation.getContainerList(), headroom, null,
    currentContPreemption, Collections.singletonList(rr),
    allocation.getNMTokenList());
}
 

public static NMContainerStatus createNMContainerStatus(
    ApplicationAttemptId appAttemptId, int id, ContainerState containerState,
    int memory) {
  ContainerId containerId = ContainerId.newContainerId(appAttemptId, id);
  NMContainerStatus containerReport =
      NMContainerStatus.newInstance(containerId, containerState,
          Resource.newInstance(memory, 1), "recover container", 0,
          Priority.newInstance(0), 0);
  return containerReport;
}
 

public CookieContainerRequest(
    Resource capability,
    ContainerId containerId,
    String[] hosts,
    String[] racks,
    Priority priority,
    CRCookie cookie) {
  this(capability, hosts, racks, priority, cookie);
  this.affinitizedContainerId = containerId;
}
 

static LeafQueue stubLeafQueue(LeafQueue queue) {

    // Mock some methods for ease in these unit tests

    // 1. LeafQueue.createContainer to return dummy containers
    doAnswer(new Answer<Container>() {
      @Override
      public Container answer(InvocationOnMock invocation) throws Throwable {
        final FiCaSchedulerApp application = (FiCaSchedulerApp) (invocation
            .getArguments()[0]);
        final ContainerId containerId = TestUtils
            .getMockContainerId(application);

        Container container = TestUtils.getMockContainer(containerId,
            ((FiCaSchedulerNode) (invocation.getArguments()[1])).getNodeID(),
            (Resource) (invocation.getArguments()[2]),
            ((Priority) invocation.getArguments()[3]));
        return container;
      }
    }).when(queue).createContainer(any(FiCaSchedulerApp.class),
        any(FiCaSchedulerNode.class), any(Resource.class), any(Priority.class));

    // 2. Stub out LeafQueue.parent.completedContainer
    CSQueue parent = queue.getParent();
    doNothing().when(parent).completedContainer(any(Resource.class),
        any(FiCaSchedulerApp.class), any(FiCaSchedulerNode.class),
        any(RMContainer.class), any(ContainerStatus.class),
        any(RMContainerEventType.class), any(CSQueue.class), anyBoolean());

    return queue;
  }
 

static LeafQueue stubLeafQueue(LeafQueue queue) {
  
  // Mock some methods for ease in these unit tests
  
  // 1. LeafQueue.createContainer to return dummy containers
  doAnswer(
      new Answer<Container>() {
        @Override
        public Container answer(InvocationOnMock invocation) 
            throws Throwable {
          final FiCaSchedulerApp application = 
              (FiCaSchedulerApp)(invocation.getArguments()[0]);
          final ContainerId containerId =                 
              TestUtils.getMockContainerId(application);

          Container container = TestUtils.getMockContainer(
              containerId,
              ((FiCaSchedulerNode)(invocation.getArguments()[1])).getNodeID(), 
              (Resource)(invocation.getArguments()[2]),
              ((Priority)invocation.getArguments()[3]));
          return container;
        }
      }
    ).
    when(queue).createContainer(
            any(FiCaSchedulerApp.class), 
            any(FiCaSchedulerNode.class), 
            any(Resource.class),
            any(Priority.class)
            );
  
  // 2. Stub out LeafQueue.parent.completedContainer
  CSQueue parent = queue.getParent();
  doNothing().when(parent).completedContainer(
      any(Resource.class), any(FiCaSchedulerApp.class), any(FiCaSchedulerNode.class), 
      any(RMContainer.class), any(ContainerStatus.class), 
      any(RMContainerEventType.class), any(CSQueue.class), anyBoolean());
  
  return queue;
}
 

Container createContainer(FiCaSchedulerApp application, FiCaSchedulerNode node, 
    Resource capability, Priority priority) {

  NodeId nodeId = node.getRMNode().getNodeID();
  ContainerId containerId = BuilderUtils.newContainerId(application
      .getApplicationAttemptId(), application.getNewContainerId());

  // Create the container
  Container container =
      BuilderUtils.newContainer(containerId, nodeId, node.getRMNode()
        .getHttpAddress(), capability, priority, null);

  return container;
}
 

/**
 * Create and return a container object reflecting an allocation for the
 * given appliction on the given node with the given capability and
 * priority.
 */
public Container createContainer(
    FSSchedulerNode node, Resource capability, Priority priority) {

  NodeId nodeId = node.getRMNode().getNodeID();
  ContainerId containerId = BuilderUtils.newContainerId(
      getApplicationAttemptId(), getNewContainerId());

  // Create the container
  Container container =
      BuilderUtils.newContainer(containerId, nodeId, node.getRMNode()
          .getHttpAddress(), capability, priority, null);

  return container;
}
 

TestingContainer(
	final ContainerId containerId,
	final NodeId nodeId,
	final Resource resource,
	final Priority priority) {

	this.containerId = containerId;
	this.nodeId = nodeId;
	this.resource = resource;
	this.priority = priority;
}
 

private Container createContainer(int id, String host, Resource resource, Priority priority) {
  ContainerId containerID = ContainerId.newInstance(
      ApplicationAttemptId.newInstance(ApplicationId.newInstance(1, 1), 1),
      id);
  NodeId nodeID = NodeId.newInstance(host, 0);
  Container container = Container.newInstance(containerID, nodeID, host + ":0",
      resource, priority, null);
  return container;
}
 

public RMContainerReservedEvent(ContainerId containerId,
    Resource reservedResource, NodeId reservedNode, 
    Priority reservedPriority) {
  super(containerId, RMContainerEventType.RESERVED);
  this.reservedResource = reservedResource;
  this.reservedNode = reservedNode;
  this.reservedPriority = reservedPriority;
}
 

public static Container newContainer(ContainerId containerId, NodeId nodeId,
    String nodeHttpAddress, Resource resource, Priority priority,
    Token containerToken) {
  Container container = recordFactory.newRecordInstance(Container.class);
  container.setId(containerId);
  container.setNodeId(nodeId);
  container.setNodeHttpAddress(nodeHttpAddress);
  container.setResource(resource);
  container.setPriority(priority);
  container.setContainerToken(containerToken);
  return container;
}
 

public FSAppAttempt(FairScheduler scheduler,
    ApplicationAttemptId applicationAttemptId, String user, FSLeafQueue queue,
    ActiveUsersManager activeUsersManager, RMContext rmContext) {
  super(applicationAttemptId, user, queue, activeUsersManager, rmContext);

  this.scheduler = scheduler;
  this.startTime = scheduler.getClock().getTime();
  this.priority = Priority.newInstance(1);
  this.resourceWeights = new ResourceWeights();
}
 

public static Container newContainer(ContainerId containerId, NodeId nodeId,
    String nodeHttpAddress, Resource resource, Priority priority,
    Token containerToken) {
  Container container = recordFactory.newRecordInstance(Container.class);
  container.setId(containerId);
  container.setNodeId(nodeId);
  container.setNodeHttpAddress(nodeHttpAddress);
  container.setResource(resource);
  container.setPriority(priority);
  container.setContainerToken(containerToken);
  return container;
}
 

public FakeSchedulable(Resource minShare, Resource maxShare,
    ResourceWeights weight, Resource fairShare, Resource usage, long startTime) {
  this.minShare = minShare;
  this.maxShare = maxShare;
  this.weights = weight;
  setFairShare(fairShare);
  this.usage = usage;
  this.priority = Records.newRecord(Priority.class);
  this.startTime = startTime;
}
 

public Container createContainer(Resource capability, Priority priority) {
  ApplicationAttemptId appAttemptId = appContext.getApplicationAttemptId();
  ContainerId containerId = ContainerId.newInstance(appAttemptId, nextId.getAndIncrement());
  NodeId nodeId = NodeId.newInstance("127.0.0.1", 0);
  String nodeHttpAddress = "127.0.0.1:0";

  Container container = Container.newInstance(containerId,
      nodeId,
      nodeHttpAddress,
      capability,
      priority,
      null);

  return container;
}
 

@Test(timeout=10000)
public void testMatchingRequestsForTopPriority() {
  String[] hosts = { "host1" };
  String[] racks = { "rack1" };
  AMRMClient.ContainerRequest req1 = new AMRMClient.ContainerRequest(
    Resource.newInstance(2048, 1), hosts, racks,
    Priority.newInstance(1));
  AMRMClient.ContainerRequest req2 = new AMRMClient.ContainerRequest(
    Resource.newInstance(1024, 1), hosts, racks,
    Priority.newInstance(2));
  AMRMClient.ContainerRequest req3 = new AMRMClient.ContainerRequest(
    Resource.newInstance(1024, 1), hosts, racks,
    Priority.newInstance(3));
  amrmClient.addContainerRequest(req1);
  amrmClient.addContainerRequest(req2);
  amrmClient.addContainerRequest(req3);

  Assert.assertTrue(amrmClient.getMatchingRequestsForTopPriority("host1",
    Resource.newInstance(1024, 1)).isEmpty());

  List<? extends Collection<AMRMClient.ContainerRequest>> ret =
    amrmClient.getMatchingRequestsForTopPriority("host1",
      Resource.newInstance(2048, 1));
  Assert.assertFalse(ret.isEmpty());
  Assert.assertEquals(req1, ret.get(0).iterator().next());

  amrmClient.removeContainerRequest(req1);

  ret = amrmClient.getMatchingRequestsForTopPriority("host1",
      Resource.newInstance(1024, 1));
  Assert.assertFalse(ret.isEmpty());
  Assert.assertEquals(req2, ret.get(0).iterator().next());
}
 

@Test(timeout=30000)
public void testAskWithInvalidNodeLabels() {
  AMRMClientImpl<ContainerRequest> client =
      new AMRMClientImpl<ContainerRequest>();

  // specified exp with more than one node labels
  verifyAddRequestFailed(client,
      new ContainerRequest(Resource.newInstance(1024, 1), null, null,
          Priority.UNDEFINED, true, "x && y"));
}
 

@Test
public void testRegisterNodeManagerRequest() {
  ApplicationId appId = ApplicationId.newInstance(123456789, 1);
  ApplicationAttemptId attemptId = ApplicationAttemptId.newInstance(appId, 1);
  ContainerId containerId = ContainerId.newContainerId(attemptId, 1);

  NMContainerStatus containerReport =
      NMContainerStatus.newInstance(containerId,
        ContainerState.RUNNING, Resource.newInstance(1024, 1, 2), "diagnostics",
        0, Priority.newInstance(10), 1234);
  List<NMContainerStatus> reports = Arrays.asList(containerReport);
  RegisterNodeManagerRequest request =
      RegisterNodeManagerRequest.newInstance(
        NodeId.newInstance("1.1.1.1", 1000), 8080,
          Resource.newInstance(1024, 1, 2), "NM-version-id", reports,
          Arrays.asList(appId));
  RegisterNodeManagerRequest requestProto =
      new RegisterNodeManagerRequestPBImpl(
        ((RegisterNodeManagerRequestPBImpl) request).getProto());
  Assert.assertEquals(containerReport, requestProto
    .getNMContainerStatuses().get(0));
  Assert.assertEquals(8080, requestProto.getHttpPort());
  Assert.assertEquals("NM-version-id", requestProto.getNMVersion());
  Assert.assertEquals(NodeId.newInstance("1.1.1.1", 1000),
    requestProto.getNodeId());
  Assert.assertEquals(Resource.newInstance(1024, 1, 2),
    requestProto.getResource());
  Assert.assertEquals(1, requestProto.getRunningApplications().size());
  Assert.assertEquals(appId, requestProto.getRunningApplications().get(0)); 
}