org.apache.hadoop.yarn.api.records.Resource Java Examples

private synchronized CSAssignment assignReservedContainer(
    FiCaSchedulerApp application, FiCaSchedulerNode node,
    RMContainer rmContainer, Resource clusterResource) {
  // Do we still need this reservation?
  Priority priority = rmContainer.getReservedPriority();
  //优先级是否已经达到要求以分配reservation 的 containers
  if (application.getTotalRequiredResources(priority) == 0) {
    // Release
    return new CSAssignment(application, rmContainer);
  }

  // Try to assign if we have sufficient resources
  assignContainersOnNode(clusterResource, node, application, priority, 
      rmContainer, new ResourceLimits(Resources.none()));
  
  // Doesn't matter... since it's already charged for at time of reservation
  // "re-reservation" is *free*
  return new CSAssignment(Resources.none(), NodeType.NODE_LOCAL);
}
 

/**
 * Transforms a collection of mesos offers into {@link Resource}.
 *
 * @param offers collection of mesos offers
 * @return a single resource object equivalent to the cumulative sum of mesos offers
 */
public static Resource getYarnResourcesFromMesosOffers(Collection<Offer> offers) {
  double cpus = 0.0;
  double mem = 0.0;

  for (Protos.Offer offer : offers) {
    for (Protos.Resource resource : offer.getResourcesList()) {
      if (resource.getName().equalsIgnoreCase("cpus")) {
        cpus += resource.getScalar().getValue();
      } else if (resource.getName().equalsIgnoreCase("mem")) {
        mem += resource.getScalar().getValue();
      }
    }
  }
  return Resource.newInstance((int) mem, (int) cpus);
}
 

private Resource getTotalResource(List<ResourceRequest> requests) {
  Resource totalResource = Resource.newInstance(0, 0);
  if (requests == null) {
    return totalResource;
  }
  for (ResourceRequest request : requests) {
    if (request.getNumContainers() == 0) {
      continue;
    }
    if (request.getResourceName().equals(ResourceRequest.ANY)) {
      Resources.addTo(
        totalResource,
        Resources.multiply(request.getCapability(),
          request.getNumContainers()));
    }
  }
  return totalResource;
}
 

private void setupLimits(ContainerId containerId,
                         Resource containerResource) throws IOException {
  String containerName = containerId.toString();

  if (isCpuWeightEnabled()) {
    int containerVCores = containerResource.getVirtualCores();
    createCgroup(CONTROLLER_CPU, containerName);
    int cpuShares = CPU_DEFAULT_WEIGHT * containerVCores;
    updateCgroup(CONTROLLER_CPU, containerName, "shares",
        String.valueOf(cpuShares));
    if (strictResourceUsageMode) {
      int nodeVCores =
          conf.getInt(YarnConfiguration.NM_VCORES,
            YarnConfiguration.DEFAULT_NM_VCORES);
      if (nodeVCores != containerVCores) {
        float containerCPU =
            (containerVCores * yarnProcessors) / (float) nodeVCores;
        int[] limits = getOverallLimits(containerCPU);
        updateCgroup(CONTROLLER_CPU, containerName, CPU_PERIOD_US,
          String.valueOf(limits[0]));
        updateCgroup(CONTROLLER_CPU, containerName, CPU_QUOTA_US,
          String.valueOf(limits[1]));
      }
    }
  }
}
 

public void allocateResources(String user, int containers, Resource res,
    boolean decrPending) {
  allocatedContainers.incr(containers);
  aggregateContainersAllocated.incr(containers);
  allocatedMB.incr(res.getMemory() * containers);
  allocatedVCores.incr(res.getVirtualCores() * containers);
  allocatedGCores.incr(res.getGpuCores() * containers);
  if (decrPending) {
    _decrPendingResources(containers, res);
  }
  QueueMetrics userMetrics = getUserMetrics(user);
  if (userMetrics != null) {
    userMetrics.allocateResources(user, containers, res, decrPending);
  }
  if (parent != null) {
    parent.allocateResources(user, containers, res, decrPending);
  }
}
 

RMContainer mockContainer(ApplicationAttemptId appAttId, int id,
    Resource r, int cpriority) {
  ContainerId cId = ContainerId.newContainerId(appAttId, id);
  Container c = mock(Container.class);
  when(c.getResource()).thenReturn(r);
  when(c.getPriority()).thenReturn(Priority.create(cpriority));
  RMContainer mC = mock(RMContainer.class);
  when(mC.getContainerId()).thenReturn(cId);
  when(mC.getContainer()).thenReturn(c);
  when(mC.getApplicationAttemptId()).thenReturn(appAttId);
  if (priority.AMCONTAINER.getValue() == cpriority) {
    when(mC.isAMContainer()).thenReturn(true);
  }
  if (priority.LABELEDCONTAINER.getValue() == cpriority) {
    when(mC.getAllocatedNode()).thenReturn(NodeId.newInstance("node1", 0));
  }
  return mC;
}
 

/**
 * Gets the resource over usage.
 * This method returns any over usage of resources(vcores and memory) by yarn applications. Criterion for declaring a resource over used is
 * the threshold(memory and vcores threshold) provided by the user. <br>
 * It returns such application IDs along with their respective resource consumptions.
 * 
 * @param memoryThresholdMB the memory threshold mb
 * @param vcoresThreshold the vcores threshold
 * @param rmCommunicator 
 * @return the resource over usage
 */
public Map<String, Map<String, Object>> getResourceOverUsage(
		int memoryThresholdMB, int vcoresThreshold, RMCommunicator rmCommunicator) {
	Resource usedResources = null;
	Map<String, Map<String, Object>> appResourceReports = new HashMap<>(5);
	Map<String, Object> resourceMap = null;
	try {
		for (ApplicationReport report : rmCommunicator.getRunningApplications()) {
			usedResources = report.getApplicationResourceUsageReport().getUsedResources();
			if (usedResources.getMemory() > memoryThresholdMB || usedResources.getVirtualCores() > vcoresThreshold) {
				resourceMap = new HashMap<>(5);
				resourceMap.put(USED_MEMORY, usedResources.getMemory());
				resourceMap.put(USED_VCORES, usedResources.getVirtualCores());
				resourceMap.put(APP_TYPE, report.getApplicationType());
				resourceMap.put(QUEUE, report.getQueue());
				resourceMap.put(USER, report.getUser());
				appResourceReports.put(report.getApplicationId().toString(), resourceMap);
			}
			
		}
	} catch (YarnException | IOException e) {
		LOGGER.error("unable to get running containers", e.getMessage());
	}
	return appResourceReports;
}
 

@Test
public void testBlocks() {
  ReservationId reservationID =
      ReservationId.newInstance(rand.nextLong(), rand.nextLong());
  int[] alloc = { 10, 10, 10, 10, 10, 10 };
  int start = 100;
  ReservationDefinition rDef =
      createSimpleReservationDefinition(start, start + alloc.length + 1,
          alloc.length);
  Map<ReservationInterval, ReservationRequest> allocations =
      generateAllocation(start, alloc, false, false);
  ReservationAllocation rAllocation =
      new InMemoryReservationAllocation(reservationID, rDef, user, planName,
          start, start + alloc.length + 1, allocations, resCalc, minAlloc);
  doAssertions(rAllocation, reservationID, rDef, allocations, start, alloc);
  Assert.assertFalse(rAllocation.containsGangs());
  for (int i = 0; i < alloc.length; i++) {
    Assert.assertEquals(Resource.newInstance(1024 * (alloc[i]), (alloc[i]), (alloc[i])),
        rAllocation.getResourcesAtTime(start + i));
  }
}
 

/**
 * Setup the request(s) that will be sent to the RM for the container ask.
 */
public ContainerRequest createContainerRequest(ContainerStartRequest csr, boolean first)
{
  int priority = csr.container.getResourceRequestPriority();
  // check for node locality constraint
  String[] nodes = null;
  String[] racks = null;

  String host = getHost(csr, first);
  Resource capability = Records.newRecord(Resource.class);
  capability.setMemory(csr.container.getRequiredMemoryMB());
  capability.setVirtualCores(csr.container.getRequiredVCores());
  if (host == INVALID_HOST) {
    return null;
  }
  if (host != null) {
    nodes = new String[]{host};
    // in order to request a host, we don't have to set the rack if the locality is false
    /*
     * if(this.nodeToRack.get(host) != null){ racks = new String[] { this.nodeToRack.get(host) }; }
     */
    return new ContainerRequest(capability, nodes, racks, Priority.newInstance(priority), false);
  }
  // For now, only memory is supported so we set memory requirements
  return new ContainerRequest(capability, nodes, racks, Priority.newInstance(priority));
}
 

private FiCaSchedulerApp getMockApplication(int appId, String user,
  Resource amResource) {
  FiCaSchedulerApp application = mock(FiCaSchedulerApp.class);
  ApplicationAttemptId applicationAttemptId =
      TestUtils.getMockApplicationAttemptId(appId, 0);
  doReturn(applicationAttemptId.getApplicationId()).
      when(application).getApplicationId();
  doReturn(applicationAttemptId). when(application).getApplicationAttemptId();
  doReturn(user).when(application).getUser();
  doReturn(amResource).when(application).getAMResource();
  return application;
}
 

public static ResourceRequest createResourceRequest(
    String resourceName, int memory, int numContainers, boolean relaxLocality,
    Priority priority, RecordFactory recordFactory) {
  ResourceRequest request = 
      recordFactory.newRecordInstance(ResourceRequest.class);
  Resource capability = Resources.createResource(memory, 1);
  
  request.setNumContainers(numContainers);
  request.setResourceName(resourceName);
  request.setCapability(capability);
  request.setRelaxLocality(relaxLocality);
  request.setPriority(priority);
  return request;
}
 

@SuppressWarnings("unchecked")
@Override
public RegisterApplicationMasterResponse registerApplicationMaster(
    String appHostName, int appHostPort, String appTrackingUrl) {
  mockRegResponse = mock(RegisterApplicationMasterResponse.class);
  Resource mockMaxResource = mock(Resource.class);
  Map<ApplicationAccessType, String> mockAcls = mock(Map.class);
  when(mockRegResponse.getMaximumResourceCapability()).thenReturn(
      mockMaxResource);
  when(mockRegResponse.getApplicationACLs()).thenReturn(mockAcls);
  return mockRegResponse;
}
 

private Token createNewContainerToken(ContainerId contId,
    String containerManagerAddr) {
  long currentTime = System.currentTimeMillis();
  return MRApp.newContainerToken(NodeId.newInstance("127.0.0.1",
      1234), "password".getBytes(), new ContainerTokenIdentifier(
      contId, containerManagerAddr, "user",
      Resource.newInstance(1024, 1),
      currentTime + 10000L, 123, currentTime, Priority.newInstance(0), 0));
}
 

@Before
public void setup() {
  conf = new Configuration();
  taskAttemptListener = mock(TaskAttemptListener.class);
  taskHeartbeatHandler = mock(TaskHeartbeatHandler.class);
  credentials = new Credentials();
  clock = new SystemClock();
  locationHint = new TaskLocationHint(null, null);

  appId = ApplicationId.newInstance(System.currentTimeMillis(), 1);
  dagId = TezDAGID.getInstance(appId, 1);
  vertexId = TezVertexID.getInstance(dagId, 1);
  appContext = mock(AppContext.class, RETURNS_DEEP_STUBS);
  mockContainerId = mock(ContainerId.class);
  mockContainer = mock(Container.class);
  mockAMContainer = mock(AMContainer.class);
  mockNodeId = mock(NodeId.class);
  when(mockContainer.getId()).thenReturn(mockContainerId);
  when(mockContainer.getNodeId()).thenReturn(mockNodeId);
  when(mockAMContainer.getContainer()).thenReturn(mockContainer);
  when(appContext.getAllContainers().get(mockContainerId)).thenReturn(mockAMContainer);
  taskResource = Resource.newInstance(1024, 1);
  localResources = new HashMap<String, LocalResource>();
  environment = new HashMap<String, String>();
  javaOpts = "";
  leafVertex = false;
  containerContext = new ContainerContext(localResources, credentials,
      environment, javaOpts);
  Vertex vertex = mock(Vertex.class);
  eventHandler = new TestEventHandler();
  
  mockTask = new MockTaskImpl(vertexId, partition,
      eventHandler, conf, taskAttemptListener, clock,
      taskHeartbeatHandler, appContext, leafVertex, locationHint,
      taskResource, containerContext, vertex);
}
 

private Resource determinePSResource(Parameter parametersHolder,
    int nPS, ClientContext clientContext)
    throws ParseException, YarnException, IOException {
  if (nPS > 0) {
    String psResourceStr =
        parametersHolder.getOptionValue(CliConstants.PS_RES);
    if (psResourceStr == null) {
      throw new ParseException("--" + CliConstants.PS_RES + " is absent.");
    }
    return ResourceUtils.createResourceFromString(psResourceStr);
  }
  return null;
}
 

@Override
public void setMaximumResourceCapability(Resource capability) {
  maybeInitBuilder();
  if(maximumResourceCapability == null) {
    builder.clearMaximumCapability();
  }
  this.maximumResourceCapability = capability;
}
 

private Resource createCapability(ResourceSpecification resourceSpec) {
  Resource capability = Records.newRecord(Resource.class);

  if (!YarnUtils.setVirtualCores(capability, resourceSpec.getVirtualCores())) {
    LOG.debug("Virtual cores limit not supported.");
  }

  capability.setMemory(resourceSpec.getMemorySize());
  return capability;
}
 

public AllocateResponse createFakeAllocateResponse() {
  return AllocateResponse.newInstance(-1,
      new ArrayList<ContainerStatus>(),
      new ArrayList<Container>(), new ArrayList<NodeReport>(),
      Resource.newInstance(1024, 2, 2), null, 1,
      null, new ArrayList<NMToken>());
}
 

@Override
public void setUsed(Resource used) {
  maybeInitBuilder();
  if (used == null)
    builder.clearUsed();
  this.used = used;
}
 

@Test(timeout = 5000)
public void testLabelsToNodesOnNodeActiveDeactive() throws Exception {
  // Activate a node without assigning any labels
  mgr.activateNode(NodeId.newInstance("n1", 1), Resource.newInstance(10, 0));
  Assert.assertTrue(mgr.getLabelsToNodes().isEmpty());
  assertLabelsToNodesEquals(
      mgr.getLabelsToNodes(), transposeNodeToLabels(mgr.getNodeLabels()));

  // Add labels and replace labels on node
  mgr.addToCluserNodeLabels(toSet("p1"));
  mgr.replaceLabelsOnNode(ImmutableMap.of(toNodeId("n1"), toSet("p1")));
  // p1 -> n1, n1:1
  Assert.assertEquals(2, mgr.getLabelsToNodes().get("p1").size());
  assertLabelsToNodesEquals(
      mgr.getLabelsToNodes(), transposeNodeToLabels(mgr.getNodeLabels()));

  // Activate a node for which host to label mapping exists
  mgr.activateNode(NodeId.newInstance("n1", 2), Resource.newInstance(10, 0));
  // p1 -> n1, n1:1, n1:2
  Assert.assertEquals(3, mgr.getLabelsToNodes().get("p1").size());
  assertLabelsToNodesEquals(
      mgr.getLabelsToNodes(), transposeNodeToLabels(mgr.getNodeLabels()));

  // Deactivate a node. n1:1 will be removed from the map
  mgr.deactivateNode(NodeId.newInstance("n1", 1));
  // p1 -> n1, n1:2
  Assert.assertEquals(2, mgr.getLabelsToNodes().get("p1").size());
  assertLabelsToNodesEquals(
      mgr.getLabelsToNodes(), transposeNodeToLabels(mgr.getNodeLabels()));
}
 

private org.apache.hadoop.yarn.server.resourcemanager.NodeManager
    registerNode(String hostName, int containerManagerPort, int httpPort,
        String rackName, Resource capability)
        throws IOException, YarnException {
  org.apache.hadoop.yarn.server.resourcemanager.NodeManager nm =
      new org.apache.hadoop.yarn.server.resourcemanager.NodeManager(
          hostName, containerManagerPort, httpPort, rackName, capability,
          resourceManager);
  NodeAddedSchedulerEvent nodeAddEvent1 = 
      new NodeAddedSchedulerEvent(resourceManager.getRMContext()
          .getRMNodes().get(nm.getNodeId()));
  resourceManager.getResourceScheduler().handle(nodeAddEvent1);
  return nm;
}
 

private void setQueueResourceLimitsInfo(
    Resource clusterResource) {
  synchronized (queueResourceLimitsInfo) {
    queueResourceLimitsInfo.setQueueCurrentLimit(cachedResourceLimitsForHeadroom
        .getLimit());
    queueResourceLimitsInfo.setClusterResource(clusterResource);
  }
}
 

private ReservationDefinition createSimpleReservationDefinition(long arrival,
    long deadline, long duration) {
  // create a request with a single atomic ask
  ReservationRequest r =
      ReservationRequest.newInstance(Resource.newInstance(1024, 1, 1), 1, 1,
          duration);
  ReservationDefinition rDef = new ReservationDefinitionPBImpl();
  ReservationRequests reqs = new ReservationRequestsPBImpl();
  reqs.setReservationResources(Collections.singletonList(r));
  reqs.setInterpreter(ReservationRequestInterpreter.R_ALL);
  rDef.setReservationRequests(reqs);
  rDef.setArrival(arrival);
  rDef.setDeadline(deadline);
  return rDef;
}
 

@Override
public Resource getAllocatedResource() {
  if (this.resource != null) {
    return this.resource;
  }
  ContainerStartDataProtoOrBuilder p = viaProto ? proto : builder;
  if (!p.hasAllocatedResource()) {
    return null;
  }
  this.resource = convertFromProtoFormat(p.getAllocatedResource());
  return this.resource;
}
 

@Test
public void testMaxAMResourcePerQueuePercentAfterQueueRefresh()
    throws Exception {
  CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration();
  Resource clusterResource = Resources
      .createResource(100 * 16 * GB, 100 * 32);
  CapacitySchedulerContext csContext = mockCSContext(csConf, clusterResource);
  when(csContext.getRMContext()).thenReturn(rmContext);
  csConf.setFloat(CapacitySchedulerConfiguration.
      MAXIMUM_APPLICATION_MASTERS_RESOURCE_PERCENT, 0.1f);
  ParentQueue root = new ParentQueue(csContext, 
      CapacitySchedulerConfiguration.ROOT, null, null);
  csConf.setCapacity(CapacitySchedulerConfiguration.ROOT + "." + A, 80);
  LeafQueue a = new LeafQueue(csContext, A, root, null);
  assertEquals(0.1f, a.getMaxAMResourcePerQueuePercent(), 1e-3f);
  assertEquals(a.getAMResourceLimit(), Resources.createResource(160 * GB, 1));
  
  csConf.setFloat(CapacitySchedulerConfiguration.
      MAXIMUM_APPLICATION_MASTERS_RESOURCE_PERCENT, 0.2f);
  LeafQueue newA = new LeafQueue(csContext, A, root, null);
  a.reinitialize(newA, clusterResource);
  assertEquals(0.2f, a.getMaxAMResourcePerQueuePercent(), 1e-3f);
  assertEquals(a.getAMResourceLimit(), Resources.createResource(320 * GB, 1));

  Resource newClusterResource = Resources.createResource(100 * 20 * GB,
      100 * 32);
  a.updateClusterResource(newClusterResource, 
      new ResourceLimits(newClusterResource));
  //  100 * 20 * 0.2 = 400
  assertEquals(a.getAMResourceLimit(), Resources.createResource(400 * GB, 1));
}
 

public FakeRMNodeImpl(NodeId nodeId, String nodeAddr, String httpAddress,
    Resource perNode, String rackName, String healthReport,
    int cmdPort, String hostName, NodeState state) {
  this.nodeId = nodeId;
  this.nodeAddr = nodeAddr;
  this.httpAddress = httpAddress;
  this.perNode = perNode;
  this.rackName = rackName;
  this.healthReport = healthReport;
  this.cmdPort = cmdPort;
  this.hostName = hostName;
  this.state = state;
  toCleanUpApplications = new ArrayList<ApplicationId>();
  toCleanUpContainers = new ArrayList<ContainerId>();
}
 

@VisibleForTesting
protected void handleStatusUpdate(RMNodeEvent event, RMContext context) {
  if (!(event instanceof RMNodeStatusEvent)) {
    logger.error("{} not an instance of {}", event.getClass().getName(), RMNodeStatusEvent.class.getName());
    return;
  }

  RMNodeStatusEvent statusEvent = (RMNodeStatusEvent) event;
  RMNode rmNode = context.getRMNodes().get(event.getNodeId());
  String hostName = rmNode.getNodeID().getHost();

  Node host = nodeStore.getNode(hostName);
  if (host != null) {
    host.snapshotRunningContainers();
  }

  /*
   * Set the new node capacity which is the sum of the current node resources plus those offered by Mesos. 
   * If the sum is greater than the max capacity of the node, reject the offer.
   */
  Resource offeredResources = getNewResourcesOfferedByMesos(hostName);
  Resource currentResources = getResourcesUnderUse(statusEvent);
  
  if (offerWithinResourceLimits(currentResources, offeredResources)) {
    yarnNodeCapacityMgr.setNodeCapacity(rmNode, Resources.add(currentResources, offeredResources));
    logger.info("Updated resources for {} with {} cores and {} memory", rmNode.getNode().getName(), 
            offeredResources.getVirtualCores(), offeredResources.getMemory());
  } else {
    logger.info("Did not update {} with {} cores and {} memory, over max cpu cores and/or max memory", 
            rmNode.getNode().getName(), offeredResources.getVirtualCores(), offeredResources.getMemory());
  }
}
 

@Override
public float ratio(Resource a, Resource b) {
  float max = Math.max(
      (float) a.getMemory() / b.getMemory(),
      (float) a.getVirtualCores() / b.getVirtualCores());
  if (b.getGpuCores() != 0) {
    max = Math.max(max,
        (float) a.getGpuCores() / b.getGpuCores());
  }
  return max;
}
 

@Override
public void setTotalCapacity(Resource cap) {
  writeLock.lock();
  try {
    totalCapacity = Resources.clone(cap);
  } finally {
    writeLock.unlock();
  }
}
 

Optional<Resource> tryComputeContainerResource(final WorkerResourceSpec workerResourceSpec) {
	final InternalContainerResource internalContainerResource = workerSpecToContainerResource.computeIfAbsent(
		Preconditions.checkNotNull(workerResourceSpec),
		this::createAndMapContainerResource);
	if (internalContainerResource != null) {
		return Optional.of(internalContainerResource.toResource());
	} else {
		return Optional.empty();
	}
}