Java Code Examples for org.apache.hadoop.yarn.api.records.Resource#setVirtualCores()

/**
 * Setup the request(s) that will be sent to the RM for the container ask.
 */
public ContainerRequest createContainerRequest(ContainerStartRequest csr, boolean first)
{
  int priority = csr.container.getResourceRequestPriority();
  // check for node locality constraint
  String[] nodes = null;
  String[] racks = null;

  String host = getHost(csr, first);
  Resource capability = Records.newRecord(Resource.class);
  capability.setMemory(csr.container.getRequiredMemoryMB());
  capability.setVirtualCores(csr.container.getRequiredVCores());
  if (host == INVALID_HOST) {
    return null;
  }
  if (host != null) {
    nodes = new String[]{host};
    // in order to request a host, we don't have to set the rack if the locality is false
    /*
     * if(this.nodeToRack.get(host) != null){ racks = new String[] { this.nodeToRack.get(host) }; }
     */
    return new ContainerRequest(capability, nodes, racks, Priority.newInstance(priority), false);
  }
  // For now, only memory is supported so we set memory requirements
  return new ContainerRequest(capability, nodes, racks, Priority.newInstance(priority));
}
 

@Test(timeout = 15000)
public void testSubmitApplicationOnHA() throws Exception {
  ApplicationSubmissionContext appContext =
      Records.newRecord(ApplicationSubmissionContext.class);
  appContext.setApplicationId(cluster.createFakeAppId());
  ContainerLaunchContext amContainer =
      Records.newRecord(ContainerLaunchContext.class);
  appContext.setAMContainerSpec(amContainer);
  Resource capability = Records.newRecord(Resource.class);
  capability.setMemory(10);
  capability.setVirtualCores(1);
  capability.setGpuCores(1);
  appContext.setResource(capability);
  ApplicationId appId = client.submitApplication(appContext);
  Assert.assertTrue(getActiveRM().getRMContext().getRMApps()
      .containsKey(appId));
}
 

/**
 * Test RegisterNodeManagerRequestPBImpl.
 */

@Test
public void testRegisterNodeManagerRequestPBImpl() {
  RegisterNodeManagerRequestPBImpl original = new RegisterNodeManagerRequestPBImpl();
  original.setHttpPort(8080);
  original.setNodeId(getNodeId());
  Resource resource = recordFactory.newRecordInstance(Resource.class);
  resource.setMemory(10000);
  resource.setVirtualCores(2);
  resource.setGpuCores(3);
  original.setResource(resource);
  RegisterNodeManagerRequestPBImpl copy = new RegisterNodeManagerRequestPBImpl(
      original.getProto());

  assertEquals(8080, copy.getHttpPort());
  assertEquals(9090, copy.getNodeId().getPort());
  assertEquals(10000, copy.getResource().getMemory());
  assertEquals(2, copy.getResource().getVirtualCores());
  assertEquals(3, copy.getResource().getGpuCores());

}
 

public static Resource subtractFroms(Resource lhs, Resource rhs){
if(lhs.getMemory() > rhs.getMemory()){  
  lhs.setMemory(lhs.getMemory() - rhs.getMemory());
}else{
  lhs.setMemory(0);	
}

if(lhs.getVirtualCores() > rhs.getVirtualCores()){
  lhs.setVirtualCores(lhs.getVirtualCores() - rhs.getVirtualCores()); 
}else{
  lhs.setVirtualCores(0);	
}

return lhs;
 }
 

private static void setResourceValue(int val, Resource resource, ResourceType type) {
  switch (type) {
  case MEMORY:
    resource.setMemory(val);
    break;
  case CPU:
    resource.setVirtualCores(val);
    break;
  case GPU:
    resource.setGpuCores(val);
    break;
  default:
    throw new IllegalArgumentException("Invalid resource");
  }
}
 

public static Resource newResource(int memory, int vCores) {
  Resource resource = recordFactory.newRecordInstance(Resource.class);
  resource.setMemory(memory);
  resource.setVirtualCores(vCores);
  resource.setGpuCores(0);
  return resource;
}
 

private synchronized Resource getResource(final ResourceRequestEvent resourceRequestEvent) {
  final Resource result = Records.newRecord(Resource.class);
  final int memory = getMemory(resourceRequestEvent.getMemorySize().get());
  final int core = resourceRequestEvent.getVirtualCores().get();
  LOG.log(Level.FINEST, "Resource requested: memory = {0}, virtual core count = {1}.", new Object[]{memory, core});
  result.setMemory(memory);
  result.setVirtualCores(core);
  return result;
}
 

public static Resource createResource(int memory, int cores, int gcores) {
  Resource resource = Records.newRecord(Resource.class);
  resource.setMemory(memory);
  resource.setVirtualCores(cores);
  resource.setGpuCores(gcores);
  return resource;
}
 

/**
 * 构建对于container的申请
 * @param containerType
 * @param mem
 * @return
 */
public static AMRMClient.ContainerRequest setupContainerAskForRM(int containerType, int mem) {
    Priority pri = Records.newRecord(Priority.class);
    pri.setPriority(containerType);
    Resource capability = Records.newRecord(Resource.class);
    capability.setMemory(mem);
    capability.setVirtualCores(1);
    AMRMClient.ContainerRequest request = new AMRMClient.ContainerRequest(capability, null, null, pri);
    LOG.error("#### 申请container:[" + request.toString()+"]");
    return request;
}
 

public static Resource multiplyTo(Resource lhs, double by) {
  lhs.setMemory((int)(lhs.getMemory() * by));
  lhs.setVirtualCores((int)(lhs.getVirtualCores() * by));
  lhs.setGpuCores((int) (lhs.getGpuCores() * by));
  return lhs;
}
 

@Override
public void deductFrom(Resource total, Resource toSubtract) {
  super.deductFrom(total, toSubtract);
  total.setVirtualCores(total.getVirtualCores() - toSubtract.getVirtualCores());
}
 

public static Resource addTo(Resource lhs, Resource rhs) {
  lhs.setMemory(lhs.getMemory() + rhs.getMemory());
  lhs.setVirtualCores(lhs.getVirtualCores() + rhs.getVirtualCores());
  lhs.setGpuCores(lhs.getGpuCores() + rhs.getGpuCores());
  return lhs;
}
 

public static Resource newResource(int memory, int vCores) {
  Resource resource = recordFactory.newRecordInstance(Resource.class);
  resource.setMemory(memory);
  resource.setVirtualCores(vCores);
  return resource;
}
 

@Test public void testNames() {
  DefaultMetricsSystem.initialize("NodeManager");
  NodeManagerMetrics metrics = NodeManagerMetrics.create();
  Resource total = Records.newRecord(Resource.class);
  total.setMemory(8*GiB);
  total.setVirtualCores(16);
  total.setGpuCores(16);
  Resource resource = Records.newRecord(Resource.class);
  resource.setMemory(512); //512MiB
  resource.setVirtualCores(2);
  resource.setGpuCores(1);


  metrics.addResource(total);

  for (int i = 10; i-- > 0;) {
    // allocate 10 containers(allocatedGB: 5GiB, availableGB: 3GiB)
    metrics.launchedContainer();
    metrics.allocateContainer(resource);
  }

  metrics.initingContainer();
  metrics.endInitingContainer();
  metrics.runningContainer();
  metrics.endRunningContainer();
  // Releasing 3 containers(allocatedGB: 3.5GiB, availableGB: 4.5GiB)
  metrics.completedContainer();
  metrics.releaseContainer(resource);

  metrics.failedContainer();
  metrics.releaseContainer(resource);

  metrics.killedContainer();
  metrics.releaseContainer(resource);

  metrics.initingContainer();
  metrics.runningContainer();

  Assert.assertTrue(!metrics.containerLaunchDuration.changed());
  metrics.addContainerLaunchDuration(1);
  Assert.assertTrue(metrics.containerLaunchDuration.changed());

  // availableGB is expected to be floored,
  // while allocatedGB is expected to be ceiled.
  // allocatedGB: 3.5GB allocated memory is shown as 4GB
  // availableGB: 4.5GB available memory is shown as 4GB
  checkMetrics(10, 1, 1, 1, 1, 1, 4, 7, 4, 14, 2, 7, 9);
}
 

private ApplicationReport startAppMaster(ApplicationSubmissionContext appContext) throws Exception {
  appContext.setMaxAppAttempts(MAX_ATTEMPT);

  Map<String, LocalResource> localResources = new HashMap<>();
  Set<Path> shippedPaths = new HashSet<>();
  collectLocalResources(localResources, shippedPaths);
  final ContainerLaunchContext amContainer = setupApplicationMasterContainer(
      this.getYarnSessionClusterEntrypoint(),
      false,
      true,
      false, (int) job.taskManagerMemoryMb());

  amContainer.setLocalResources(localResources);

  final String classPath = localResources.keySet().stream().collect(Collectors.joining(File.pathSeparator));
  final String shippedFiles = shippedPaths.stream().map(x -> x.getName() + "=" + x.toString())
      .collect(Collectors.joining(","));

  // Setup CLASSPATH and environment variables for ApplicationMaster
  ApplicationId appId = appContext.getApplicationId();
  final Map<String, String> appMasterEnv = setUpAmEnvironment(
      appId,
      classPath,
      shippedFiles,
      getDynamicPropertiesEncoded()
  );

  amContainer.setEnvironment(appMasterEnv);

  // Set up resource type requirements for ApplicationMaster
  Resource capability = Records.newRecord(Resource.class);
  capability.setMemory(getFlinkConfiguration()
      .getInteger(JobManagerOptions.JOB_MANAGER_HEAP_MEMORY));
  capability.setVirtualCores(1);

  appContext.setApplicationName(job.name());
  appContext.setApplicationType(ATHENAX_APPLICATION_TYPE);
  appContext.setAMContainerSpec(amContainer);
  appContext.setResource(capability);
  appContext.setApplicationTags(Collections.singleton(job.metadata().serialize()));
  if (job.queue() != null) {
    appContext.setQueue(job.queue());
  }

  LOG.info("Submitting application master {}", appId);
  yarnClient.submitApplication(appContext);

  PollDeploymentStatus poll = new PollDeploymentStatus(appId);
  YARN_POLL_EXECUTOR.submit(poll);
  try {
    return poll.result.get();
  } catch (ExecutionException e) {
    LOG.warn("Failed to deploy {}, cause: {}", appId.toString(), e.getCause());
    yarnClient.killApplication(appId);
    throw (Exception) e.getCause();
  }
}
 

Resource offer(Resource avail, ResourceCalculator rc,
    Resource clusterResource) {
 	
  Resource absMaxCapIdealAssignedDelta = Resources.componentwiseMax(
                  Resources.subtract(maxCapacity, idealAssigned),
                  Resource.newInstance(0, 0));
  
  // remain = avail - min(avail, (max - assigned), (current + pending - assigned))
  // we have bug here. in some case:
  //(current + pending - assigned).core > avail.core
  //(current + pending - assigned).memo < avail.memo
  //so we get least cores of the three and least memory of the three
  Resource possibleAccepted = 
      Resources.mins(rc, clusterResource, 
          absMaxCapIdealAssignedDelta,
      Resources.mins(rc, clusterResource, avail, Resources.subtract(
          Resources.add(current, pending), idealAssigned)));
  
  //final allocation resource
  Resource finalAccepted = Resources.clone(possibleAccepted);
  //in extrame case where avail cores are more less than the available memory, it may preempt mroe memory
  //Max:      1310720   320
  //avail:    542634    26
  //Delta:    734280    60  
  //Pending:  525312    120
  //current:  576512    260
  //ideal:    576512    260
  //then the accepted will be (525312,26) in which the memory is far more beyond the requirement
  
  if(isSuspended){
	  
  if(dominantResource == Resources.CPU && !Resources.equals(pending,Resources.none())){
	  //pending must be either none() or resource(int ,int)
	  if(avail.getVirtualCores() == 0){
	      //if the dominant resource is cpu, we will stop allocation even we have memory
		  finalAccepted.setMemory(0);
		  //but if we still have more available memory, we can allocate, to avoid preemption
		  //we set memory to current usage
		  int gapMemory = current.getMemory() - idealAssigned.getMemory();
		  if(gapMemory > 0 && possibleAccepted.getMemory() > gapMemory){
			  finalAccepted.setMemory(gapMemory);
			  LOG.info("gap memory: "+gapMemory);
		  }
		  
	  }else{
	  double memoryRatio   = pending.getMemory()*1.0/pending.getVirtualCores();
	  int    ratioedMemory = (int)(memoryRatio*possibleAccepted.getVirtualCores());
	  finalAccepted.setMemory(ratioedMemory < possibleAccepted.getMemory() ? 
			                   ratioedMemory:possibleAccepted.getMemory());
	  }
	  LOG.info("queue: "+queueName+" cpu dominant ");
	  
	  if(finalAccepted.getMemory() < possibleAccepted.getMemory()){
		  LOG.info("previous memory: "+possibleAccepted.getMemory()+"  final memory: "+finalAccepted.getMemory());
	  }
	  
	  
  }else if(dominantResource == Resources.MEMORY && !Resources.equals(pending, Resources.none())){
	  if(avail.getMemory() == 0){
		  finalAccepted.setVirtualCores(0);
		  int gapCores = current.getVirtualCores() - idealAssigned.getVirtualCores();
		  if(gapCores > 0 && possibleAccepted.getVirtualCores() > gapCores){
			  finalAccepted.setVirtualCores(gapCores);
			  LOG.info("gap cores: "+gapCores);
		  }
		  
	  }else{
	  double cpuRatio   = pending.getVirtualCores()*1.0/pending.getMemory();
	  int    ratioedcpu = (int)(cpuRatio*possibleAccepted.getMemory());
	  finalAccepted.setVirtualCores(ratioedcpu < possibleAccepted.getMemory() ? 
			                 ratioedcpu:possibleAccepted.getMemory());
	  }
	  LOG.info("queue: "+queueName+" memory dominant ");
  }else{
	  LOG.info("queue: "+queueName+" empty ");
  }
  
  }
 
  LOG.info("queueName:   "+queueName);
  LOG.info("beforeideal: "+idealAssigned);  
  Resource remain = Resources.subtract(avail, finalAccepted);
  Resources.addTo(idealAssigned, finalAccepted);
  LOG.info("avaul:       "+avail);
  LOG.info("absMaxDelta: "+absMaxCapIdealAssignedDelta);
  LOG.info("max:         "+maxCapacity);
  LOG.info("current:     "+current);
  LOG.info("pending:     "+pending);
  LOG.info("acceped:     "+finalAccepted);
  LOG.info("ideal:       "+idealAssigned);
  
  return remain;
  
}
 

@Test
public void testNodeRegistrationWithMinimumAllocations() throws Exception {
  Configuration conf = new Configuration();
  conf.set(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, "2048");
  conf.set(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES, "4");
  rm = new MockRM(conf);
  rm.start();

  ResourceTrackerService resourceTrackerService
    = rm.getResourceTrackerService();
  RegisterNodeManagerRequest req = Records.newRecord(
      RegisterNodeManagerRequest.class);
  NodeId nodeId = BuilderUtils.newNodeId("host", 1234);
  req.setNodeId(nodeId);

  Resource capability = BuilderUtils.newResource(1024, 1);
  req.setResource(capability);
  RegisterNodeManagerResponse response1 =
      resourceTrackerService.registerNodeManager(req);
  Assert.assertEquals(NodeAction.SHUTDOWN,response1.getNodeAction());
  
  capability.setMemory(2048);
  capability.setVirtualCores(1);
  req.setResource(capability);
  RegisterNodeManagerResponse response2 =
      resourceTrackerService.registerNodeManager(req);
  Assert.assertEquals(NodeAction.SHUTDOWN,response2.getNodeAction());
  
  capability.setMemory(1024);
  capability.setVirtualCores(4);
  req.setResource(capability);
  RegisterNodeManagerResponse response3 =
      resourceTrackerService.registerNodeManager(req);
  Assert.assertEquals(NodeAction.SHUTDOWN,response3.getNodeAction());
  
  capability.setMemory(2048);
  capability.setVirtualCores(4);
  req.setResource(capability);
  RegisterNodeManagerResponse response4 =
      resourceTrackerService.registerNodeManager(req);
  Assert.assertEquals(NodeAction.NORMAL,response4.getNodeAction());
}
 

public static Resource createResource(int memory, int cores) {
  Resource resource = Records.newRecord(Resource.class);
  resource.setMemory(memory);
  resource.setVirtualCores(cores);
  return resource;
}
 

public static Resource addTo(Resource lhs, Resource rhs) {
  lhs.setMemory(lhs.getMemory() + rhs.getMemory());
  lhs.setVirtualCores(lhs.getVirtualCores() + rhs.getVirtualCores());
  return lhs;
}
 

/**
 * Runs application master.
 *
 * @throws Exception If failed.
 */
public void run() throws Exception {
    // Register with ResourceManager
    rmClient.registerApplicationMaster("", 0, "");

    log.log(Level.INFO, "Application master registered.");

    // Priority for worker containers - priorities are intra-application
    Priority priority = Records.newRecord(Priority.class);
    priority.setPriority(0);

    try {
        // Check ignite cluster.
        while (!nmClient.isInState(Service.STATE.STOPPED)) {
            int runningCnt = containers.size();

            if (runningCnt < props.instances() && checkAvailableResource()) {
                // Resource requirements for worker containers.
                Resource capability = Records.newRecord(Resource.class);

                capability.setMemory((int)props.totalMemoryPerNode());
                capability.setVirtualCores((int)props.cpusPerNode());

                for (int i = 0; i < props.instances() - runningCnt; ++i) {
                    // Make container requests to ResourceManager
                    AMRMClient.ContainerRequest containerAsk =
                        new AMRMClient.ContainerRequest(capability, null, null, priority);

                    rmClient.addContainerRequest(containerAsk);

                    log.log(Level.INFO, "Making request. Memory: {0}, cpu {1}.",
                        new Object[]{props.totalMemoryPerNode(), props.cpusPerNode()});
                }
            }

            TimeUnit.MILLISECONDS.sleep(schedulerTimeout);
        }
    }
    catch (InterruptedException ignored) {
        // Un-register with ResourceManager
        rmClient.unregisterApplicationMaster(FinalApplicationStatus.KILLED, "", "");

        log.log(Level.WARNING, "Application master killed.");
    }
    catch (Exception e) {
        // Un-register with ResourceManager
        rmClient.unregisterApplicationMaster(FinalApplicationStatus.FAILED, "", "");

        log.log(Level.SEVERE, "Application master failed.", e);
    }
}