Java Code Examples for org.apache.hadoop.yarn.api.records.Resource#getVirtualCores()

/**
 * Get the per queue setting for the maximum limit to allocate to
 * each container request.
 *
 * @param queue
 *          name of the queue
 * @return setting specified per queue else falls back to the cluster setting
 */
public Resource getMaximumAllocationPerQueue(String queue) {
  String queuePrefix = getQueuePrefix(queue);
  int maxAllocationMbPerQueue = getInt(queuePrefix + MAXIMUM_ALLOCATION_MB,
      (int)UNDEFINED);
  int maxAllocationVcoresPerQueue = getInt(
      queuePrefix + MAXIMUM_ALLOCATION_VCORES, (int)UNDEFINED);
  if (LOG.isDebugEnabled()) {
    LOG.debug("max alloc mb per queue for " + queue + " is "
        + maxAllocationMbPerQueue);
    LOG.debug("max alloc vcores per queue for " + queue + " is "
        + maxAllocationVcoresPerQueue);
  }
  Resource clusterMax = getMaximumAllocation();
  if (maxAllocationMbPerQueue == (int)UNDEFINED) {
    LOG.info("max alloc mb per queue for " + queue + " is undefined");
    maxAllocationMbPerQueue = clusterMax.getMemory();
  }
  if (maxAllocationVcoresPerQueue == (int)UNDEFINED) {
     LOG.info("max alloc vcore per queue for " + queue + " is undefined");
    maxAllocationVcoresPerQueue = clusterMax.getVirtualCores();
  }
  Resource result = Resources.createResource(maxAllocationMbPerQueue,
      maxAllocationVcoresPerQueue);
  if (maxAllocationMbPerQueue > clusterMax.getMemory()
      || maxAllocationVcoresPerQueue > clusterMax.getVirtualCores()) {
    throw new IllegalArgumentException(
        "Queue maximum allocation cannot be larger than the cluster setting"
        + " for queue " + queue
        + " max allocation per queue: " + result
        + " cluster setting: " + clusterMax);
  }
  return result;
}
 

private static void updateAttemptMetrics(RMContainerImpl container) {
  // If this is a preempted container, update preemption metrics
  Resource resource = container.getContainer().getResource();
  RMAppAttempt rmAttempt = container.rmContext.getRMApps()
      .get(container.getApplicationAttemptId().getApplicationId())
      .getCurrentAppAttempt();
  if (ContainerExitStatus.PREEMPTED == container.finishedStatus
    .getExitStatus()) {
    rmAttempt.getRMAppAttemptMetrics().updatePreemptionInfo(resource,
      container);
  }
  
  if (rmAttempt != null) {
    long usedMillis = container.finishTime - container.creationTime;
    long memorySeconds = (long)(resource.getMemory()*container.utilization)
                          * usedMillis / DateUtils.MILLIS_PER_SECOND;
    long vcoreSeconds = (long)(resource.getVirtualCores()*container.utilization)
                         * usedMillis / DateUtils.MILLIS_PER_SECOND;
    
    if (container.suspendTime.size() >0 && container.resumeTime.size() >0 && container.suspendTime.size() == container.resumeTime.size()){
    	double acc=0;
    	for(int i=0; i < container.suspendTime.size();i++){
    		
    		acc = acc + (container.resumeTime.get(i) - container.suspendTime.get(i));
    	}
    	container.utilization = acc/usedMillis;  	
    }
    rmAttempt.getRMAppAttemptMetrics()
              .updateAggregateAppResourceUsage(memorySeconds,vcoreSeconds);
  }
}
 

@Override
public int compareTo(Resource o) {
  int diff = 0 - o.getMemory();
  if (diff == 0) {
    diff = 0 - o.getVirtualCores();
    if (diff == 0) {
      diff = 0 - o.getGpuCores();
    }
  }
  return diff;
}
 

@Override
public int compareTo(Resource o) {
  int diff = 0 - o.getMemory();
  if (diff == 0) {
    diff = 0 - o.getVirtualCores();
    if (diff == 0) {
      diff = 0 - o.getGpuCores();
    }
  }
  return diff;
}
 

/**
 * 1. Updates {@link RMNode#getTotalCapability()} with newCapacity.
 * 2. Sends out a {@link NodeResourceUpdateSchedulerEvent} that's handled by YARN's scheduler.
 * The scheduler updates the corresponding {@link SchedulerNode} with the newCapacity.
 *
 * @param rmNode
 * @param newCapacity
 */
@SuppressWarnings("unchecked")
public void setNodeCapacity(RMNode rmNode, Resource newCapacity) {
  //NOOP prevent YARN warning changing to same size
  if ((Resources.equals(rmNode.getTotalCapability(), newCapacity))) {
    return;
  }
  if (yarnScheduler.getSchedulerNode(rmNode.getNodeID()) == null) {
    LOGGER.info("Yarn Scheduler doesn't have node {}, probably UNHEALTHY", rmNode.getNodeID());
    return;
  }
  yarnSchedulerLock.lock();
  try {
    if (newCapacity.getMemory() < 0 || newCapacity.getVirtualCores() < 0) {
      Resource zeroed = ResourceUtils.componentwiseMax(ZERO_RESOURCE, newCapacity);
      rmNode.getTotalCapability().setMemory(zeroed.getMemory());
      rmNode.getTotalCapability().setVirtualCores(zeroed.getVirtualCores());
      LOGGER.warn("Asked to set Node {} to a value less than zero!  Had {}, setting to {}.",
          rmNode.getHttpAddress(), rmNode.getTotalCapability().toString(), zeroed.toString());
    } else {
      rmNode.getTotalCapability().setMemory(newCapacity.getMemory());
      rmNode.getTotalCapability().setVirtualCores(newCapacity.getVirtualCores());
      if (LOGGER.isInfoEnabled()) {
        LOGGER.info("Setting capacity for node {} to {}", rmNode.getHostName(), newCapacity);
      }
    }
    // updates the scheduler with the new capacity for the NM.
    // the event is handled by the scheduler asynchronously
    rmContext.getDispatcher().getEventHandler().handle(new NodeResourceUpdateSchedulerEvent(rmNode, ResourceOption.newInstance(
        rmNode.getTotalCapability(), RMNode.OVER_COMMIT_TIMEOUT_MILLIS_DEFAULT)));
  } finally {
    yarnSchedulerLock.unlock();
  }
}
 

private void showClusterMetrics(YarnClient yarnClient)
		throws YarnException, IOException {
	YarnClusterMetrics metrics = yarnClient.getYarnClusterMetrics();
	System.out.println("NodeManagers in Cluster " + metrics.getNumNodeManagers());
	List<NodeReport> nodes = yarnClient.getNodeReports();
	final String format = "|%-16s |%-16s %n";
	System.out.printf("|Property         |Value          %n");
	System.out.println("+---------------------------------------+");
	int totalMemory = 0;
	int totalCores = 0;
	for(NodeReport rep : nodes) {
		final Resource res = rep.getCapability();
		totalMemory += res.getMemory();
		totalCores += res.getVirtualCores();
		System.out.format(format, "NodeID", rep.getNodeId());
		System.out.format(format, "Memory", res.getMemory()+" MB");
		System.out.format(format, "vCores", res.getVirtualCores());
		System.out.format(format, "HealthReport", rep.getHealthReport());
		System.out.format(format, "Containers", rep.getNumContainers());
		System.out.println("+---------------------------------------+");
	}
	System.out.println("Summary: totalMemory "+totalMemory+" totalCores "+totalCores);
	List<QueueInfo> qInfo = yarnClient.getAllQueues();
	for(QueueInfo q : qInfo) {
		System.out.println("Queue: "+q.getQueueName()+", Current Capacity: "+q.getCurrentCapacity()+" Max Capacity: "+q.getMaximumCapacity()+" Applications: "+q.getApplications().size());
	}
	yarnClient.stop();
	System.exit(0);
}
 

@Override
public boolean isInvalidDivisor(Resource r) {
  if (r.getMemory() == 0.0f || r.getVirtualCores() == 0.0f || r.getGpuCores() == 0.0f) {
    return true;
  }
  return false;
}
 

private static int getResourceValue(Resource resource, ResourceType type) {
  switch (type) {
  case MEMORY:
    return resource.getMemory();
  case CPU:
    return resource.getVirtualCores();
  case GPU:
    return resource.getGpuCores();
  default:
    throw new IllegalArgumentException("Invalid resource");
  }
}
 

boolean canFit(Resource arg0, Resource arg1) {
  int mem0 = arg0.getMemory();
  int mem1 = arg1.getMemory();
  int cpu0 = arg0.getVirtualCores();
  int cpu1 = arg1.getVirtualCores();
  
  if(mem0 <= mem1 && cpu0 <= cpu1) { 
    return true;
  }
  return false; 
}
 

@Override
public int compareTo(Resource other) {
  int diff = this.getMemory() - other.getMemory();
  if (diff == 0) {
    diff = this.getVirtualCores() - other.getVirtualCores();
  }
  return diff;
}
 

@VisibleForTesting
protected boolean offerWithinResourceLimits(Resource currentResources, Resource offeredResources) {
  int newMemory = currentResources.getMemory() + offeredResources.getMemory();
  int newCores = currentResources.getVirtualCores() + offeredResources.getVirtualCores();
     
  return (newMemory <= conf.getJvmMaxMemoryMB() && newCores <= conf.getMaxCpus());
}
 

synchronized void addAvailableResource(Resource resource) {
  if (resource == null) {
    LOG.error("Invalid resource addition of null resource for "
        + rmNode.getNodeAddress());
    return;
  }
  //chekc if the add resource operation is good to go
  if(resource.getMemory() < 0 && resource.getMemory()+availableResource.getMemory() < 0){
    LOG.error("Invalid resource addition of null resource for "
  	          + rmNode.getNodeAddress());
    return; 	 	
  }
  
  if(resource.getVirtualCores() < 0 && resource.getVirtualCores() + availableResource.getVirtualCores() < 0){
  LOG.error("Invalid resource addition of null resource for "
	          + rmNode.getNodeAddress());
	  return; 	
  }
  
  if(resource.getMemory() > 0 && resource.getMemory()+availableResource.getMemory() > totalResourceCapability.getMemory()){
  LOG.error("Invalid resource addition of null resource for "
  	          + rmNode.getNodeAddress());
  	  return; 		
  }
  
  if(resource.getVirtualCores() > 0 && resource.getVirtualCores() + availableResource.getVirtualCores() > totalResourceCapability.getVirtualCores()){
  LOG.error("Invalid resource addition of null resource for "
  	          + rmNode.getNodeAddress());
  	  return; 		    	
  }
  
  Resources.addTo(availableResource, resource);
  Resources.subtractFrom(usedResource, resource);
}
 

@Override
public int compare(Resource arg0, Resource arg1) {
  int mem0 = arg0.getMemory();
  int mem1 = arg1.getMemory();
  int cpu0 = arg0.getVirtualCores();
  int cpu1 = arg1.getVirtualCores();
  int gpu0 = arg0.getGpuCores();
  int gpu1 = arg1.getGpuCores();
  if(mem0 == mem1) {
    if(cpu0 == cpu1) {
      if(gpu0 == gpu1) {
        return 0;
      }
      if(gpu0 < gpu1) {
        return 1;
      }
      return -1;
    }
    if(cpu0 < cpu1) {
      return 1;
    }
    return -1;
  }
  if(mem0 < mem1) { 
    return 1;
  }
  return -1;
}
 

/**
 * @return {@code True} if cluster contains available resources.
 */
private boolean checkAvailableResource() {
    Resource availableRes = rmClient.getAvailableResources();

    return availableRes == null || availableRes.getMemory() >= props.totalMemoryPerNode()
        && availableRes.getVirtualCores() >= props.cpusPerNode();
}
 

/** 
 *  Return a command to execute the given command in OS shell.
 *  On Windows, the passed in groupId can be used to launch
 *  and associate the given groupId in a process group. On
 *  non-Windows, groupId is ignored. 
 */
protected String[] getRunCommand(String command, String groupId,
    String userName, Path pidFile, Configuration conf, Resource resource) {
  boolean containerSchedPriorityIsSet = false;
  int containerSchedPriorityAdjustment = 
      YarnConfiguration.DEFAULT_NM_CONTAINER_EXECUTOR_SCHED_PRIORITY;

  if (conf.get(YarnConfiguration.NM_CONTAINER_EXECUTOR_SCHED_PRIORITY) != 
      null) {
    containerSchedPriorityIsSet = true;
    containerSchedPriorityAdjustment = conf 
        .getInt(YarnConfiguration.NM_CONTAINER_EXECUTOR_SCHED_PRIORITY, 
        YarnConfiguration.DEFAULT_NM_CONTAINER_EXECUTOR_SCHED_PRIORITY);
  }

  if (Shell.WINDOWS) {
    int cpuRate = -1;
    int memory = -1;
    if (resource != null) {
      if (conf
          .getBoolean(
              YarnConfiguration.NM_WINDOWS_CONTAINER_MEMORY_LIMIT_ENABLED,
              YarnConfiguration.DEFAULT_NM_WINDOWS_CONTAINER_MEMORY_LIMIT_ENABLED)) {
        memory = resource.getMemory();
      }

      if (conf.getBoolean(
          YarnConfiguration.NM_WINDOWS_CONTAINER_CPU_LIMIT_ENABLED,
          YarnConfiguration.DEFAULT_NM_WINDOWS_CONTAINER_CPU_LIMIT_ENABLED)) {
        int containerVCores = resource.getVirtualCores();
        int nodeVCores = conf.getInt(YarnConfiguration.NM_VCORES,
            YarnConfiguration.DEFAULT_NM_VCORES);
        // cap overall usage to the number of cores allocated to YARN
        int nodeCpuPercentage = Math
            .min(
                conf.getInt(
                    YarnConfiguration.NM_RESOURCE_PERCENTAGE_PHYSICAL_CPU_LIMIT,
                    YarnConfiguration.DEFAULT_NM_RESOURCE_PERCENTAGE_PHYSICAL_CPU_LIMIT),
                100);
        nodeCpuPercentage = Math.max(0, nodeCpuPercentage);
        if (nodeCpuPercentage == 0) {
          String message = "Illegal value for "
              + YarnConfiguration.NM_RESOURCE_PERCENTAGE_PHYSICAL_CPU_LIMIT
              + ". Value cannot be less than or equal to 0.";
          throw new IllegalArgumentException(message);
        }
        float yarnVCores = (nodeCpuPercentage * nodeVCores) / 100.0f;
        // CPU should be set to a percentage * 100, e.g. 20% cpu rate limit
        // should be set as 20 * 100. The following setting is equal to:
        // 100 * (100 * (vcores / Total # of cores allocated to YARN))
        cpuRate = Math.min(10000,
            (int) ((containerVCores * 10000) / yarnVCores));
      }
    }
    return new String[] { Shell.WINUTILS, "task", "create", "-m",
        String.valueOf(memory), "-c", String.valueOf(cpuRate), groupId,
        "cmd /c " + command };
  } else {
    List<String> retCommand = new ArrayList<String>();
    if (containerSchedPriorityIsSet) {
      retCommand.addAll(Arrays.asList("nice", "-n",
          Integer.toString(containerSchedPriorityAdjustment)));
    }
    retCommand.addAll(Arrays.asList("bash", command));
    return retCommand.toArray(new String[retCommand.size()]);
  }

}
 

boolean canFit(Resource capability) {
  Resource cr = container.getResource();
  return cr.getMemory() >= capability.getMemory() && cr.getVirtualCores() >= capability.getVirtualCores();
}
 

public static boolean fitsIn(Resource smaller, Resource bigger) {
  return smaller.getMemory() <= bigger.getMemory() &&
      smaller.getVirtualCores() <= bigger.getVirtualCores();
}
 

public ResourceInfo(Resource res) {
  memory = res.getMemory();
  vCores = res.getVirtualCores();
  gCores = res.getGpuCores();
}
 

public IntegralResource(Resource resource) {
  this.memory = resource.getMemory();
  this.vcores = resource.getVirtualCores();
}
 

@Override
public boolean anyAvailable(Resource rsrc) {
  return super.anyAvailable(rsrc) || rsrc.getVirtualCores() > 0;
}