Java Code Examples for org.apache.hadoop.yarn.api.records.Resource#getGpuCores()

protected void refreshMaximumAllocation(Resource newMaxAlloc) {
  maxAllocWriteLock.lock();
  try {
    configuredMaximumAllocation = Resources.clone(newMaxAlloc);
    int maxMemory = newMaxAlloc.getMemory();
    if (maxNodeMemory != -1) {
      maxMemory = Math.min(maxMemory, maxNodeMemory);
    }
    int maxVcores = newMaxAlloc.getVirtualCores();
    if (maxNodeVCores != -1) {
      maxVcores = Math.min(maxVcores, maxNodeVCores);
    }
    int maxGcores = newMaxAlloc.getGpuCores();
    if (maxNodeGCores != -1) {
      maxGcores = Math.min(maxGcores, maxNodeGCores);
    }
    maximumAllocation = Resources.createResource(maxMemory, maxVcores, maxGcores);
  } finally {
    maxAllocWriteLock.unlock();
  }
}
 

private static void updateAttemptMetrics(RMContainerImpl container) {
  // If this is a preempted container, update preemption metrics
  Resource resource = container.getContainer().getResource();
  RMAppAttempt rmAttempt = container.rmContext.getRMApps()
      .get(container.getApplicationAttemptId().getApplicationId())
      .getCurrentAppAttempt();
  if (ContainerExitStatus.PREEMPTED == container.finishedStatus
    .getExitStatus()) {
    rmAttempt.getRMAppAttemptMetrics().updatePreemptionInfo(resource,
      container);
  }

  if (rmAttempt != null) {
    long usedMillis = container.finishTime - container.creationTime;
    long memorySeconds = resource.getMemory()
                          * usedMillis / DateUtils.MILLIS_PER_SECOND;
    long vcoreSeconds = resource.getVirtualCores()
                         * usedMillis / DateUtils.MILLIS_PER_SECOND;
    long gcoreSeconds = resource.getGpuCores()
                         * usedMillis / DateUtils.MILLIS_PER_SECOND;
    rmAttempt.getRMAppAttemptMetrics()
              .updateAggregateAppResourceUsage(memorySeconds,vcoreSeconds, gcoreSeconds);
  }
}
 

@Override
public int compareTo(Resource o) {
  int diff = 0 - o.getMemory();
  if (diff == 0) {
    diff = 0 - o.getVirtualCores();
    if (diff == 0) {
      diff = 0 - o.getGpuCores();
    }
  }
  return diff;
}
 

@Override
public int compareTo(Resource o) {
  int diff = 0 - o.getMemory();
  if (diff == 0) {
    diff = 0 - o.getVirtualCores();
    if (diff == 0) {
      diff = 0 - o.getGpuCores();
    }
  }
  return diff;
}
 

@Override
public int compareTo(Resource other) {
  int diff = this.getMemory() - other.getMemory();
  if (diff == 0) {
    diff = this.getVirtualCores() - other.getVirtualCores();
    if (diff == 0) {
      diff = this.getGpuCores() - other.getGpuCores();
    }
  }
  return diff;
}
 

@Override
public int compare(Resource clusterResource, Resource lhs, Resource rhs) {
  
  if (lhs.equals(rhs)) {
    return 0;
  }

  float[] lValues = new float[] {
    (clusterResource.getMemory() != 0) ? (float) lhs.getMemory() / clusterResource.getMemory() : lhs.getMemory(),
    (clusterResource.getVirtualCores() != 0) ? (float) lhs.getVirtualCores() / clusterResource.getVirtualCores() : lhs.getVirtualCores(),
    (clusterResource.getGpuCores() != 0) ? (float) lhs.getGpuCores() / clusterResource.getGpuCores() : 0.0f };
  Arrays.sort(lValues);

  float[] rValues = new float[] {
    (clusterResource.getMemory() != 0) ? (float) rhs.getMemory() / clusterResource.getMemory() : rhs.getMemory(),
    (clusterResource.getVirtualCores() != 0) ? (float) rhs.getVirtualCores() / clusterResource.getVirtualCores() : rhs.getVirtualCores(),
    (clusterResource.getGpuCores() != 0) ? (float) rhs.getGpuCores() / clusterResource.getGpuCores() : 0.0f };
  Arrays.sort(rValues);

  int diff = 0;
  for(int i = 0; i < 3; i++) {
    float l = lValues[i];
    float r = rValues[i];
    if (l < r) {
      diff = -1;
    } else if (l > r) {
      diff = 1;
    }
  }
  
  return diff;
}
 

public static int divideAndCeilContainers(Resource required, Resource factor,
    EnumSet<SchedulerResourceTypes> resourceTypes) {
  if (resourceTypes.contains(SchedulerResourceTypes.CPU)) {
    if (factor.getGpuCores() == 0) {
      return Math.max(divideAndCeil(required.getMemory(), factor.getMemory()),
              divideAndCeil(required.getVirtualCores(), factor.getVirtualCores()));
    }
    else {
      return Math.max(divideAndCeil(required.getMemory(), factor.getMemory()),
              Math.max(divideAndCeil(required.getVirtualCores(), factor.getVirtualCores()),
                      divideAndCeil(required.getGpuCores(), factor.getGpuCores())));
    }
  }
  return divideAndCeil(required.getMemory(), factor.getMemory());
}
 

@Override
public int computeAvailableContainers(Resource available, Resource required) {
  int min = Math.min(
      available.getMemory() / required.getMemory(),
      available.getVirtualCores() / required.getVirtualCores());
  if (required.getGpuCores() != 0) {
    min = Math.min(min,
        available.getGpuCores() / required.getGpuCores());
  }
  return min;
}
 

@Override
public boolean isInvalidDivisor(Resource r) {
  if (r.getMemory() == 0.0f || r.getVirtualCores() == 0.0f || r.getGpuCores() == 0.0f) {
    return true;
  }
  return false;
}
 

@Override
public int compare(Resource arg0, Resource arg1) {
  int mem0 = arg0.getMemory();
  int mem1 = arg1.getMemory();
  int cpu0 = arg0.getVirtualCores();
  int cpu1 = arg1.getVirtualCores();
  int gpu0 = arg0.getGpuCores();
  int gpu1 = arg1.getGpuCores();
  if(mem0 == mem1) {
    if(cpu0 == cpu1) {
      if(gpu0 == gpu1) {
        return 0;
      }
      if(gpu0 < gpu1) {
        return 1;
      }
      return -1;
    }
    if(cpu0 < cpu1) {
      return 1;
    }
    return -1;
  }
  if(mem0 < mem1) { 
    return 1;
  }
  return -1;
}
 

@Override
public synchronized void reinitialize(
    CSQueue newlyParsedQueue, Resource clusterResource) 
throws IOException {
  // Sanity check
  if (!(newlyParsedQueue instanceof LeafQueue) || 
      !newlyParsedQueue.getQueuePath().equals(getQueuePath())) {
    throw new IOException("Trying to reinitialize " + getQueuePath() + 
        " from " + newlyParsedQueue.getQueuePath());
  }

  LeafQueue newlyParsedLeafQueue = (LeafQueue)newlyParsedQueue;

  // don't allow the maximum allocation to be decreased in size
  // since we have already told running AM's the size
  Resource oldMax = getMaximumAllocation();
  Resource newMax = newlyParsedLeafQueue.getMaximumAllocation();
  if (newMax.getMemory() < oldMax.getMemory()
      || newMax.getVirtualCores() < oldMax.getVirtualCores()
      || newMax.getGpuCores() < oldMax.getGpuCores()) {
    throw new IOException(
        "Trying to reinitialize "
            + getQueuePath()
            + " the maximum allocation size can not be decreased!"
            + " Current setting: " + oldMax
            + ", trying to set it to: " + newMax);
  }

  setupQueueConfigs(clusterResource);

  // queue metrics are updated, more resource may be available
  // activate the pending applications if possible
  activateApplications();
}
 

static boolean canFit(Resource arg0, Resource arg1) {
  int mem0 = arg0.getMemory();
  int mem1 = arg1.getMemory();
  int cpu0 = arg0.getVirtualCores();
  int cpu1 = arg1.getVirtualCores();
  int gpu0 = arg0.getGpuCores();
  int gpu1 = arg1.getGpuCores();
  
  if(mem0 <= mem1 && cpu0 <= cpu1 && gpu0 <= gpu1) {
    return true;
  }
  return false; 
}
 

private static int getResourceValue(Resource resource, ResourceType type) {
  switch (type) {
  case MEMORY:
    return resource.getMemory();
  case CPU:
    return resource.getVirtualCores();
  case GPU:
    return resource.getGpuCores();
  default:
    throw new IllegalArgumentException("Invalid resource");
  }
}
 

public static int computeAvailableContainers(Resource available,
    Resource required, EnumSet<SchedulerResourceTypes> resourceTypes) {
  if (resourceTypes.contains(SchedulerResourceTypes.CPU)) {
    if (required.getGpuCores() == 0) {
      return Math.min(available.getMemory() / required.getMemory(),
              available.getVirtualCores() / required.getVirtualCores());
    }
    else {
      return Math.min(available.getMemory() / required.getMemory(),
              Math.min(available.getVirtualCores() / required.getVirtualCores(),
                      available.getGpuCores() / required.getGpuCores()));
    }
  }
  return available.getMemory() / required.getMemory();
}
 

/**
 * Get the per queue setting for the maximum limit to allocate to
 * each container request.
 *
 * @param queue
 *          name of the queue
 * @return setting specified per queue else falls back to the cluster setting
 */
public Resource getMaximumAllocationPerQueue(String queue) {
  String queuePrefix = getQueuePrefix(queue);
  int maxAllocationMbPerQueue = getInt(queuePrefix + MAXIMUM_ALLOCATION_MB,
      (int)UNDEFINED);
  int maxAllocationVcoresPerQueue = getInt(
      queuePrefix + MAXIMUM_ALLOCATION_VCORES, (int)UNDEFINED);
  int maxAllocationGcoresPerQueue = getInt(
      queuePrefix + MAXIMUM_ALLOCATION_GCORES, (int)UNDEFINED);
  if (LOG.isDebugEnabled()) {
    LOG.debug("max alloc mb per queue for " + queue + " is "
        + maxAllocationMbPerQueue);
    LOG.debug("max alloc vcores per queue for " + queue + " is "
        + maxAllocationVcoresPerQueue);
    LOG.debug("max alloc gcores per queue for " + queue + " is "
        + maxAllocationGcoresPerQueue);
  }
  Resource clusterMax = getMaximumAllocation();
  if (maxAllocationMbPerQueue == (int)UNDEFINED) {
    LOG.info("max alloc mb per queue for " + queue + " is undefined");
    maxAllocationMbPerQueue = clusterMax.getMemory();
  }
  if (maxAllocationVcoresPerQueue == (int)UNDEFINED) {
     LOG.info("max alloc vcore per queue for " + queue + " is undefined");
    maxAllocationVcoresPerQueue = clusterMax.getVirtualCores();
  }
  if (maxAllocationGcoresPerQueue == (int)UNDEFINED) {
    LOG.info("max alloc gcore per queue for " + queue + " is undefined");
    maxAllocationGcoresPerQueue = clusterMax.getGpuCores();
  }
  Resource result = Resources.createResource(maxAllocationMbPerQueue,
      maxAllocationVcoresPerQueue, maxAllocationGcoresPerQueue);
  if (maxAllocationMbPerQueue > clusterMax.getMemory()
      || maxAllocationVcoresPerQueue > clusterMax.getVirtualCores()
      || maxAllocationGcoresPerQueue > clusterMax.getGpuCores()) {
    throw new IllegalArgumentException(
        "Queue maximum allocation cannot be larger than the cluster setting"
        + " for queue " + queue
        + " max allocation per queue: " + result
        + " cluster setting: " + clusterMax);
  }
  return result;
}
 

/**
 * Utility method to validate a resource request, by insuring that the
 * requested memory/vcore/gcore is non-negative and not greater than max
 * 
 * @throws InvalidResourceRequestException when there is invalid request
 */
private static void validateResourceRequest(ResourceRequest resReq,
    Resource maximumResource, QueueInfo queueInfo, RMContext rmContext)
    throws InvalidResourceRequestException {
  if (resReq.getCapability().getMemory() < 0 ||
      resReq.getCapability().getMemory() > maximumResource.getMemory()) {
    throw new InvalidResourceRequestException("Invalid resource request"
        + ", requested memory < 0"
        + ", or requested memory > max configured"
        + ", requestedMemory=" + resReq.getCapability().getMemory()
        + ", maxMemory=" + maximumResource.getMemory());
  }
  if (resReq.getCapability().getVirtualCores() < 0 ||
      resReq.getCapability().getVirtualCores() >
      maximumResource.getVirtualCores()) {
    throw new InvalidResourceRequestException("Invalid resource request"
        + ", requested virtual cores < 0"
        + ", or requested virtual cores > max configured"
        + ", requestedVirtualCores="
        + resReq.getCapability().getVirtualCores()
        + ", maxVirtualCores=" + maximumResource.getVirtualCores());
  }
  if (resReq.getCapability().getGpuCores() < 0 ||
      resReq.getCapability().getGpuCores() >
      maximumResource.getGpuCores()) {
    throw new InvalidResourceRequestException("Invalid resource request"
        + ", requested gpu cores < 0"
        + ", or requested gpu cores > max configured"
        + ", requestedGpuCores="
        + resReq.getCapability().getGpuCores()
        + ", maxGpuCores=" + maximumResource.getGpuCores());
  }
  String labelExp = resReq.getNodeLabelExpression();

  // we don't allow specify label expression other than resourceName=ANY now
  if (!ResourceRequest.ANY.equals(resReq.getResourceName())
      && labelExp != null && !labelExp.trim().isEmpty()) {
    throw new InvalidResourceRequestException(
        "Invailid resource request, queue=" + queueInfo.getQueueName()
            + " specified node label expression in a "
            + "resource request has resource name = "
            + resReq.getResourceName());
  }
  
  // we don't allow specify label expression with more than one node labels now
  if (labelExp != null && labelExp.contains("&&")) {
    throw new InvalidResourceRequestException(
        "Invailid resource request, queue=" + queueInfo.getQueueName()
            + " specified more than one node label "
            + "in a node label expression, node label expression = "
            + labelExp);
  }
  
  if (labelExp != null && !labelExp.trim().isEmpty() && queueInfo != null) {
    if (!checkQueueLabelExpression(queueInfo.getAccessibleNodeLabels(),
        labelExp, rmContext)) {
      throw new InvalidResourceRequestException("Invalid resource request"
          + ", queue="
          + queueInfo.getQueueName()
          + " doesn't have permission to access all labels "
          + "in resource request. labelExpression of resource request="
          + labelExp
          + ". Queue labels="
          + (queueInfo.getAccessibleNodeLabels() == null ? "" : StringUtils.join(queueInfo
              .getAccessibleNodeLabels().iterator(), ',')));
    }
  }
}
 

public void subtract(Resource r) {
  memory -= r.getMemory();
  vcores -= r.getVirtualCores();
  gcores -= r.getGpuCores();
}
 

public void add(Resource r) {
  memory += r.getMemory();
  vcores += r.getVirtualCores();
  gcores += r.getGpuCores();
}
 

@SuppressWarnings({ "rawtypes", "unchecked" })
public AppInfo(ResourceManager rm, RMApp app, Boolean hasAccess,
    String schemePrefix) {
  this.schemePrefix = schemePrefix;
  if (app != null) {
    String trackingUrl = app.getTrackingUrl();
    this.state = app.createApplicationState();
    this.trackingUrlIsNotReady = trackingUrl == null || trackingUrl.isEmpty()
        || YarnApplicationState.NEW == this.state
        || YarnApplicationState.NEW_SAVING == this.state
        || YarnApplicationState.SUBMITTED == this.state
        || YarnApplicationState.ACCEPTED == this.state;
    this.trackingUI = this.trackingUrlIsNotReady ? "UNASSIGNED" : (app
        .getFinishTime() == 0 ? "ApplicationMaster" : "History");
    if (!trackingUrlIsNotReady) {
      this.trackingUrl =
          WebAppUtils.getURLWithScheme(schemePrefix,
              trackingUrl);
      this.trackingUrlPretty = this.trackingUrl;
    } else {
      this.trackingUrlPretty = "UNASSIGNED";
    }
    this.applicationId = app.getApplicationId();
    this.applicationType = app.getApplicationType();
    this.appIdNum = String.valueOf(app.getApplicationId().getId());
    this.id = app.getApplicationId().toString();
    this.user = app.getUser().toString();
    this.name = app.getName().toString();
    this.queue = app.getQueue().toString();
    this.progress = app.getProgress() * 100;
    this.diagnostics = app.getDiagnostics().toString();
    if (diagnostics == null || diagnostics.isEmpty()) {
      this.diagnostics = "";
    }
    if (app.getApplicationTags() != null && !app.getApplicationTags().isEmpty()) {
      this.applicationTags = Joiner.on(',').join(app.getApplicationTags());
    }
    this.finalStatus = app.getFinalApplicationStatus();
    this.clusterId = ResourceManager.getClusterTimeStamp();
    if (hasAccess) {
      this.startedTime = app.getStartTime();
      this.finishedTime = app.getFinishTime();
      this.elapsedTime = Times.elapsed(app.getStartTime(),
          app.getFinishTime());

      RMAppAttempt attempt = app.getCurrentAppAttempt();
      if (attempt != null) {
        Container masterContainer = attempt.getMasterContainer();
        if (masterContainer != null) {
          this.amContainerLogsExist = true;
          this.amContainerLogs = WebAppUtils.getRunningLogURL(
              schemePrefix + masterContainer.getNodeHttpAddress(),
              ConverterUtils.toString(masterContainer.getId()),
              app.getUser());
          this.amHostHttpAddress = masterContainer.getNodeHttpAddress();
        }
        
        ApplicationResourceUsageReport resourceReport = attempt
            .getApplicationResourceUsageReport();
        if (resourceReport != null) {
          Resource usedResources = resourceReport.getUsedResources();
          allocatedMB = usedResources.getMemory();
          allocatedVCores = usedResources.getVirtualCores();
          allocatedGCores = usedResources.getGpuCores();
          runningContainers = resourceReport.getNumUsedContainers();
        }
        resourceRequests =
            ((AbstractYarnScheduler) rm.getRMContext().getScheduler())
              .getPendingResourceRequestsForAttempt(attempt.getAppAttemptId());
      }
    }

    // copy preemption info fields
    RMAppMetrics appMetrics = app.getRMAppMetrics();
    numAMContainerPreempted =
        appMetrics.getNumAMContainersPreempted();
    preemptedResourceMB =
        appMetrics.getResourcePreempted().getMemory();
    numNonAMContainerPreempted =
        appMetrics.getNumNonAMContainersPreempted();
    preemptedResourceVCores =
        appMetrics.getResourcePreempted().getVirtualCores();
    preemptedResourceGCores =
        appMetrics.getResourcePreempted().getGpuCores();
    memorySeconds = appMetrics.getMemorySeconds();
    vcoreSeconds = appMetrics.getVcoreSeconds();
    gcoreSeconds = appMetrics.getGcoreSeconds();
  }
}
 

public static boolean fitsIn(Resource smaller, Resource bigger) {
  return smaller.getMemory() <= bigger.getMemory() &&
      smaller.getVirtualCores() <= bigger.getVirtualCores() &&
      smaller.getGpuCores() <= bigger.getGpuCores();
}