org.apache.hadoop.yarn.util.ResourceCalculatorProcessTree Java Examples

The following examples show how to use org.apache.hadoop.yarn.util.ResourceCalculatorProcessTree. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: DAGAppMaster.java    From tez with Apache License 2.0 6 votes vote down vote up
private void initResourceCalculatorPlugins() {
  Class<? extends ResourceCalculatorProcessTree> clazz = amConf.getClass(
      TezConfiguration.TEZ_TASK_RESOURCE_CALCULATOR_PROCESS_TREE_CLASS,
      TezMxBeanResourceCalculator.class,
      ResourceCalculatorProcessTree.class);

  // this is set by YARN NM
  String pid = System.getenv().get("JVM_PID");
  // for the local debug test cases fallback to JVM hooks that are not portable
  if (pid == null || pid.length() == 0) {
    String processName = java.lang.management.ManagementFactory.getRuntimeMXBean().getName();
    pid = processName.split("@")[0];
  }
  cpuPlugin = ResourceCalculatorProcessTree.getResourceCalculatorProcessTree(pid, clazz, amConf);

  gcPlugin = new GcTimeUpdater(null);
}
 
Example #2
Source File: ContainersMonitorImpl.java    From hadoop with Apache License 2.0 6 votes vote down vote up
private boolean isEnabled() {
  if (resourceCalculatorPlugin == null) {
          LOG.info("ResourceCalculatorPlugin is unavailable on this system. "
              + this.getClass().getName() + " is disabled.");
          return false;
  }
  if (ResourceCalculatorProcessTree.getResourceCalculatorProcessTree("0", processTreeClass, conf) == null) {
      LOG.info("ResourceCalculatorProcessTree is unavailable on this system. "
              + this.getClass().getName() + " is disabled.");
          return false;
  }
  if (!(isPmemCheckEnabled() || isVmemCheckEnabled())) {
    LOG.info("Neither virutal-memory nor physical-memory monitoring is " +
        "needed. Not running the monitor-thread");
    return false;
  }

  return true;
}
 
Example #3
Source File: ContainersMonitorImpl.java    From big-c with Apache License 2.0 6 votes vote down vote up
private boolean isEnabled() {
  if (resourceCalculatorPlugin == null) {
          LOG.info("ResourceCalculatorPlugin is unavailable on this system. "
              + this.getClass().getName() + " is disabled.");
          return false;
  }
  if (ResourceCalculatorProcessTree.getResourceCalculatorProcessTree("0", processTreeClass, conf) == null) {
      LOG.info("ResourceCalculatorProcessTree is unavailable on this system. "
              + this.getClass().getName() + " is disabled.");
          return false;
  }
  if (!(isPmemCheckEnabled() || isVmemCheckEnabled())) {
    LOG.info("Neither virutal-memory nor physical-memory monitoring is " +
        "needed. Not running the monitor-thread");
    return false;
  }

  return true;
}
 
Example #4
Source File: ContainerReporter.java    From XLearning with Apache License 2.0 5 votes vote down vote up
public ProcessTreeInfo(ContainerId containerId, String pid,
                       ResourceCalculatorProcessTree pTree, long vmemLimit, long pmemLimit,
                       int cpuVcores) {
  this.containerId = containerId;
  this.pid = pid;
  this.pTree = pTree;
  this.vmemLimit = vmemLimit;
  this.pmemLimit = pmemLimit;
  this.cpuVcores = cpuVcores;
}
 
Example #5
Source File: TaskCounterUpdater.java    From tez with Apache License 2.0 5 votes vote down vote up
private void initResourceCalculatorPlugin() {
  Class<? extends ResourceCalculatorProcessTree> clazz = this.conf.getClass(
      TezConfiguration.TEZ_TASK_RESOURCE_CALCULATOR_PROCESS_TREE_CLASS,
      TezMxBeanResourceCalculator.class,
      ResourceCalculatorProcessTree.class); 

  pTree = ResourceCalculatorProcessTree.getResourceCalculatorProcessTree(pid, clazz, conf);

  LOG.info("Using ResourceCalculatorProcessTree : " + clazz.getName());
}
 
Example #6
Source File: TestTezMxBeanResourceCalculator.java    From tez with Apache License 2.0 5 votes vote down vote up
@Before
public void setup() throws Exception {
  Configuration conf = new TezConfiguration();
  conf.set(TezConfiguration.TEZ_TASK_RESOURCE_CALCULATOR_PROCESS_TREE_CLASS,
      TezMxBeanResourceCalculator.class.getName());

  Class<? extends ResourceCalculatorProcessTree> clazz = conf.getClass(
      TezConfiguration.TEZ_TASK_RESOURCE_CALCULATOR_PROCESS_TREE_CLASS, null,
      ResourceCalculatorProcessTree.class);
  resourceCalculator = ResourceCalculatorProcessTree.getResourceCalculatorProcessTree(
      "", clazz, conf);
}
 
Example #7
Source File: TaskCounterUpdater.java    From incubator-tez with Apache License 2.0 5 votes vote down vote up
private void initResourceCalculatorPlugin() {
  Class<? extends ResourceCalculatorProcessTree> clazz = this.conf.getClass(
      TezJobConfig.TEZ_RUNTIME_RESOURCE_CALCULATOR_PROCESS_TREE_CLASS, null,
      ResourceCalculatorProcessTree.class); 

  pTree = ResourceCalculatorProcessTree.getResourceCalculatorProcessTree(
      System.getenv().get("JVM_PID"), clazz, conf);

  LOG.info(" Using ResourceCalculatorProcessTree : " + pTree);
}
 
Example #8
Source File: Task.java    From big-c with Apache License 2.0 5 votes vote down vote up
/**
 * Update resource information counters
 */
void updateResourceCounters() {
  // Update generic resource counters
  updateHeapUsageCounter();

  // Updating resources specified in ResourceCalculatorProcessTree
  if (pTree == null) {
    return;
  }
  pTree.updateProcessTree();
  long cpuTime = pTree.getCumulativeCpuTime();
  long pMem = pTree.getRssMemorySize();
  long vMem = pTree.getVirtualMemorySize();
  // Remove the CPU time consumed previously by JVM reuse
  if (cpuTime != ResourceCalculatorProcessTree.UNAVAILABLE &&
      initCpuCumulativeTime != ResourceCalculatorProcessTree.UNAVAILABLE) {
    cpuTime -= initCpuCumulativeTime;
  }
  
  if (cpuTime != ResourceCalculatorProcessTree.UNAVAILABLE) {
    counters.findCounter(TaskCounter.CPU_MILLISECONDS).setValue(cpuTime);
  }
  
  if (pMem != ResourceCalculatorProcessTree.UNAVAILABLE) {
    counters.findCounter(TaskCounter.PHYSICAL_MEMORY_BYTES).setValue(pMem);
  }

  if (vMem != ResourceCalculatorProcessTree.UNAVAILABLE) {
    counters.findCounter(TaskCounter.VIRTUAL_MEMORY_BYTES).setValue(vMem);
  }
}
 
Example #9
Source File: ContainersMonitorImpl.java    From big-c with Apache License 2.0 5 votes vote down vote up
private String formatErrorMessage(String memTypeExceeded,
    long currentVmemUsage, long vmemLimit,
    long currentPmemUsage, long pmemLimit,
    String pId, ContainerId containerId, ResourceCalculatorProcessTree pTree) {
  return
    String.format("Container [pid=%s,containerID=%s] is running beyond %s memory limits. ",
        pId, containerId, memTypeExceeded) +
    "Current usage: " +
    formatUsageString(currentVmemUsage, vmemLimit,
                      currentPmemUsage, pmemLimit) +
    ". Killing container.\n" +
    "Dump of the process-tree for " + containerId + " :\n" +
    pTree.getProcessTreeDump();
}
 
Example #10
Source File: ContainersMonitorImpl.java    From big-c with Apache License 2.0 5 votes vote down vote up
boolean isProcessTreeOverLimit(ResourceCalculatorProcessTree pTree,
    String containerId, long limit) {
  long currentMemUsage = pTree.getVirtualMemorySize();
  // as processes begin with an age 1, we want to see if there are processes
  // more than 1 iteration old.
  long curMemUsageOfAgedProcesses = pTree.getVirtualMemorySize(1);
  return isProcessTreeOverLimit(containerId, currentMemUsage,
                                curMemUsageOfAgedProcesses, limit);
}
 
Example #11
Source File: ContainersMonitorImpl.java    From big-c with Apache License 2.0 5 votes vote down vote up
public ProcessTreeInfo(ContainerId containerId, String pid,
    ResourceCalculatorProcessTree pTree, long vmemLimit, long pmemLimit,
    int cpuVcores) {
  this.containerId = containerId;
  this.pid = pid;
  this.pTree = pTree;
  this.vmemLimit = vmemLimit;
  this.pmemLimit = pmemLimit;
  this.cpuVcores = cpuVcores;
}
 
Example #12
Source File: Task.java    From hadoop with Apache License 2.0 5 votes vote down vote up
/**
 * Update resource information counters
 */
void updateResourceCounters() {
  // Update generic resource counters
  updateHeapUsageCounter();

  // Updating resources specified in ResourceCalculatorProcessTree
  if (pTree == null) {
    return;
  }
  pTree.updateProcessTree();
  long cpuTime = pTree.getCumulativeCpuTime();
  long pMem = pTree.getRssMemorySize();
  long vMem = pTree.getVirtualMemorySize();
  // Remove the CPU time consumed previously by JVM reuse
  if (cpuTime != ResourceCalculatorProcessTree.UNAVAILABLE &&
      initCpuCumulativeTime != ResourceCalculatorProcessTree.UNAVAILABLE) {
    cpuTime -= initCpuCumulativeTime;
  }
  
  if (cpuTime != ResourceCalculatorProcessTree.UNAVAILABLE) {
    counters.findCounter(TaskCounter.CPU_MILLISECONDS).setValue(cpuTime);
  }
  
  if (pMem != ResourceCalculatorProcessTree.UNAVAILABLE) {
    counters.findCounter(TaskCounter.PHYSICAL_MEMORY_BYTES).setValue(pMem);
  }

  if (vMem != ResourceCalculatorProcessTree.UNAVAILABLE) {
    counters.findCounter(TaskCounter.VIRTUAL_MEMORY_BYTES).setValue(vMem);
  }
}
 
Example #13
Source File: ContainersMonitorImpl.java    From hadoop with Apache License 2.0 5 votes vote down vote up
private String formatErrorMessage(String memTypeExceeded,
    long currentVmemUsage, long vmemLimit,
    long currentPmemUsage, long pmemLimit,
    String pId, ContainerId containerId, ResourceCalculatorProcessTree pTree) {
  return
    String.format("Container [pid=%s,containerID=%s] is running beyond %s memory limits. ",
        pId, containerId, memTypeExceeded) +
    "Current usage: " +
    formatUsageString(currentVmemUsage, vmemLimit,
                      currentPmemUsage, pmemLimit) +
    ". Killing container.\n" +
    "Dump of the process-tree for " + containerId + " :\n" +
    pTree.getProcessTreeDump();
}
 
Example #14
Source File: ContainersMonitorImpl.java    From hadoop with Apache License 2.0 5 votes vote down vote up
boolean isProcessTreeOverLimit(ResourceCalculatorProcessTree pTree,
    String containerId, long limit) {
  long currentMemUsage = pTree.getVirtualMemorySize();
  // as processes begin with an age 1, we want to see if there are processes
  // more than 1 iteration old.
  long curMemUsageOfAgedProcesses = pTree.getVirtualMemorySize(1);
  return isProcessTreeOverLimit(containerId, currentMemUsage,
                                curMemUsageOfAgedProcesses, limit);
}
 
Example #15
Source File: ContainersMonitorImpl.java    From hadoop with Apache License 2.0 5 votes vote down vote up
public ProcessTreeInfo(ContainerId containerId, String pid,
    ResourceCalculatorProcessTree pTree, long vmemLimit, long pmemLimit,
    int cpuVcores, int gpuGcores) {
  this.containerId = containerId;
  this.pid = pid;
  this.pTree = pTree;
  this.vmemLimit = vmemLimit;
  this.pmemLimit = pmemLimit;
  this.cpuVcores = cpuVcores;
  this.gpuGcores = gpuGcores;
}
 
Example #16
Source File: ContainerReporter.java    From XLearning with Apache License 2.0 5 votes vote down vote up
public ContainerReporter(ApplicationContainerProtocol protocol, Configuration conf,
                         XLearningContainerId xlearningContainerId, String xlearningCmdProcessId) {
  this.protocol = protocol;
  this.conf = conf;
  this.containerId = xlearningContainerId;
  this.xlearningCmdProcessId = xlearningCmdProcessId;
  this.containerProcessId = null;
  this.processTreeClass = conf.getClass(YarnConfiguration.NM_CONTAINER_MON_PROCESS_TREE, null,
      ResourceCalculatorProcessTree.class);
  this.cpuMetrics = new ConcurrentHashMap<>();
  this.containerType = conf.get(XLearningConfiguration.XLEARNING_CONTAINER_TYPE,
      XLearningConfiguration.DEFAULT_XLEARNING_CONTAINER_TYPE);
}
 
Example #17
Source File: Task.java    From hadoop with Apache License 2.0 4 votes vote down vote up
public void initialize(JobConf job, JobID id, 
                       Reporter reporter,
                       boolean useNewApi) throws IOException, 
                                                 ClassNotFoundException,
                                                 InterruptedException {
  jobContext = new JobContextImpl(job, id, reporter);
  taskContext = new TaskAttemptContextImpl(job, taskId, reporter);
  if (getState() == TaskStatus.State.UNASSIGNED) {
    setState(TaskStatus.State.RUNNING);
  }
  if (useNewApi) {
    if (LOG.isDebugEnabled()) {
      LOG.debug("using new api for output committer");
    }
    outputFormat =
      ReflectionUtils.newInstance(taskContext.getOutputFormatClass(), job);
    committer = outputFormat.getOutputCommitter(taskContext);
  } else {
    committer = conf.getOutputCommitter();
  }
  Path outputPath = FileOutputFormat.getOutputPath(conf);
  if (outputPath != null) {
    if ((committer instanceof FileOutputCommitter)) {
      FileOutputFormat.setWorkOutputPath(conf, 
        ((FileOutputCommitter)committer).getTaskAttemptPath(taskContext));
    } else {
      FileOutputFormat.setWorkOutputPath(conf, outputPath);
    }
  }
  committer.setupTask(taskContext);
  Class<? extends ResourceCalculatorProcessTree> clazz =
      conf.getClass(MRConfig.RESOURCE_CALCULATOR_PROCESS_TREE,
          null, ResourceCalculatorProcessTree.class);
  pTree = ResourceCalculatorProcessTree
          .getResourceCalculatorProcessTree(System.getenv().get("JVM_PID"), clazz, conf);
  LOG.info(" Using ResourceCalculatorProcessTree : " + pTree);
  if (pTree != null) {
    pTree.updateProcessTree();
    initCpuCumulativeTime = pTree.getCumulativeCpuTime();
  }
}
 
Example #18
Source File: ContainersMonitorImpl.java    From big-c with Apache License 2.0 4 votes vote down vote up
@Override
protected void serviceInit(Configuration conf) throws Exception {
  this.monitoringInterval =
      conf.getLong(YarnConfiguration.NM_CONTAINER_MON_INTERVAL_MS,
          YarnConfiguration.DEFAULT_NM_CONTAINER_MON_INTERVAL_MS);

  Class<? extends ResourceCalculatorPlugin> clazz =
      conf.getClass(YarnConfiguration.NM_CONTAINER_MON_RESOURCE_CALCULATOR, null,
          ResourceCalculatorPlugin.class);
  this.resourceCalculatorPlugin =
      ResourceCalculatorPlugin.getResourceCalculatorPlugin(clazz, conf);
  LOG.info(" Using ResourceCalculatorPlugin : "
      + this.resourceCalculatorPlugin);
  processTreeClass = conf.getClass(YarnConfiguration.NM_CONTAINER_MON_PROCESS_TREE, null,
          ResourceCalculatorProcessTree.class);
  this.conf = conf;
  LOG.info(" Using ResourceCalculatorProcessTree : "
      + this.processTreeClass);

  this.containerMetricsEnabled =
      conf.getBoolean(YarnConfiguration.NM_CONTAINER_METRICS_ENABLE,
          YarnConfiguration.DEFAULT_NM_CONTAINER_METRICS_ENABLE);
  this.containerMetricsPeriodMs =
      conf.getLong(YarnConfiguration.NM_CONTAINER_METRICS_PERIOD_MS,
          YarnConfiguration.DEFAULT_NM_CONTAINER_METRICS_PERIOD_MS);

  long configuredPMemForContainers = conf.getLong(
      YarnConfiguration.NM_PMEM_MB,
      YarnConfiguration.DEFAULT_NM_PMEM_MB) * 1024 * 1024l;

  long configuredVCoresForContainers = conf.getLong(
      YarnConfiguration.NM_VCORES,
      YarnConfiguration.DEFAULT_NM_VCORES);


  // Setting these irrespective of whether checks are enabled. Required in
  // the UI.
  // ///////// Physical memory configuration //////
  this.maxPmemAllottedForContainers = configuredPMemForContainers;
  this.maxVCoresAllottedForContainers = configuredVCoresForContainers;

  // ///////// Virtual memory configuration //////
  float vmemRatio = conf.getFloat(YarnConfiguration.NM_VMEM_PMEM_RATIO,
      YarnConfiguration.DEFAULT_NM_VMEM_PMEM_RATIO);
  Preconditions.checkArgument(vmemRatio > 0.99f,
      YarnConfiguration.NM_VMEM_PMEM_RATIO + " should be at least 1.0");
  this.maxVmemAllottedForContainers =
      (long) (vmemRatio * configuredPMemForContainers);

  pmemCheckEnabled = conf.getBoolean(YarnConfiguration.NM_PMEM_CHECK_ENABLED,
      YarnConfiguration.DEFAULT_NM_PMEM_CHECK_ENABLED);
  vmemCheckEnabled = conf.getBoolean(YarnConfiguration.NM_VMEM_CHECK_ENABLED,
      YarnConfiguration.DEFAULT_NM_VMEM_CHECK_ENABLED);
  LOG.info("Physical memory check enabled: " + pmemCheckEnabled);
  LOG.info("Virtual memory check enabled: " + vmemCheckEnabled);

  nodeCpuPercentageForYARN =
      NodeManagerHardwareUtils.getNodeCpuPercentage(conf);

  if (pmemCheckEnabled) {
    // Logging if actual pmem cannot be determined.
    long totalPhysicalMemoryOnNM = UNKNOWN_MEMORY_LIMIT;
    if (this.resourceCalculatorPlugin != null) {
      totalPhysicalMemoryOnNM = this.resourceCalculatorPlugin
          .getPhysicalMemorySize();
      if (totalPhysicalMemoryOnNM <= 0) {
        LOG.warn("NodeManager's totalPmem could not be calculated. "
            + "Setting it to " + UNKNOWN_MEMORY_LIMIT);
        totalPhysicalMemoryOnNM = UNKNOWN_MEMORY_LIMIT;
      }
    }

    if (totalPhysicalMemoryOnNM != UNKNOWN_MEMORY_LIMIT &&
        this.maxPmemAllottedForContainers > totalPhysicalMemoryOnNM * 0.80f) {
      LOG.warn("NodeManager configured with "
          + TraditionalBinaryPrefix.long2String(maxPmemAllottedForContainers,
              "", 1)
          + " physical memory allocated to containers, which is more than "
          + "80% of the total physical memory available ("
          + TraditionalBinaryPrefix.long2String(totalPhysicalMemoryOnNM, "",
              1) + "). Thrashing might happen.");
    }
  }
  super.serviceInit(conf);
}
 
Example #19
Source File: ContainersMonitorImpl.java    From big-c with Apache License 2.0 4 votes vote down vote up
public ResourceCalculatorProcessTree getProcessTree() {
  return this.pTree;
}
 
Example #20
Source File: ContainersMonitorImpl.java    From big-c with Apache License 2.0 4 votes vote down vote up
public void setProcessTree(ResourceCalculatorProcessTree pTree) {
  this.pTree = pTree;
}
 
Example #21
Source File: ContainersMonitorImpl.java    From hadoop with Apache License 2.0 4 votes vote down vote up
public void setProcessTree(ResourceCalculatorProcessTree pTree) {
  this.pTree = pTree;
}
 
Example #22
Source File: ContainersMonitorImpl.java    From hadoop with Apache License 2.0 4 votes vote down vote up
public ResourceCalculatorProcessTree getProcessTree() {
  return this.pTree;
}
 
Example #23
Source File: Task.java    From big-c with Apache License 2.0 4 votes vote down vote up
public void initialize(JobConf job, JobID id, 
                       Reporter reporter,
                       boolean useNewApi) throws IOException, 
                                                 ClassNotFoundException,
                                                 InterruptedException {
  jobContext = new JobContextImpl(job, id, reporter);
  taskContext = new TaskAttemptContextImpl(job, taskId, reporter);
  if (getState() == TaskStatus.State.UNASSIGNED) {
    setState(TaskStatus.State.RUNNING);
  }
  if (useNewApi) {
    if (LOG.isDebugEnabled()) {
      LOG.debug("using new api for output committer");
    }
    outputFormat =
      ReflectionUtils.newInstance(taskContext.getOutputFormatClass(), job);
    committer = outputFormat.getOutputCommitter(taskContext);
  } else {
    committer = conf.getOutputCommitter();
  }
  Path outputPath = FileOutputFormat.getOutputPath(conf);
  if (outputPath != null) {
    if ((committer instanceof FileOutputCommitter)) {
      FileOutputFormat.setWorkOutputPath(conf, 
        ((FileOutputCommitter)committer).getTaskAttemptPath(taskContext));
    } else {
      FileOutputFormat.setWorkOutputPath(conf, outputPath);
    }
  }
  committer.setupTask(taskContext);
  Class<? extends ResourceCalculatorProcessTree> clazz =
      conf.getClass(MRConfig.RESOURCE_CALCULATOR_PROCESS_TREE,
          null, ResourceCalculatorProcessTree.class);
  pTree = ResourceCalculatorProcessTree
          .getResourceCalculatorProcessTree(System.getenv().get("JVM_PID"), clazz, conf);
  LOG.info(" Using ResourceCalculatorProcessTree : " + pTree);
  if (pTree != null) {
    pTree.updateProcessTree();
    initCpuCumulativeTime = pTree.getCumulativeCpuTime();
  }
}
 
Example #24
Source File: ContainersMonitorImpl.java    From hadoop with Apache License 2.0 4 votes vote down vote up
@Override
protected void serviceInit(Configuration conf) throws Exception {
  this.monitoringInterval =
      conf.getLong(YarnConfiguration.NM_CONTAINER_MON_INTERVAL_MS,
          YarnConfiguration.DEFAULT_NM_CONTAINER_MON_INTERVAL_MS);

  Class<? extends ResourceCalculatorPlugin> clazz =
      conf.getClass(YarnConfiguration.NM_CONTAINER_MON_RESOURCE_CALCULATOR, null,
          ResourceCalculatorPlugin.class);
  this.resourceCalculatorPlugin =
      ResourceCalculatorPlugin.getResourceCalculatorPlugin(clazz, conf);
  LOG.info(" Using ResourceCalculatorPlugin : "
      + this.resourceCalculatorPlugin);
  processTreeClass = conf.getClass(YarnConfiguration.NM_CONTAINER_MON_PROCESS_TREE, null,
          ResourceCalculatorProcessTree.class);
  this.conf = conf;
  LOG.info(" Using ResourceCalculatorProcessTree : "
      + this.processTreeClass);

  this.containerMetricsEnabled =
      conf.getBoolean(YarnConfiguration.NM_CONTAINER_METRICS_ENABLE,
          YarnConfiguration.DEFAULT_NM_CONTAINER_METRICS_ENABLE);
  this.containerMetricsPeriodMs =
      conf.getLong(YarnConfiguration.NM_CONTAINER_METRICS_PERIOD_MS,
          YarnConfiguration.DEFAULT_NM_CONTAINER_METRICS_PERIOD_MS);

  long configuredPMemForContainers = conf.getLong(
      YarnConfiguration.NM_PMEM_MB,
      YarnConfiguration.DEFAULT_NM_PMEM_MB) * 1024 * 1024l;

  long configuredVCoresForContainers = conf.getLong(
      YarnConfiguration.NM_VCORES,
      YarnConfiguration.DEFAULT_NM_VCORES);

  long configuredGCoresForContainers = conf.getLong(
      YarnConfiguration.NM_GCORES,
      YarnConfiguration.DEFAULT_NM_GCORES);

  // Setting these irrespective of whether checks are enabled. Required in
  // the UI.
  // ///////// Physical memory configuration //////
  this.maxPmemAllottedForContainers = configuredPMemForContainers;
  this.maxVCoresAllottedForContainers = configuredVCoresForContainers;
  this.maxGCoresAllottedForContainers = configuredGCoresForContainers;

  // ///////// Virtual memory configuration //////
  float vmemRatio = conf.getFloat(YarnConfiguration.NM_VMEM_PMEM_RATIO,
      YarnConfiguration.DEFAULT_NM_VMEM_PMEM_RATIO);
  Preconditions.checkArgument(vmemRatio > 0.99f,
      YarnConfiguration.NM_VMEM_PMEM_RATIO + " should be at least 1.0");
  this.maxVmemAllottedForContainers =
      (long) (vmemRatio * configuredPMemForContainers);

  pmemCheckEnabled = conf.getBoolean(YarnConfiguration.NM_PMEM_CHECK_ENABLED,
      YarnConfiguration.DEFAULT_NM_PMEM_CHECK_ENABLED);
  vmemCheckEnabled = conf.getBoolean(YarnConfiguration.NM_VMEM_CHECK_ENABLED,
      YarnConfiguration.DEFAULT_NM_VMEM_CHECK_ENABLED);
  LOG.info("Physical memory check enabled: " + pmemCheckEnabled);
  LOG.info("Virtual memory check enabled: " + vmemCheckEnabled);

  nodeCpuPercentageForYARN =
      NodeManagerHardwareUtils.getNodeCpuPercentage(conf);

  if (pmemCheckEnabled) {
    // Logging if actual pmem cannot be determined.
    long totalPhysicalMemoryOnNM = UNKNOWN_MEMORY_LIMIT;
    if (this.resourceCalculatorPlugin != null) {
      totalPhysicalMemoryOnNM = this.resourceCalculatorPlugin
          .getPhysicalMemorySize();
      if (totalPhysicalMemoryOnNM <= 0) {
        LOG.warn("NodeManager's totalPmem could not be calculated. "
            + "Setting it to " + UNKNOWN_MEMORY_LIMIT);
        totalPhysicalMemoryOnNM = UNKNOWN_MEMORY_LIMIT;
      }
    }

    if (totalPhysicalMemoryOnNM != UNKNOWN_MEMORY_LIMIT &&
        this.maxPmemAllottedForContainers > totalPhysicalMemoryOnNM * 0.80f) {
      LOG.warn("NodeManager configured with "
          + TraditionalBinaryPrefix.long2String(maxPmemAllottedForContainers,
              "", 1)
          + " physical memory allocated to containers, which is more than "
          + "80% of the total physical memory available ("
          + TraditionalBinaryPrefix.long2String(totalPhysicalMemoryOnNM, "",
              1) + "). Thrashing might happen.");
    }
  }
  super.serviceInit(conf);
}
 
Example #25
Source File: ContainerReporter.java    From XLearning with Apache License 2.0 4 votes vote down vote up
public void setProcessTree(ResourceCalculatorProcessTree pTree) {
  this.pTree = pTree;
}
 
Example #26
Source File: ContainerReporter.java    From XLearning with Apache License 2.0 4 votes vote down vote up
public ResourceCalculatorProcessTree getProcessTree() {
  return this.pTree;
}