Java Code Examples for org.apache.hadoop.yarn.conf.YarnConfiguration#setDouble()

The following examples show how to use org.apache.hadoop.yarn.conf.YarnConfiguration#setDouble() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: SpliceTestYarnPlatform.java    From spliceengine with GNU Affero General Public License v3.0 5 votes vote down vote up 
private void configForTesting(String classPathRoot) throws URISyntaxException {
    yarnSiteConfigURL = Thread.currentThread().getContextClassLoader().getResource("yarn-site.xml");
    if (yarnSiteConfigURL == null) {
        throw new RuntimeException("Could not find 'yarn-site.xml' file in classpath");
    } else {
        LOG.info("Found 'yarn-site.xml' at "+ yarnSiteConfigURL.toURI().toString());
    }

    conf = new YarnConfiguration();
    conf.set(FileSystem.FS_DEFAULT_NAME_KEY, "file:///");

    keytab = classPathRoot.substring(0, classPathRoot.lastIndexOf('/'))+"/splice.keytab";
    if (secure) {
        conf.set("hadoop.security.authentication", "kerberos");
        conf.set("yarn.resourcemanager.principal", "yarn/[email protected]");
        conf.set("yarn.resourcemanager.keytab", keytab);
        conf.set("yarn.nodemanager.principal", "yarn/[email protected]");
        conf.set("yarn.nodemanager.keytab", keytab);
    }
    conf.setDouble("yarn.nodemanager.resource.io-spindles",2.0);
    conf.set("fs.default.name", "file:///");
    conf.set("yarn.nodemanager.container-executor.class","org.apache.hadoop.yarn.server.nodemanager.DefaultContainerExecutor");
    System.setProperty("zookeeper.sasl.client", "false");
    System.setProperty("zookeeper.sasl.serverconfig", "fake");

    conf.setInt(YarnConfiguration.RM_NM_HEARTBEAT_INTERVAL_MS, DEFAULT_HEARTBEAT_INTERVAL);
    conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 128);
    conf.setClass(YarnConfiguration.RM_SCHEDULER, FifoScheduler.class, ResourceScheduler.class);
    conf.set("yarn.application.classpath", new File(yarnSiteConfigURL.getPath()).getParent());
}
Example 2
Source File: YarnService.java    From dremio-oss with Apache License 2.0 4 votes vote down vote up 
@VisibleForTesting
protected String updateYarnConfiguration(Cluster cluster, YarnConfiguration yarnConfiguration) {
  String rmAddress = null;
  // make sure we set defaults first - before we overwrite it with props from ClusterConfig
  setYarnDefaults(cluster, yarnConfiguration);
  List<Property> keyValues = cluster.getClusterConfig().getSubPropertyList();
  if ( keyValues != null && !keyValues.isEmpty()) {
    for (Property property : keyValues) {
      yarnConfiguration.set(property.getKey(), property.getValue());
      if (RM_HOSTNAME.equalsIgnoreCase(property.getKey())) {
        rmAddress = property.getValue();
      }
    }
  }
  String queue = cluster.getClusterConfig().getClusterSpec().getQueue();
  if (queue != null && !queue.isEmpty()) {
    yarnConfiguration.set(DacDaemonYarnApplication.YARN_QUEUE_NAME, queue);
  }
  Integer memoryOnHeap = cluster.getClusterConfig().getClusterSpec().getMemoryMBOnHeap();
  Integer memoryOffHeap = cluster.getClusterConfig().getClusterSpec().getMemoryMBOffHeap();
  Preconditions.checkNotNull(memoryOnHeap, "Heap Memory is not specified");
  Preconditions.checkNotNull(memoryOffHeap, "Direct Memory is not specified");

  final int totalMemory = memoryOnHeap.intValue() + memoryOffHeap.intValue();
  // ratio between onheap and total memory. Since we need more direct memory
  // trying to make this ratio small, so heap would be as specified
  // setting this ratio based on onheap and offheap memory
  // so it will never go over the limit
  final double heapToDirectMemRatio = (double) (memoryOnHeap.intValue()) / totalMemory;

  yarnConfiguration.setDouble(HEAP_RESERVED_MIN_RATIO, Math.min(heapToDirectMemRatio, 0.1D));
  yarnConfiguration.setInt(DacDaemonYarnApplication.YARN_MEMORY_ON_HEAP, memoryOnHeap.intValue());
  yarnConfiguration.setInt(DacDaemonYarnApplication.YARN_MEMORY_OFF_HEAP, memoryOffHeap.intValue());
  yarnConfiguration.setInt(JAVA_RESERVED_MEMORY_MB, memoryOffHeap.intValue());

  Integer cpu = cluster.getClusterConfig().getClusterSpec().getVirtualCoreCount();
  if (cpu != null) {
    yarnConfiguration.setInt(DacDaemonYarnApplication.YARN_CPU, cpu.intValue());
  }
  Integer containerCount = cluster.getClusterConfig().getClusterSpec().getContainerCount();
  if (containerCount != null) {
    yarnConfiguration.setInt(DacDaemonYarnApplication.YARN_CONTAINER_COUNT, containerCount.intValue());
  }
  String clusterName = cluster.getClusterConfig().getName();
  if (clusterName != null) {
    yarnConfiguration.set(DacDaemonYarnApplication.YARN_APP_NAME, clusterName);
  }
  yarnConfiguration.set(YARN_CLUSTER_ID, cluster.getId().getId());

  return rmAddress;
}