org.apache.hadoop.yarn.api.ApplicationConstants.Environment Java Examples

/**
 * Makes MDC properties
 */
public static void setupMDC(String service)
{
  MDC.put("apex.service", service);

  String value = StramClientUtils.getHostName();
  MDC.put("apex.node", value == null ? "unknown" : value);

  value = System.getenv(Environment.USER.key());
  if (value != null) {
    MDC.put("apex.user", value);
  }

  value = System.getenv(Environment.CONTAINER_ID.name());
  if (value != null) {
    ContainerId containerId = ConverterUtils.toContainerId(value);
    ApplicationId applicationId = containerId.getApplicationAttemptId().getApplicationId();
    MDC.put("apex.containerId", containerId.toString());
    MDC.put("apex.applicationId", applicationId.toString());
  }

  value = System.getProperty(APPLICATION_NAME.getLongName());
  if (value != null) {
    MDC.put("apex.application", value);
  }
}
 

private void testCommonEnvSettingsForMRTasks(Map<String, String> env) {
    Assert.assertTrue(env.containsKey("foo"));
    Assert.assertTrue(env.containsKey("bar"));
    Assert.assertTrue(env.containsKey(Environment.LD_LIBRARY_PATH.name()));
    Assert.assertTrue(env.containsKey(Environment.SHELL.name()));
    Assert.assertTrue(env.containsKey("HADOOP_ROOT_LOGGER"));
    Assert.assertEquals("$PWD:$TEZ_ADMIN_ENV_TEST/lib/native",
        env.get(Environment.LD_LIBRARY_PATH.name()));

//    TEZ-273 will reinstate this or similar. 
//    for (String val : YarnConfiguration.DEFAULT_YARN_APPLICATION_CLASSPATH) {
//      Assert.assertTrue(env.get(Environment.CLASSPATH.name()).contains(val));
//    }
//    Assert.assertTrue(0 ==
//        env.get(Environment.CLASSPATH.name()).indexOf(Environment.PWD.$()));
  }
 

/**
 * Setup
 * 
 * @param containerTask
 *          the new task specification. Must be a valid task
 * @param childUGI
 *          the old UGI instance being used
 * @return
 */
UserGroupInformation handleNewTaskCredentials(ContainerTask containerTask,
    UserGroupInformation childUGI) {
  // Re-use the UGI only if the Credentials have not changed.
  Preconditions.checkState(containerTask.shouldDie() != true);
  Preconditions.checkState(containerTask.getTaskSpec() != null);
  if (containerTask.haveCredentialsChanged()) {
    LOG.info("Refreshing UGI since Credentials have changed");
    Credentials taskCreds = containerTask.getCredentials();
    if (taskCreds != null) {
      LOG.info("Credentials : #Tokens=" + taskCreds.numberOfTokens() + ", #SecretKeys="
          + taskCreds.numberOfSecretKeys());
      childUGI = UserGroupInformation.createRemoteUser(System
          .getenv(ApplicationConstants.Environment.USER.toString()));
      childUGI.addCredentials(containerTask.getCredentials());
    } else {
      LOG.info("Not loading any credentials, since no credentials provided");
    }
  }
  return childUGI;
}
 

public static String getFrameworkClasspath(Configuration conf) {
  Map<String, String> environment = new HashMap<String, String>();

  TezYARNUtils.addToEnvironment(environment,
      Environment.CLASSPATH.name(),
      Environment.PWD.$(),
      File.pathSeparator);

  TezYARNUtils.addToEnvironment(environment,
      Environment.CLASSPATH.name(),
      Environment.PWD.$() + File.separator + "*",
      File.pathSeparator);

  // Add YARN/COMMON/HDFS jars and conf locations to path
  for (String c : conf.getStrings(
      YarnConfiguration.YARN_APPLICATION_CLASSPATH,
      YarnConfiguration.DEFAULT_YARN_APPLICATION_CLASSPATH)) {
    TezYARNUtils.addToEnvironment(environment, Environment.CLASSPATH.name(),
        c.trim(), File.pathSeparator);
  }
  return StringInterner.weakIntern(environment.get(Environment.CLASSPATH.name()));
}
 

public static void setupDefaultEnv(Map<String, String> env,
    Configuration conf,  String confEnvKey, String confEnvKeyDefault) {
  // Setup the CLASSPATH in environment
  // i.e. add { Hadoop jars, job jar, CWD } to classpath.
  String classpath = getFrameworkClasspath(conf);
  TezYARNUtils.addToEnvironment(env,
      ApplicationConstants.Environment.CLASSPATH.name(),
      classpath, File.pathSeparator);

  // set any env from config if it is not set already
  TezYARNUtils.setEnvIfAbsentFromInputString(env, conf.get(
      confEnvKey, confEnvKeyDefault));
  
  // Append pwd to LD_LIBRARY_PATH
  // Done separately here because this is known to work platform independent
  TezYARNUtils.addToEnvironment(env, Environment.LD_LIBRARY_PATH.name(),
      Environment.PWD.$(), File.pathSeparator);
}
 

/**
 * Creates a {@link ApplicationClassLoader} if
 * {@link MRJobConfig#MAPREDUCE_JOB_CLASSLOADER} is set to true, and
 * the APP_CLASSPATH environment variable is set.
 * @param conf
 * @return the created job classloader, or null if the job classloader is not
 * enabled or the APP_CLASSPATH environment variable is not set
 * @throws IOException
 */
public static ClassLoader createJobClassLoader(Configuration conf)
    throws IOException {
  ClassLoader jobClassLoader = null;
  if (conf.getBoolean(MRJobConfig.MAPREDUCE_JOB_CLASSLOADER, false)) {
    String appClasspath = System.getenv(Environment.APP_CLASSPATH.key());
    if (appClasspath == null) {
      LOG.warn("Not creating job classloader since APP_CLASSPATH is not set.");
    } else {
      LOG.info("Creating job classloader");
      if (LOG.isDebugEnabled()) {
        LOG.debug("APP_CLASSPATH=" + appClasspath);
      }
      String[] systemClasses = getSystemClasses(conf);
      jobClassLoader = createJobClassLoader(appClasspath,
          systemClasses);
    }
  }
  return jobClassLoader;
}
 

private void setupAppMasterEnv(Map<String, String> appMasterEnv) {
  StringBuilder classPathEnv = new StringBuilder();
  classPathEnv.append(Environment.CLASSPATH.$()).append(File.pathSeparatorChar);
  classPathEnv.append("./*");

  for (String c : conf.getStrings(
      YarnConfiguration.YARN_APPLICATION_CLASSPATH,
      YarnConfiguration.DEFAULT_YARN_APPLICATION_CLASSPATH)) {
    classPathEnv.append(File.pathSeparatorChar);
    classPathEnv.append(c.trim());
  }

  String envStr = classPathEnv.toString();
  LOG.info("env: " + envStr);
  appMasterEnv.put(Environment.CLASSPATH.name(), envStr);
}
 

public void startContainer(Container container, VMConfig appVMConfig) throws YarnException, IOException {
  String command = appVMConfig.buildCommand();
  ContainerLaunchContext ctx = Records.newRecord(ContainerLaunchContext.class);
  if(vmConfig.getVmResources().size() > 0) {
    appVMConfig.getVmResources().putAll(vmConfig.getVmResources());
    VMResources vmResources = new VMResources(conf, appVMConfig);
    ctx.setLocalResources(vmResources);
  }
  Map<String, String> appEnv = new HashMap<String, String>();
  boolean miniClusterEnv = vmConfig.getEnvironment() == VMConfig.Environment.YARN_MINICLUSTER;
  setupAppClasspath(miniClusterEnv , conf, appEnv);
  ctx.setEnvironment(appEnv);
  
  StringBuilder sb = new StringBuilder();
  List<String> commands = Collections.singletonList(
      sb.append(command).
      append(" 1> ").append(ApplicationConstants.LOG_DIR_EXPANSION_VAR).append("/stdout").
      append(" 2> ").append(ApplicationConstants.LOG_DIR_EXPANSION_VAR).append("/stderr").toString()
  );
  ctx.setCommands(commands);
  nmClient.startContainer(container, ctx);
  //TODO: update vm descriptor status
}
 

void setupAppClasspath(boolean miniClusterEnv, Configuration conf, Map<String, String> appMasterEnv) {
  if(miniClusterEnv) {
    String cps = System.getProperty("java.class.path") ;
    String[] cp = cps.split(":") ;
    for(String selCp : cp) {
      Apps.addToEnvironment(appMasterEnv, Environment.CLASSPATH.name(), selCp, ":");
    }
  } else {
    StringBuilder classPathEnv = new StringBuilder();
    classPathEnv.append(Environment.CLASSPATH.$()).append(File.pathSeparatorChar);
    classPathEnv.append("./*");

    String[] classpath = conf.getStrings(
        YarnConfiguration.YARN_APPLICATION_CLASSPATH,
        YarnConfiguration.DEFAULT_YARN_APPLICATION_CLASSPATH
    ) ;
    for (String selClasspath : classpath) {
      classPathEnv.append(File.pathSeparatorChar);
      classPathEnv.append(selClasspath.trim());
    }
    appMasterEnv.put(Environment.CLASSPATH.name(), classPathEnv.toString());
    System.err.println("CLASSPATH: " + classPathEnv);
  }
  Apps.addToEnvironment(appMasterEnv, Environment.CLASSPATH.name(), Environment.PWD.$() + File.separator + "*", ":");
}
 

private void testCommonEnvSettingsForMRTasks(Map<String, String> env) {
    Assert.assertTrue(env.containsKey("foo"));
    Assert.assertTrue(env.containsKey("bar"));
    Assert.assertTrue(env.containsKey(Environment.LD_LIBRARY_PATH.name()));
    Assert.assertTrue(env.containsKey(Environment.SHELL.name()));
    Assert.assertTrue(env.containsKey("HADOOP_ROOT_LOGGER"));

    /* On non-windows platform ensure that LD_LIBRARY_PATH is being set and PWD is present.
     * on windows platform LD_LIBRARY_PATH is not applicable. check the PATH is being appended
     * by the user setting (ex user may set HADOOP_HOME\\bin.
     */
    if (!Shell.WINDOWS) {
      Assert.assertEquals("$PWD:$TEZ_ADMIN_ENV_TEST/lib/native",
          env.get(Environment.LD_LIBRARY_PATH.name()));
    } else {
      Assert.assertTrue(env.get(Environment.PATH.name()).contains(";%TEZ_ADMIN_ENV%\\bin"));
    }

//    TEZ-273 will reinstate this or similar. 
//    for (String val : YarnConfiguration.DEFAULT_YARN_APPLICATION_CLASSPATH) {
//      Assert.assertTrue(env.get(Environment.CLASSPATH.name()).contains(val));
//    }
//    Assert.assertTrue(0 ==
//        env.get(Environment.CLASSPATH.name()).indexOf(Environment.PWD.$()));
  }
 

private TezChild createTezChild(Configuration defaultConf, ContainerId containerId,
                                String tokenIdentifier, int attemptNumber, String[] localDirs,
                                TezTaskUmbilicalProtocol tezTaskUmbilicalProtocol,
                                Credentials credentials) throws
    InterruptedException, TezException, IOException {
  Map<String, String> containerEnv = new HashMap<String, String>();
  containerEnv.putAll(localEnv);
  // Use the user from env if it's available.
  String user = isLocalMode ? System.getenv(Environment.USER.name()) : context.getUser();
  containerEnv.put(Environment.USER.name(), user);

  long memAvailable;
  synchronized (this) { // needed to fix findbugs Inconsistent synchronization warning
    memAvailable = Runtime.getRuntime().maxMemory() / numExecutors;
  }
  TezChild tezChild =
      TezChild.newTezChild(defaultConf, null, 0, containerId.toString(), tokenIdentifier,
          attemptNumber, localDirs, workingDirectory, containerEnv, "", executionContext, credentials,
          memAvailable, context.getUser(), tezTaskUmbilicalProtocol, false,
          context.getHadoopShim());
  return tezChild;
}
 

/**
 * Creates a {@link ApplicationClassLoader} if
 * {@link MRJobConfig#MAPREDUCE_JOB_CLASSLOADER} is set to true, and
 * the APP_CLASSPATH environment variable is set.
 * @param conf
 * @return the created job classloader, or null if the job classloader is not
 * enabled or the APP_CLASSPATH environment variable is not set
 * @throws IOException
 */
public static ClassLoader createJobClassLoader(Configuration conf)
    throws IOException {
  ClassLoader jobClassLoader = null;
  if (conf.getBoolean(MRJobConfig.MAPREDUCE_JOB_CLASSLOADER, false)) {
    String appClasspath = System.getenv(Environment.APP_CLASSPATH.key());
    if (appClasspath == null) {
      LOG.warn("Not creating job classloader since APP_CLASSPATH is not set.");
    } else {
      LOG.info("Creating job classloader");
      if (LOG.isDebugEnabled()) {
        LOG.debug("APP_CLASSPATH=" + appClasspath);
      }
      String[] systemClasses = getSystemClasses(conf);
      jobClassLoader = createJobClassLoader(appClasspath,
          systemClasses);
    }
  }
  return jobClassLoader;
}
 

@Private
@VisibleForTesting
static String constructAMLaunchOpts(TezConfiguration tezConf, Resource capability) {
  String defaultOpts = tezConf.get(TezConfiguration.TEZ_AM_LAUNCH_CLUSTER_DEFAULT_CMD_OPTS,
      TezConfiguration.TEZ_AM_LAUNCH_CLUSTER_DEFAULT_CMD_OPTS_DEFAULT);
  Path tmpDir = new Path(Environment.PWD.$(),
      YarnConfiguration.DEFAULT_CONTAINER_TEMP_DIR);
  String amOpts = "-Djava.io.tmpdir=" + tmpDir + " ";

  if (defaultOpts != null && !defaultOpts.isEmpty()) {
    amOpts = amOpts + defaultOpts + " ";
  }
  amOpts = amOpts + tezConf.get(TezConfiguration.TEZ_AM_LAUNCH_CMD_OPTS,
      TezConfiguration.TEZ_AM_LAUNCH_CMD_OPTS_DEFAULT);

  amOpts = maybeAddDefaultMemoryJavaOpts(amOpts, capability,
      tezConf.getDouble(TezConfiguration.TEZ_CONTAINER_MAX_JAVA_HEAP_FRACTION,
          TezConfiguration.TEZ_CONTAINER_MAX_JAVA_HEAP_FRACTION_DEFAULT));

  return amOpts;
}
 

public static void setupDefaultEnv(Map<String, String> env, Configuration conf,  String userEnvKey, String userEnvDefault,
    String clusterDefaultEnvKey, String clusterDefaultEnvDefault, boolean usingArchive) {
  // Setup the CLASSPATH in environment
  // i.e. add { Hadoop jars, job jar, CWD } to classpath.
  String classpath = getFrameworkClasspath(conf, usingArchive);
  TezYARNUtils.addToEnvironment(env,
      ApplicationConstants.Environment.CLASSPATH.name(),
      classpath, File.pathSeparator);

  // Pre-pend pwd to LD_LIBRARY_PATH
  // Done separately here because this is known to work platform
  // independent
  TezYARNUtils.addToEnvironment(env,
      Environment.LD_LIBRARY_PATH.name(), Environment.PWD.$(), File.pathSeparator);
  TezYARNUtils.appendToEnvFromInputString(env,
      conf.get(userEnvKey, userEnvDefault), File.pathSeparator);
  // set any env from config if it is not set already
  TezYARNUtils.appendToEnvFromInputString(env,
      conf.get(clusterDefaultEnvKey, clusterDefaultEnvDefault), File.pathSeparator);
}
 

/**
 * Makes MDC properties
 */
public static void setupMDC(String service)
{
  MDC.put("apex.service", service);

  String value = StramClientUtils.getHostName();
  MDC.put("apex.node", value == null ? "unknown" : value);

  value = System.getenv(Environment.USER.key());
  if (value != null) {
    MDC.put("apex.user", value);
  }

  value = System.getenv(Environment.CONTAINER_ID.name());
  if (value != null) {
    ContainerId containerId = ConverterUtils.toContainerId(value);
    ApplicationId applicationId = containerId.getApplicationAttemptId().getApplicationId();
    MDC.put("apex.containerId", containerId.toString());
    MDC.put("apex.applicationId", applicationId.toString());
  }

  value = System.getProperty(APPLICATION_NAME.getLongName());
  if (value != null) {
    MDC.put("apex.application", value);
  }
}
 

@Test(timeout = 5000)
public void testNoHadoopConfInClasspath() {
  Configuration conf = new Configuration(false);
  conf.setBoolean(TezConfiguration.TEZ_CLASSPATH_ADD_HADOOP_CONF, true);
  String classpath = TezYARNUtils.getFrameworkClasspath(conf, true);
  Assert.assertTrue(classpath.contains(Environment.PWD.$()));
  Assert.assertTrue(classpath.contains(Environment.PWD.$() + File.separator + "*"));
  Assert.assertTrue(classpath.contains(TezConstants.TEZ_TAR_LR_NAME + File.separator + "*"));
  Assert.assertTrue(classpath.contains(TezConstants.TEZ_TAR_LR_NAME + File.separator
      + "lib" + File.separator + "*"));
  Assert.assertTrue(classpath.contains(Environment.HADOOP_CONF_DIR.$()));
  Assert.assertTrue(classpath.indexOf(Environment.PWD.$()) <
      classpath.indexOf(TezConstants.TEZ_TAR_LR_NAME));
  Assert.assertTrue(classpath.indexOf(TezConstants.TEZ_TAR_LR_NAME) <
      classpath.indexOf(Environment.HADOOP_CONF_DIR.$()));
}
 

@Test(timeout = 5000)
public void testSetupDefaultEnvironment() {
  Configuration conf = new Configuration(false);
  conf.set(TezConfiguration.TEZ_AM_LAUNCH_ENV, "LD_LIBRARY_PATH=USER_PATH,USER_KEY=USER_VALUE");
  conf.set(TezConfiguration.TEZ_AM_LAUNCH_CLUSTER_DEFAULT_ENV, "LD_LIBRARY_PATH=DEFAULT_PATH,DEFAULT_KEY=DEFAULT_VALUE");

  Map<String, String> environment = new TreeMap<String, String>();
  TezYARNUtils.setupDefaultEnv(environment, conf,
      TezConfiguration.TEZ_AM_LAUNCH_ENV,
      TezConfiguration.TEZ_AM_LAUNCH_ENV_DEFAULT,
      TezConfiguration.TEZ_AM_LAUNCH_CLUSTER_DEFAULT_ENV,
      TezConfiguration.TEZ_AM_LAUNCH_CLUSTER_DEFAULT_ENV_DEFAULT, false);

  String value1 = environment.get("USER_KEY");
  Assert.assertEquals("User env should merge with default env", "USER_VALUE", value1);
  String value2 = environment.get("DEFAULT_KEY");
  Assert.assertEquals("User env should merge with default env", "DEFAULT_VALUE", value2);
  String value3 = environment.get("LD_LIBRARY_PATH");
  Assert.assertEquals("User env should append default env",
      Environment.PWD.$() + File.pathSeparator + "USER_PATH" + File.pathSeparator + "DEFAULT_PATH", value3);
  }
 

private static void addUserSpecifiedClasspath(StringBuilder classpathBuilder,
    Configuration conf) {
  // Add any additional user-specified classpath
  String additionalClasspath = conf.get(TezConfiguration.TEZ_CLUSTER_ADDITIONAL_CLASSPATH_PREFIX);
  if (additionalClasspath != null && !additionalClasspath.trim().isEmpty()) {
    classpathBuilder.append(additionalClasspath)
        .append(File.pathSeparator);
  }

  // Add PWD:PWD/*
  classpathBuilder.append(Environment.PWD.$())
      .append(File.pathSeparator)
      .append(Environment.PWD.$() + File.separator + "*")
      .append(File.pathSeparator);
}
 

@Test(timeout = 20000)
public void testUserClasspathFirstFalse() {
  Configuration conf = new Configuration(false);
  conf.setBoolean(TezConfiguration.TEZ_USER_CLASSPATH_FIRST, false);
  conf.set(TezConfiguration.TEZ_CLUSTER_ADDITIONAL_CLASSPATH_PREFIX, "foobar");
  String classpath = TezYARNUtils.getFrameworkClasspath(conf, true);
  Assert.assertTrue(classpath.contains("foobar"));
  Assert.assertTrue(classpath.indexOf("foobar") >
      classpath.indexOf(TezConstants.TEZ_TAR_LR_NAME));
  Assert.assertTrue(classpath.indexOf("foobar") >
      classpath.indexOf(Environment.PWD.$()));
}
 

@Test(timeout = 5000)
public void testAuxClasspath() {
  Configuration conf = new Configuration(false);
  conf.set(TezConfiguration.TEZ_CLUSTER_ADDITIONAL_CLASSPATH_PREFIX, "foobar");
  String classpath = TezYARNUtils.getFrameworkClasspath(conf, true);
  Assert.assertTrue(classpath.contains("foobar"));
  Assert.assertTrue(classpath.indexOf("foobar") <
      classpath.indexOf(TezConstants.TEZ_TAR_LR_NAME));
  Assert.assertTrue(classpath.indexOf("foobar") <
      classpath.indexOf(Environment.PWD.$()));
}
 

@Test(timeout = 5000)
public void testBasicArchiveClasspath() {
  Configuration conf = new Configuration(false);
  String classpath = TezYARNUtils.getFrameworkClasspath(conf, true);
  Assert.assertTrue(classpath.contains(Environment.PWD.$()));
  Assert.assertTrue(classpath.contains(Environment.PWD.$() + File.separator + "*"));
  Assert.assertTrue(classpath.contains(TezConstants.TEZ_TAR_LR_NAME + File.separator + "*"));
  Assert.assertTrue(classpath.contains(TezConstants.TEZ_TAR_LR_NAME + File.separator
      + "lib" + File.separator + "*"));
  Assert.assertTrue(!classpath.contains(Environment.HADOOP_CONF_DIR.$()));
  Assert.assertTrue(classpath.indexOf(Environment.PWD.$()) <
      classpath.indexOf(TezConstants.TEZ_TAR_LR_NAME));
}
 

private String getAppMasterCommand() {
  StringBuilder sb = new StringBuilder();
  sb.append(Environment.JAVA_HOME.$()).append("/bin/java").append(" ");
  sb.append("-Xmx").append(this.applicationMasterMem).append("M").append(" ");
  sb.append(this.applicationMasterClassName).append(" ");
  sb.append("--").append(Constants.OPT_CONTAINER_MEM).append(" ").append(this.containerMem)
      .append(" ");
  sb.append("--").append(Constants.OPT_KAFKA_SEED_BROKERS).append(" ")
      .append(StringUtils.join(this.kafkaSeedBrokers, " ")).append(" ");
  sb.append("--").append(Constants.OPT_KAFKA_TOPIC).append(" ")
      .append(StringUtils.join(this.topicList, " ")).append(" ");
  sb.append(" --").append(Constants.OPT_YARN_SITE_XML).append(" ").append(this.yarnSiteXml);
  sb.append(" --").append(Constants.OPT_PRE_COMMIT_PATH_PREFIX).append(" ").append(this.preCommitPrefix);
  sb.append(" --").append(Constants.OPT_COMMIT_PATH_PREFIX).append(" ").append(this.commitPrefix);
  sb.append(" --").append(Constants.OPT_CHECK_POINT_INTERVAL).append(" ").append(Long.toString(this.commitCheckPointInterval));
  
  if(this.hdfsPath != null){
    sb.append(" --").append(Constants.OPT_HDFS_PATH).append(" ").append(this.hdfsPath);
  }
  if(this.cleanstart){
    sb.append(" --").append(Constants.OPT_CLEAN_START);
  }
  
  
  sb.append(" 1> ").append(ApplicationConstants.LOG_DIR_EXPANSION_VAR).append("/stdout")
      .append(" ");
  sb.append(" 2> ").append(ApplicationConstants.LOG_DIR_EXPANSION_VAR).append("/stderr");
  String r = sb.toString();
  LOG.info("ApplicationConstants.getCommand() : " + r);
  return r;
}
 

/**
 * Setup classpath and other environment variables
 * @param conf Configuration to retrieve settings from
 * @param environment Environment to update
 * @param isMap Whether task is a map or reduce task
 */
public static void updateEnvironmentForMRTasks(Configuration conf,
    Map<String, String> environment, boolean isMap) {
  // Shell
  environment.put(Environment.SHELL.name(), conf.get(
      MRJobConfig.MAPRED_ADMIN_USER_SHELL, MRJobConfig.DEFAULT_SHELL));

  // Add pwd to LD_LIBRARY_PATH, add this before adding anything else
  TezYARNUtils.addToEnvironment(environment, Environment.LD_LIBRARY_PATH.name(),
      Environment.PWD.$(), File.pathSeparator);

  // Add the env variables passed by the admin
  TezYARNUtils.appendToEnvFromInputString(environment, conf.get(
      MRJobConfig.MAPRED_ADMIN_USER_ENV,
      MRJobConfig.DEFAULT_MAPRED_ADMIN_USER_ENV),
      File.pathSeparator);

  // Add the env variables passed by the user
  String mapredChildEnv = (isMap ?
      conf.get(MRJobConfig.MAP_ENV, "")
      : conf.get(MRJobConfig.REDUCE_ENV, ""));
  TezYARNUtils.appendToEnvFromInputString(environment, mapredChildEnv, File.pathSeparator);

  // Set logging level in the environment.
  environment.put(
      "HADOOP_ROOT_LOGGER",
      getChildLogLevel(conf, isMap) + ",CLA");
}
 

@SuppressWarnings("deprecation")
private void setupAppMasterEnv(Map<String, String> appMasterEnv) {
    for (String c : conf.getStrings(
            YarnConfiguration.YARN_APPLICATION_CLASSPATH,
            YarnConfiguration.DEFAULT_YARN_APPLICATION_CLASSPATH)) {
        Apps.addToEnvironment(appMasterEnv, Environment.CLASSPATH.name(),
                c.trim());
    }
    Apps.addToEnvironment(appMasterEnv,
            Environment.CLASSPATH.name(),
            Environment.PWD.$() + File.separator + "*");
}
 

public static String getContainerLogDir() {
  String logDirsStr  = System.getenv(Environment.LOG_DIRS.name());
  if (logDirsStr == null || logDirsStr.isEmpty()) {
    return null;
  }
  String[] logDirs = StringUtils.split(logDirsStr, ',');
  if (logDirs.length == 0) {
    return null;
  }
  int logIndex = RANDOM.nextInt(logDirs.length);
  return logDirs[logIndex];
}
 

protected void setupAppMasterEnv(CommandLine cli, Map<String, String> appMasterEnv) throws IOException {
  for (String c : conf.getStrings(
          YarnConfiguration.YARN_APPLICATION_CLASSPATH,
          YarnConfiguration.DEFAULT_YARN_APPLICATION_CLASSPATH)) {
    Apps.addToEnvironment(appMasterEnv, Environment.CLASSPATH.name(), c.trim());
  }
  Apps.addToEnvironment(appMasterEnv,
          Environment.CLASSPATH.name(),
          Environment.PWD.$() + File.separator + "*");

  if (cli.hasOption("extclasspath")) {
    String extclasspathArg = cli.getOptionValue("extclasspath");
    File extclasspathFile = new File(extclasspathArg);
    StringBuilder sb = new StringBuilder();
    BufferedReader reader = null;
    String line = null;
    try {
      reader = new BufferedReader(
        new InputStreamReader(new FileInputStream(extclasspathFile), Charset.forName("UTF-8")));
      while ((line = reader.readLine()) != null)
        sb.append(line.trim()).append(":");
    } finally {
      if (reader != null) {
        reader.close();
      }
    }
    for (String part : sb.toString().split(":")) {
      if (part.length() > 0)
        Apps.addToEnvironment(appMasterEnv, Environment.CLASSPATH.name(), part);
    }
  }
}
 

@VisibleForTesting
static void setupConfigurationAndInstallSecurityContext(Configuration configuration, String currDir, Map<String, String> variables) throws Exception {
	final String localDirs = variables.get(Environment.LOCAL_DIRS.key());
	LOG.info("Current working/local Directory: {}", localDirs);

	BootstrapTools.updateTmpDirectoriesInConfiguration(configuration, localDirs);

	setupConfigurationFromVariables(configuration, currDir, variables);

	SecurityUtils.install(new SecurityConfiguration(configuration));
}
 

private Map<String, String> getAMEnvironment(Map<String, LocalResource> localResources
    , FileSystem fs) throws IOException{
  Map<String, String> env = new HashMap<String, String>();

  // Set ApplicationMaster jar file
  LocalResource appJarResource = localResources.get(Constants.AM_JAR_NAME);
  Path hdfsAppJarPath = new Path(fs.getHomeDirectory(), appJarResource.getResource().getFile());
  FileStatus hdfsAppJarStatus = fs.getFileStatus(hdfsAppJarPath);
  long hdfsAppJarLength = hdfsAppJarStatus.getLen();
  long hdfsAppJarTimestamp = hdfsAppJarStatus.getModificationTime();

  env.put(Constants.AM_JAR_PATH, hdfsAppJarPath.toString());
  env.put(Constants.AM_JAR_TIMESTAMP, Long.toString(hdfsAppJarTimestamp));
  env.put(Constants.AM_JAR_LENGTH, Long.toString(hdfsAppJarLength));

  // Add AppMaster.jar location to classpath
  // At some point we should not be required to add
  // the hadoop specific classpaths to the env.
  // It should be provided out of the box.
  // For now setting all required classpaths including
  // the classpath to "." for the application jar
  StringBuilder classPathEnv = new StringBuilder(Environment.CLASSPATH.$$())
      .append(ApplicationConstants.CLASS_PATH_SEPARATOR).append("./*");
  for (String c : conf.getStrings(
      YarnConfiguration.YARN_APPLICATION_CLASSPATH,
      YarnConfiguration.DEFAULT_YARN_CROSS_PLATFORM_APPLICATION_CLASSPATH)) {
    classPathEnv.append(ApplicationConstants.CLASS_PATH_SEPARATOR);
    classPathEnv.append(c.trim());
  }
  env.put("CLASSPATH", classPathEnv.toString());

  return env;
}
 

public static void setupYarnClassPath(Configuration conf, Map<String, String> appMasterEnv) {
	addToEnvironment(
		appMasterEnv,
		Environment.CLASSPATH.name(),
		appMasterEnv.get(ENV_FLINK_CLASSPATH));
	String[] applicationClassPathEntries = conf.getStrings(
		YarnConfiguration.YARN_APPLICATION_CLASSPATH,
		YarnConfiguration.DEFAULT_YARN_APPLICATION_CLASSPATH);
	for (String c : applicationClassPathEntries) {
		addToEnvironment(appMasterEnv, Environment.CLASSPATH.name(), c.trim());
	}
}
 

/**
 * Lock this on initialClasspath so that there is only one fork in the AM for
 * getting the initial class-path. TODO: We already construct
 * a parent CLC and use it for all the containers, so this should go away
 * once the mr-generated-classpath stuff is gone.
 */
private static String getInitialClasspath(Configuration conf) {
  synchronized (classpathLock) {
    if (initialClasspathFlag.get()) {
      return initialClasspath;
    }
    Map<String, String> env = new HashMap<String, String>();

    initialClasspath = env.get(Environment.CLASSPATH.name());
    initialClasspathFlag.set(true);
    return initialClasspath;
  }
}