Java Code Examples for org.apache.hadoop.yarn.api.ApplicationConstants#CLASS_PATH_SEPARATOR

The following examples show how to use org.apache.hadoop.yarn.api.ApplicationConstants#CLASS_PATH_SEPARATOR . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: TestContainerLaunch.java    From hadoop with Apache License 2.0 6 votes vote down vote up 
@Test(timeout = 10000)
public void testEnvExpansion() throws IOException {
  Path logPath = new Path("/nm/container/logs");
  String input =
      Apps.crossPlatformify("HADOOP_HOME") + "/share/hadoop/common/*"
          + ApplicationConstants.CLASS_PATH_SEPARATOR
          + Apps.crossPlatformify("HADOOP_HOME") + "/share/hadoop/common/lib/*"
          + ApplicationConstants.CLASS_PATH_SEPARATOR
          + Apps.crossPlatformify("HADOOP_LOG_HOME")
          + ApplicationConstants.LOG_DIR_EXPANSION_VAR;

  String res = ContainerLaunch.expandEnvironment(input, logPath);

  if (Shell.WINDOWS) {
    Assert.assertEquals("%HADOOP_HOME%/share/hadoop/common/*;"
        + "%HADOOP_HOME%/share/hadoop/common/lib/*;"
        + "%HADOOP_LOG_HOME%/nm/container/logs", res);
  } else {
    Assert.assertEquals("$HADOOP_HOME/share/hadoop/common/*:"
        + "$HADOOP_HOME/share/hadoop/common/lib/*:"
        + "$HADOOP_LOG_HOME/nm/container/logs", res);
  }
  System.out.println(res);
}
Example 2
Source File: TestContainerLaunch.java    From big-c with Apache License 2.0 6 votes vote down vote up 
@Test(timeout = 10000)
public void testEnvExpansion() throws IOException {
  Path logPath = new Path("/nm/container/logs");
  String input =
      Apps.crossPlatformify("HADOOP_HOME") + "/share/hadoop/common/*"
          + ApplicationConstants.CLASS_PATH_SEPARATOR
          + Apps.crossPlatformify("HADOOP_HOME") + "/share/hadoop/common/lib/*"
          + ApplicationConstants.CLASS_PATH_SEPARATOR
          + Apps.crossPlatformify("HADOOP_LOG_HOME")
          + ApplicationConstants.LOG_DIR_EXPANSION_VAR;

  String res = ContainerLaunch.expandEnvironment(input, logPath);

  if (Shell.WINDOWS) {
    Assert.assertEquals("%HADOOP_HOME%/share/hadoop/common/*;"
        + "%HADOOP_HOME%/share/hadoop/common/lib/*;"
        + "%HADOOP_LOG_HOME%/nm/container/logs", res);
  } else {
    Assert.assertEquals("$HADOOP_HOME/share/hadoop/common/*:"
        + "$HADOOP_HOME/share/hadoop/common/lib/*:"
        + "$HADOOP_LOG_HOME/nm/container/logs", res);
  }
  System.out.println(res);
}
Example 3
Source File: TestYARNRunner.java    From hadoop with Apache License 2.0 5 votes vote down vote up 
@Test
public void testAMStandardEnv() throws Exception {
  final String ADMIN_LIB_PATH = "foo";
  final String USER_LIB_PATH = "bar";
  final String USER_SHELL = "shell";
  JobConf jobConf = new JobConf();

  jobConf.set(MRJobConfig.MR_AM_ADMIN_USER_ENV, "LD_LIBRARY_PATH=" +
      ADMIN_LIB_PATH);
  jobConf.set(MRJobConfig.MR_AM_ENV, "LD_LIBRARY_PATH="
      + USER_LIB_PATH);
  jobConf.set(MRJobConfig.MAPRED_ADMIN_USER_SHELL, USER_SHELL);

  YARNRunner yarnRunner = new YARNRunner(jobConf);
  ApplicationSubmissionContext appSubCtx =
      buildSubmitContext(yarnRunner, jobConf);

  // make sure PWD is first in the lib path
  ContainerLaunchContext clc = appSubCtx.getAMContainerSpec();
  Map<String, String> env = clc.getEnvironment();
  String libPath = env.get(Environment.LD_LIBRARY_PATH.name());
  assertNotNull("LD_LIBRARY_PATH not set", libPath);
  String cps = jobConf.getBoolean(
      MRConfig.MAPREDUCE_APP_SUBMISSION_CROSS_PLATFORM,
      MRConfig.DEFAULT_MAPREDUCE_APP_SUBMISSION_CROSS_PLATFORM)
      ? ApplicationConstants.CLASS_PATH_SEPARATOR : File.pathSeparator;
  assertEquals("Bad AM LD_LIBRARY_PATH setting",
      MRApps.crossPlatformifyMREnv(conf, Environment.PWD)
      + cps + ADMIN_LIB_PATH + cps + USER_LIB_PATH, libPath);

  // make sure SHELL is set
  String shell = env.get(Environment.SHELL.name());
  assertNotNull("SHELL not set", shell);
  assertEquals("Bad SHELL setting", USER_SHELL, shell);
}
Example 4
Source File: MRApps.java    From hadoop with Apache License 2.0 5 votes vote down vote up 
public static void setEnvFromInputString(Map<String, String> env,
    String envString, Configuration conf) {
  String classPathSeparator =
      conf.getBoolean(MRConfig.MAPREDUCE_APP_SUBMISSION_CROSS_PLATFORM,
        MRConfig.DEFAULT_MAPREDUCE_APP_SUBMISSION_CROSS_PLATFORM)
          ? ApplicationConstants.CLASS_PATH_SEPARATOR : File.pathSeparator;
  Apps.setEnvFromInputString(env, envString, classPathSeparator);
}
Example 5
Source File: MRApps.java    From hadoop with Apache License 2.0 5 votes vote down vote up 
@Public
@Unstable
public static void addToEnvironment(Map<String, String> environment,
    String variable, String value, Configuration conf) {
  String classPathSeparator =
      conf.getBoolean(MRConfig.MAPREDUCE_APP_SUBMISSION_CROSS_PLATFORM,
        MRConfig.DEFAULT_MAPREDUCE_APP_SUBMISSION_CROSS_PLATFORM)
          ? ApplicationConstants.CLASS_PATH_SEPARATOR : File.pathSeparator;
  Apps.addToEnvironment(environment, variable, value, classPathSeparator);
}
Example 6
Source File: TestYARNRunner.java    From big-c with Apache License 2.0 5 votes vote down vote up 
@Test
public void testAMStandardEnv() throws Exception {
  final String ADMIN_LIB_PATH = "foo";
  final String USER_LIB_PATH = "bar";
  final String USER_SHELL = "shell";
  JobConf jobConf = new JobConf();

  jobConf.set(MRJobConfig.MR_AM_ADMIN_USER_ENV, "LD_LIBRARY_PATH=" +
      ADMIN_LIB_PATH);
  jobConf.set(MRJobConfig.MR_AM_ENV, "LD_LIBRARY_PATH="
      + USER_LIB_PATH);
  jobConf.set(MRJobConfig.MAPRED_ADMIN_USER_SHELL, USER_SHELL);

  YARNRunner yarnRunner = new YARNRunner(jobConf);
  ApplicationSubmissionContext appSubCtx =
      buildSubmitContext(yarnRunner, jobConf);

  // make sure PWD is first in the lib path
  ContainerLaunchContext clc = appSubCtx.getAMContainerSpec();
  Map<String, String> env = clc.getEnvironment();
  String libPath = env.get(Environment.LD_LIBRARY_PATH.name());
  assertNotNull("LD_LIBRARY_PATH not set", libPath);
  String cps = jobConf.getBoolean(
      MRConfig.MAPREDUCE_APP_SUBMISSION_CROSS_PLATFORM,
      MRConfig.DEFAULT_MAPREDUCE_APP_SUBMISSION_CROSS_PLATFORM)
      ? ApplicationConstants.CLASS_PATH_SEPARATOR : File.pathSeparator;
  assertEquals("Bad AM LD_LIBRARY_PATH setting",
      MRApps.crossPlatformifyMREnv(conf, Environment.PWD)
      + cps + ADMIN_LIB_PATH + cps + USER_LIB_PATH, libPath);

  // make sure SHELL is set
  String shell = env.get(Environment.SHELL.name());
  assertNotNull("SHELL not set", shell);
  assertEquals("Bad SHELL setting", USER_SHELL, shell);
}
Example 7
Source File: MRApps.java    From big-c with Apache License 2.0 5 votes vote down vote up 
public static void setEnvFromInputString(Map<String, String> env,
    String envString, Configuration conf) {
  String classPathSeparator =
      conf.getBoolean(MRConfig.MAPREDUCE_APP_SUBMISSION_CROSS_PLATFORM,
        MRConfig.DEFAULT_MAPREDUCE_APP_SUBMISSION_CROSS_PLATFORM)
          ? ApplicationConstants.CLASS_PATH_SEPARATOR : File.pathSeparator;
  Apps.setEnvFromInputString(env, envString, classPathSeparator);
}
Example 8
Source File: MRApps.java    From big-c with Apache License 2.0 5 votes vote down vote up 
@Public
@Unstable
public static void addToEnvironment(Map<String, String> environment,
    String variable, String value, Configuration conf) {
  String classPathSeparator =
      conf.getBoolean(MRConfig.MAPREDUCE_APP_SUBMISSION_CROSS_PLATFORM,
        MRConfig.DEFAULT_MAPREDUCE_APP_SUBMISSION_CROSS_PLATFORM)
          ? ApplicationConstants.CLASS_PATH_SEPARATOR : File.pathSeparator;
  Apps.addToEnvironment(environment, variable, value, classPathSeparator);
}
Example 9
Source File: YarnRemoteInterpreterProcess.java    From zeppelin with Apache License 2.0 5 votes vote down vote up 
/**
 * Populate the classpath entry in the given environment map with any application
 * classpath specified through the Hadoop and Yarn configurations.
 */
private void populateHadoopClasspath(Map<String, String> envs) {
  List<String> yarnClassPath = Lists.newArrayList(getYarnAppClasspath());
  List<String> mrClassPath = Lists.newArrayList(getMRAppClasspath());
  yarnClassPath.addAll(mrClassPath);
  LOGGER.info("Adding hadoop classpath: " + org.apache.commons.lang3.StringUtils.join(yarnClassPath, ":"));
  for (String path : yarnClassPath) {
    String newValue = path;
    if (envs.containsKey(ApplicationConstants.Environment.CLASSPATH.name())) {
      newValue = envs.get(ApplicationConstants.Environment.CLASSPATH.name()) +
              ApplicationConstants.CLASS_PATH_SEPARATOR + newValue;
    }
    envs.put(ApplicationConstants.Environment.CLASSPATH.name(), newValue);
  }
}
Example 10
Source File: OlapServerSubmitter.java    From spliceengine with GNU Affero General Public License v3.0 5 votes vote down vote up 
private void addPathToEnvironment(Map<String, String> env, String key, String value) {
    String newValue = value;
    if (env.containsKey(key)) {
        newValue = env.get(key) + ApplicationConstants.CLASS_PATH_SEPARATOR  + value;
    }
    env.put(key, newValue);
}