org.apache.hadoop.yarn.util.Apps Java Exaples

Source File: TestContainerLaunch.java From hadoop with Apache License 2.0

6 votes

@Test(timeout = 10000)
public void testEnvExpansion() throws IOException {
  Path logPath = new Path("/nm/container/logs");
  String input =
      Apps.crossPlatformify("HADOOP_HOME") + "/share/hadoop/common/*"
          + ApplicationConstants.CLASS_PATH_SEPARATOR
          + Apps.crossPlatformify("HADOOP_HOME") + "/share/hadoop/common/lib/*"
          + ApplicationConstants.CLASS_PATH_SEPARATOR
          + Apps.crossPlatformify("HADOOP_LOG_HOME")
          + ApplicationConstants.LOG_DIR_EXPANSION_VAR;

  String res = ContainerLaunch.expandEnvironment(input, logPath);

  if (Shell.WINDOWS) {
    Assert.assertEquals("%HADOOP_HOME%/share/hadoop/common/*;"
        + "%HADOOP_HOME%/share/hadoop/common/lib/*;"
        + "%HADOOP_LOG_HOME%/nm/container/logs", res);
  } else {
    Assert.assertEquals("$HADOOP_HOME/share/hadoop/common/*:"
        + "$HADOOP_HOME/share/hadoop/common/lib/*:"
        + "$HADOOP_LOG_HOME/nm/container/logs", res);
  }
  System.out.println(res);
}

Source File: TestContainerLaunch.java From big-c with Apache License 2.0

6 votes

@Test(timeout = 10000)
public void testEnvExpansion() throws IOException {
  Path logPath = new Path("/nm/container/logs");
  String input =
      Apps.crossPlatformify("HADOOP_HOME") + "/share/hadoop/common/*"
          + ApplicationConstants.CLASS_PATH_SEPARATOR
          + Apps.crossPlatformify("HADOOP_HOME") + "/share/hadoop/common/lib/*"
          + ApplicationConstants.CLASS_PATH_SEPARATOR
          + Apps.crossPlatformify("HADOOP_LOG_HOME")
          + ApplicationConstants.LOG_DIR_EXPANSION_VAR;

  String res = ContainerLaunch.expandEnvironment(input, logPath);

  if (Shell.WINDOWS) {
    Assert.assertEquals("%HADOOP_HOME%/share/hadoop/common/*;"
        + "%HADOOP_HOME%/share/hadoop/common/lib/*;"
        + "%HADOOP_LOG_HOME%/nm/container/logs", res);
  } else {
    Assert.assertEquals("$HADOOP_HOME/share/hadoop/common/*:"
        + "$HADOOP_HOME/share/hadoop/common/lib/*:"
        + "$HADOOP_LOG_HOME/nm/container/logs", res);
  }
  System.out.println(res);
}

Source File: YarnManager.java From Scribengin with GNU Affero General Public License v3.0

6 votes

void setupAppClasspath(boolean miniClusterEnv, Configuration conf, Map<String, String> appMasterEnv) {
  if(miniClusterEnv) {
    String cps = System.getProperty("java.class.path") ;
    String[] cp = cps.split(":") ;
    for(String selCp : cp) {
      Apps.addToEnvironment(appMasterEnv, Environment.CLASSPATH.name(), selCp, ":");
    }
  } else {
    StringBuilder classPathEnv = new StringBuilder();
    classPathEnv.append(Environment.CLASSPATH.$()).append(File.pathSeparatorChar);
    classPathEnv.append("./*");

    String[] classpath = conf.getStrings(
        YarnConfiguration.YARN_APPLICATION_CLASSPATH,
        YarnConfiguration.DEFAULT_YARN_APPLICATION_CLASSPATH
    ) ;
    for (String selClasspath : classpath) {
      classPathEnv.append(File.pathSeparatorChar);
      classPathEnv.append(selClasspath.trim());
    }
    appMasterEnv.put(Environment.CLASSPATH.name(), classPathEnv.toString());
    System.err.println("CLASSPATH: " + classPathEnv);
  }
  Apps.addToEnvironment(appMasterEnv, Environment.CLASSPATH.name(), Environment.PWD.$() + File.separator + "*", ":");
}

Source File: YarnJobContainer.java From sylph with Apache License 2.0

5 votes

public JobContainer build()
{
    final YarnJobContainer yarnJobContainer = new YarnJobContainer(yarnClient, submitter);
    if (lastRunId != null) {
        yarnJobContainer.yarnAppId = Apps.toAppID(lastRunId);
        yarnJobContainer.setStatus(RUNNING);
        try {
            yarnJobContainer.webUi = yarnClient.getApplicationReport(yarnJobContainer.yarnAppId).getTrackingUrl();
        }
        catch (Exception e) {
            logger.warn(e.getMessage());
        }
    }

    //----create JobContainer Proxy
    return AopFactory.proxy(JobContainer.class)
            .byInstance(yarnJobContainer)
            .around(proxyContext -> {
                /*
                 * 通过这个 修改当前YarnClient的ClassLoader的为当前runner的加载器
                 * 默认hadoop Configuration使用jvm的AppLoader,会出现 akka.version not setting的错误 原因是找不到akka相关jar包
                 * 原因是hadoop Configuration 初始化: this.classLoader = Thread.currentThread().getContextClassLoader();
                 * */
                try (Closeables ignored = Closeables.openThreadContextClassLoader(jobClassLoader)) {
                    return proxyContext.proceed();
                }
            });
}

Source File: MRApps.java From hadoop with Apache License 2.0

5 votes

public static void setEnvFromInputString(Map<String, String> env,
    String envString, Configuration conf) {
  String classPathSeparator =
      conf.getBoolean(MRConfig.MAPREDUCE_APP_SUBMISSION_CROSS_PLATFORM,
        MRConfig.DEFAULT_MAPREDUCE_APP_SUBMISSION_CROSS_PLATFORM)
          ? ApplicationConstants.CLASS_PATH_SEPARATOR : File.pathSeparator;
  Apps.setEnvFromInputString(env, envString, classPathSeparator);
}

Source File: MRApps.java From hadoop with Apache License 2.0

5 votes

@Public
@Unstable
public static void addToEnvironment(Map<String, String> environment,
    String variable, String value, Configuration conf) {
  String classPathSeparator =
      conf.getBoolean(MRConfig.MAPREDUCE_APP_SUBMISSION_CROSS_PLATFORM,
        MRConfig.DEFAULT_MAPREDUCE_APP_SUBMISSION_CROSS_PLATFORM)
          ? ApplicationConstants.CLASS_PATH_SEPARATOR : File.pathSeparator;
  Apps.addToEnvironment(environment, variable, value, classPathSeparator);
}

Source File: MRApps.java From big-c with Apache License 2.0

5 votes

public static void setEnvFromInputString(Map<String, String> env,
    String envString, Configuration conf) {
  String classPathSeparator =
      conf.getBoolean(MRConfig.MAPREDUCE_APP_SUBMISSION_CROSS_PLATFORM,
        MRConfig.DEFAULT_MAPREDUCE_APP_SUBMISSION_CROSS_PLATFORM)
          ? ApplicationConstants.CLASS_PATH_SEPARATOR : File.pathSeparator;
  Apps.setEnvFromInputString(env, envString, classPathSeparator);
}

Source File: MRApps.java From big-c with Apache License 2.0

5 votes

@Public
@Unstable
public static void addToEnvironment(Map<String, String> environment,
    String variable, String value, Configuration conf) {
  String classPathSeparator =
      conf.getBoolean(MRConfig.MAPREDUCE_APP_SUBMISSION_CROSS_PLATFORM,
        MRConfig.DEFAULT_MAPREDUCE_APP_SUBMISSION_CROSS_PLATFORM)
          ? ApplicationConstants.CLASS_PATH_SEPARATOR : File.pathSeparator;
  Apps.addToEnvironment(environment, variable, value, classPathSeparator);
}

Source File: IgniteYarnClient.java From ignite with Apache License 2.0

5 votes

/**
 * @param envs Environment variables.
 * @param conf Yarn configuration.
 */
private static void setupAppMasterEnv(Map<String, String> envs, YarnConfiguration conf) {
    for (String c : conf.getStrings(YarnConfiguration.YARN_APPLICATION_CLASSPATH,
        YarnConfiguration.DEFAULT_YARN_APPLICATION_CLASSPATH))
        Apps.addToEnvironment(envs, Environment.CLASSPATH.name(),
                c.trim(), File.pathSeparator);

    Apps.addToEnvironment(envs,
            Environment.CLASSPATH.name(),
            Environment.PWD.$() + File.separator + "*",
            File.pathSeparator);
}

Source File: Client.java From hadoop-mini-clusters with Apache License 2.0

5 votes

@SuppressWarnings("deprecation")
private void setupAppMasterEnv(Map<String, String> appMasterEnv) {
    for (String c : conf.getStrings(
            YarnConfiguration.YARN_APPLICATION_CLASSPATH,
            YarnConfiguration.DEFAULT_YARN_APPLICATION_CLASSPATH)) {
        Apps.addToEnvironment(appMasterEnv, Environment.CLASSPATH.name(),
                c.trim());
    }
    Apps.addToEnvironment(appMasterEnv,
            Environment.CLASSPATH.name(),
            Environment.PWD.$() + File.separator + "*");
}

Source File: SolrClient.java From yarn-proto with Apache License 2.0

5 votes

protected void setupAppMasterEnv(CommandLine cli, Map<String, String> appMasterEnv) throws IOException {
  for (String c : conf.getStrings(
          YarnConfiguration.YARN_APPLICATION_CLASSPATH,
          YarnConfiguration.DEFAULT_YARN_APPLICATION_CLASSPATH)) {
    Apps.addToEnvironment(appMasterEnv, Environment.CLASSPATH.name(), c.trim());
  }
  Apps.addToEnvironment(appMasterEnv,
          Environment.CLASSPATH.name(),
          Environment.PWD.$() + File.separator + "*");

  if (cli.hasOption("extclasspath")) {
    String extclasspathArg = cli.getOptionValue("extclasspath");
    File extclasspathFile = new File(extclasspathArg);
    StringBuilder sb = new StringBuilder();
    BufferedReader reader = null;
    String line = null;
    try {
      reader = new BufferedReader(
        new InputStreamReader(new FileInputStream(extclasspathFile), Charset.forName("UTF-8")));
      while ((line = reader.readLine()) != null)
        sb.append(line.trim()).append(":");
    } finally {
      if (reader != null) {
        reader.close();
      }
    }
    for (String part : sb.toString().split(":")) {
      if (part.length() > 0)
        Apps.addToEnvironment(appMasterEnv, Environment.CLASSPATH.name(), part);
    }
  }
}

Source File: Utils.java From stratosphere with Apache License 2.0

4 votes

public static void setupEnv(Configuration conf, Map<String, String> appMasterEnv) {
	for (String c : conf.getStrings(YarnConfiguration.YARN_APPLICATION_CLASSPATH,YarnConfiguration.DEFAULT_YARN_APPLICATION_CLASSPATH)) {
		Apps.addToEnvironment(appMasterEnv, Environment.CLASSPATH.name(), c.trim());
	}
	Apps.addToEnvironment(appMasterEnv, Environment.CLASSPATH.name(), Environment.PWD.$() + File.separator + "*");
}

org.apache.hadoop.yarn.util.Apps Java Examples