org.apache.hadoop.yarn.api.ApplicationConstants Java Examples

@Test (timeout = 120000)
public void testSetClasspathWithNoUserPrecendence() {
  Configuration conf = new Configuration();
  conf.setBoolean(MRConfig.MAPREDUCE_APP_SUBMISSION_CROSS_PLATFORM, true);
  conf.setBoolean(MRJobConfig.MAPREDUCE_JOB_USER_CLASSPATH_FIRST, false);
  Map<String, String> env = new HashMap<String, String>();
  try {
    MRApps.setClasspath(env, conf);
  } catch (Exception e) {
    fail("Got exception while setting classpath");
  }
  String env_str = env.get("CLASSPATH");
  String expectedClasspath = StringUtils.join(ApplicationConstants.CLASS_PATH_SEPARATOR,
    Arrays.asList("job.jar/job.jar", "job.jar/classes/", "job.jar/lib/*",
      ApplicationConstants.Environment.PWD.$$() + "/*"));
  assertTrue("MAPREDUCE_JOB_USER_CLASSPATH_FIRST false, and job.jar is not in"
    + " the classpath!", env_str.contains(expectedClasspath));
  assertFalse("MAPREDUCE_JOB_USER_CLASSPATH_FIRST false, but taking effect!",
    env_str.startsWith(expectedClasspath));
}
 

@VisibleForTesting
public static String expandEnvironment(String var,
    Path containerLogDir) {
  var = var.replace(ApplicationConstants.LOG_DIR_EXPANSION_VAR,
    containerLogDir.toString());
  var =  var.replace(ApplicationConstants.CLASS_PATH_SEPARATOR,
    File.pathSeparator);

  // replace parameter expansion marker. e.g. {{VAR}} on Windows is replaced
  // as %VAR% and on Linux replaced as "$VAR"
  if (Shell.WINDOWS) {
    var = var.replaceAll("(\\{\\{)|(\\}\\})", "%");
  } else {
    var = var.replace(ApplicationConstants.PARAMETER_EXPANSION_LEFT, "$");
    var = var.replace(ApplicationConstants.PARAMETER_EXPANSION_RIGHT, "");
  }
  return var;
}
 

/**
 * Runs a process as specified by the command builder on the container.
 * @param processorId id of the samza processor to run (passed as a command line parameter to the process)
 * @param container the yarn container to run the processor on.
 * @param cmdBuilder the command builder that encapsulates the command, and the context
 * @throws IOException on IO exceptions running the container
 */
public void runProcessor(String processorId, Container container, CommandBuilder cmdBuilder) throws IOException {
  String containerIdStr = ConverterUtils.toString(container.getId());
  String cmdPath = "./__package/";
  cmdBuilder.setCommandPath(cmdPath);
  String command = cmdBuilder.buildCommand();

  Map<String, String> env = getEscapedEnvironmentVariablesMap(cmdBuilder);
  env.put(ShellCommandConfig.ENV_EXECUTION_ENV_CONTAINER_ID, Util.envVarEscape(container.getId().toString()));

  Path packagePath = new Path(yarnConfig.getPackagePath());
  String formattedCommand =
      getFormattedCommand(ApplicationConstants.LOG_DIR_EXPANSION_VAR, command, ApplicationConstants.STDOUT,
          ApplicationConstants.STDERR);

  log.info("Running Processor ID: {} on Container ID: {} on host: {} using command: {} and env: {} and package path: {}",
      processorId, containerIdStr, container.getNodeHttpAddress(), formattedCommand, env, packagePath);
  state.pendingProcessors.put(processorId, new YarnContainer(container));

  startContainer(packagePath, container, env, formattedCommand);

  log.info("Made start request for Processor ID: {} on Container ID: {} on host: {} (http://{}/node/containerlogs/{}).",
      processorId, containerIdStr, container.getNodeId().getHost(), container.getNodeHttpAddress(), containerIdStr);
}
 

@Test (timeout = 120000)
public void testSetClasspathWithUserPrecendence() {
   Configuration conf = new Configuration();
   conf.setBoolean(MRConfig.MAPREDUCE_APP_SUBMISSION_CROSS_PLATFORM, true);
   conf.setBoolean(MRJobConfig.MAPREDUCE_JOB_USER_CLASSPATH_FIRST, true);
   Map<String, String> env = new HashMap<String, String>();
   try {
     MRApps.setClasspath(env, conf);
   } catch (Exception e) {
     fail("Got exception while setting classpath");
   }
   String env_str = env.get("CLASSPATH");
   String expectedClasspath = StringUtils.join(ApplicationConstants.CLASS_PATH_SEPARATOR,
     Arrays.asList(ApplicationConstants.Environment.PWD.$$(), "job.jar/job.jar",
       "job.jar/classes/", "job.jar/lib/*",
       ApplicationConstants.Environment.PWD.$$() + "/*"));
   assertTrue("MAPREDUCE_JOB_USER_CLASSPATH_FIRST set, but not taking effect!",
     env_str.startsWith(expectedClasspath));
 }
 

private String makeContainerCommand(long containerMemory, String clusterSpec,
    String jobName, int taskIndex) {
  String[] commands = new String[]{
      ApplicationConstants.Environment.JAVA_HOME.$$() + "/bin/java",
      "-Xmx" + containerMemory + "m",
      TFServerRunner.class.getName() + " ",
      Utils.mkOption(Constants.OPT_CLUSTER_SPEC, clusterSpec),
      Utils.mkOption(Constants.OPT_JOB_NAME, jobName),
      Utils.mkOption(Constants.OPT_TASK_INDEX, taskIndex),
      "1>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/TFServerRunner." +
          ApplicationConstants.STDOUT,
      "2>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/TFServerRunner." +
          ApplicationConstants.STDERR
  };

  return Utils.mkString(commands, " ");
}
 

private String makeAppMasterCommand(String tfLib, String tfJar) {
  String[] commands = new String[]{
      ApplicationConstants.Environment.JAVA_HOME.$$() + "/bin/java",
      // Set Xmx based on am memory size
      "-Xmx" + amMemory + "m",
      // Set class name
      ApplicationMaster.class.getName(),
      Utils.mkOption(Constants.OPT_TF_CONTAINER_MEMORY, containerMemory),
      Utils.mkOption(Constants.OPT_TF_CONTAINER_VCORES, containerVCores),
      Utils.mkOption(Constants.OPT_TF_WORKER_NUM, workerNum),
      Utils.mkOption(Constants.OPT_TF_PS_NUM, psNum),
      Utils.mkOption(Constants.OPT_TF_LIB, tfLib),
      Utils.mkOption(Constants.OPT_TF_JAR, tfJar),
      "1>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/AppMaster.stdout",
      "2>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/AppMaster.stderr"
  };
  return Utils.mkString(commands, " ");
}
 

public static void setupDefaultEnv(Map<String, String> env,
    Configuration conf,  String confEnvKey, String confEnvKeyDefault) {
  // Setup the CLASSPATH in environment
  // i.e. add { Hadoop jars, job jar, CWD } to classpath.
  String classpath = getFrameworkClasspath(conf);
  TezYARNUtils.addToEnvironment(env,
      ApplicationConstants.Environment.CLASSPATH.name(),
      classpath, File.pathSeparator);

  // set any env from config if it is not set already
  TezYARNUtils.setEnvIfAbsentFromInputString(env, conf.get(
      confEnvKey, confEnvKeyDefault));
  
  // Append pwd to LD_LIBRARY_PATH
  // Done separately here because this is known to work platform independent
  TezYARNUtils.addToEnvironment(env, Environment.LD_LIBRARY_PATH.name(),
      Environment.PWD.$(), File.pathSeparator);
}
 

@Test (timeout = 120000)
public void testSetClasspathWithNoUserPrecendence() {
  Configuration conf = new Configuration();
  conf.setBoolean(MRConfig.MAPREDUCE_APP_SUBMISSION_CROSS_PLATFORM, true);
  conf.setBoolean(MRJobConfig.MAPREDUCE_JOB_USER_CLASSPATH_FIRST, false);
  Map<String, String> env = new HashMap<String, String>();
  try {
    MRApps.setClasspath(env, conf);
  } catch (Exception e) {
    fail("Got exception while setting classpath");
  }
  String env_str = env.get("CLASSPATH");
  String expectedClasspath = StringUtils.join(ApplicationConstants.CLASS_PATH_SEPARATOR,
    Arrays.asList("job.jar/job.jar", "job.jar/classes/", "job.jar/lib/*",
      ApplicationConstants.Environment.PWD.$$() + "/*"));
  assertTrue("MAPREDUCE_JOB_USER_CLASSPATH_FIRST false, and job.jar is not in"
    + " the classpath!", env_str.contains(expectedClasspath));
  assertFalse("MAPREDUCE_JOB_USER_CLASSPATH_FIRST false, but taking effect!",
    env_str.startsWith(expectedClasspath));
}
 

private RunContainerRequestProto constructRunContainerRequest(ContainerLaunchRequest launchRequest) throws
    IOException {
  RunContainerRequestProto.Builder builder = RunContainerRequestProto.newBuilder();
  Preconditions.checkArgument(launchRequest.getTaskCommunicatorName().equals(
      TezConstants.getTezYarnServicePluginName()));
  InetSocketAddress address = (InetSocketAddress) getContext().getTaskCommunicatorMetaInfo(launchRequest.getTaskCommunicatorName());
  builder.setAmHost(address.getHostName()).setAmPort(address.getPort());
  builder.setAppAttemptNumber(appAttemptId.getAttemptId());
  builder.setApplicationIdString(appAttemptId.getApplicationId().toString());
  builder.setTokenIdentifier(tokenIdentifier);
  builder.setContainerIdString(launchRequest.getContainerId().toString());
  builder.setCredentialsBinary(
      ByteString.copyFrom(launchRequest.getContainerLaunchContext().getTokens()));
  // TODO Avoid reading this from the environment
  builder.setUser(System.getenv(ApplicationConstants.Environment.USER.name()));
  return builder.build();
}
 

@Test (timeout = 120000)
public void testSetClasspathWithJobClassloader() throws IOException {
  Configuration conf = new Configuration();
  conf.setBoolean(MRConfig.MAPREDUCE_APP_SUBMISSION_CROSS_PLATFORM, true);
  conf.setBoolean(MRJobConfig.MAPREDUCE_JOB_CLASSLOADER, true);
  Map<String, String> env = new HashMap<String, String>();
  MRApps.setClasspath(env, conf);
  String cp = env.get("CLASSPATH");
  String appCp = env.get("APP_CLASSPATH");
  assertFalse("MAPREDUCE_JOB_CLASSLOADER true, but job.jar is in the"
    + " classpath!", cp.contains("jar" + ApplicationConstants.CLASS_PATH_SEPARATOR + "job"));
  assertFalse("MAPREDUCE_JOB_CLASSLOADER true, but PWD is in the classpath!",
    cp.contains("PWD"));
  String expectedAppClasspath = StringUtils.join(ApplicationConstants.CLASS_PATH_SEPARATOR,
    Arrays.asList(ApplicationConstants.Environment.PWD.$$(), "job.jar/job.jar",
      "job.jar/classes/", "job.jar/lib/*",
      ApplicationConstants.Environment.PWD.$$() + "/*"));
  assertEquals("MAPREDUCE_JOB_CLASSLOADER true, but job.jar is not in the app"
    + " classpath!", expectedAppClasspath, appCp);
}
 

public static void setupDefaultEnv(Map<String, String> env, Configuration conf,  String userEnvKey, String userEnvDefault,
    String clusterDefaultEnvKey, String clusterDefaultEnvDefault, boolean usingArchive) {
  // Setup the CLASSPATH in environment
  // i.e. add { Hadoop jars, job jar, CWD } to classpath.
  String classpath = getFrameworkClasspath(conf, usingArchive);
  TezYARNUtils.addToEnvironment(env,
      ApplicationConstants.Environment.CLASSPATH.name(),
      classpath, File.pathSeparator);

  // Pre-pend pwd to LD_LIBRARY_PATH
  // Done separately here because this is known to work platform
  // independent
  TezYARNUtils.addToEnvironment(env,
      Environment.LD_LIBRARY_PATH.name(), Environment.PWD.$(), File.pathSeparator);
  TezYARNUtils.appendToEnvFromInputString(env,
      conf.get(userEnvKey, userEnvDefault), File.pathSeparator);
  // set any env from config if it is not set already
  TezYARNUtils.appendToEnvFromInputString(env,
      conf.get(clusterDefaultEnvKey, clusterDefaultEnvDefault), File.pathSeparator);
}
 

@VisibleForTesting
public static String expandEnvironment(String var,
    Path containerLogDir) {
  var = var.replace(ApplicationConstants.LOG_DIR_EXPANSION_VAR,
    containerLogDir.toString());
  var =  var.replace(ApplicationConstants.CLASS_PATH_SEPARATOR,
    File.pathSeparator);

  // replace parameter expansion marker. e.g. {{VAR}} on Windows is replaced
  // as %VAR% and on Linux replaced as "$VAR"
  if (Shell.WINDOWS) {
    var = var.replaceAll("(\\{\\{)|(\\}\\})", "%");
  } else {
    var = var.replace(ApplicationConstants.PARAMETER_EXPANSION_LEFT, "$");
    var = var.replace(ApplicationConstants.PARAMETER_EXPANSION_RIGHT, "");
  }
  return var;
}
 

public static void main(String[] args) {
	// startup checks and logging
	EnvironmentInformation.logEnvironmentInfo(LOG, YarnSessionClusterEntrypoint.class.getSimpleName(), args);
	SignalHandler.register(LOG);
	JvmShutdownSafeguard.installAsShutdownHook(LOG);

	Map<String, String> env = System.getenv();

	final String workingDirectory = env.get(ApplicationConstants.Environment.PWD.key());
	Preconditions.checkArgument(
		workingDirectory != null,
		"Working directory variable (%s) not set",
		ApplicationConstants.Environment.PWD.key());

	try {
		YarnEntrypointUtils.logYarnEnvironmentInformation(env, LOG);
	} catch (IOException e) {
		LOG.warn("Could not log YARN environment information.", e);
	}

	Configuration configuration = YarnEntrypointUtils.loadConfiguration(workingDirectory, env);

	YarnSessionClusterEntrypoint yarnSessionClusterEntrypoint = new YarnSessionClusterEntrypoint(configuration);

	ClusterEntrypoint.runClusterEntrypoint(yarnSessionClusterEntrypoint);
}
 

@GET
@Path(PATH_PHYSICAL_PLAN_CONTAINERS + "/{containerId}")
@Produces(MediaType.APPLICATION_JSON)
public JSONObject getContainer(@PathParam("containerId") String containerId) throws Exception
{
  init();
  ContainerInfo ci = null;
  if (containerId.equals(System.getenv(ApplicationConstants.Environment.CONTAINER_ID.toString()))) {
    ci = dagManager.getAppMasterContainerInfo();
  } else {
    for (ContainerInfo containerInfo : dagManager.getCompletedContainerInfo()) {
      if (containerInfo.id.equals(containerId)) {
        ci = containerInfo;
      }
    }
    if (ci == null) {
      StreamingContainerAgent sca = dagManager.getContainerAgent(containerId);
      if (sca == null) {
        throw new NotFoundException();
      }
      ci = sca.getContainerInfo();
    }
  }
  return new JSONObject(objectMapper.writeValueAsString(ci));
}
 

/**
 * Setup
 * 
 * @param containerTask
 *          the new task specification. Must be a valid task
 * @param childUGI
 *          the old UGI instance being used
 * @return
 */
UserGroupInformation handleNewTaskCredentials(ContainerTask containerTask,
    UserGroupInformation childUGI) {
  // Re-use the UGI only if the Credentials have not changed.
  Preconditions.checkState(containerTask.shouldDie() != true);
  Preconditions.checkState(containerTask.getTaskSpec() != null);
  if (containerTask.haveCredentialsChanged()) {
    LOG.info("Refreshing UGI since Credentials have changed");
    Credentials taskCreds = containerTask.getCredentials();
    if (taskCreds != null) {
      LOG.info("Credentials : #Tokens=" + taskCreds.numberOfTokens() + ", #SecretKeys="
          + taskCreds.numberOfSecretKeys());
      childUGI = UserGroupInformation.createRemoteUser(System
          .getenv(ApplicationConstants.Environment.USER.toString()));
      childUGI.addCredentials(containerTask.getCredentials());
    } else {
      LOG.info("Not loading any credentials, since no credentials provided");
    }
  }
  return childUGI;
}
 

@Test (timeout = 120000)
public void testSetClasspathWithJobClassloader() throws IOException {
  Configuration conf = new Configuration();
  conf.setBoolean(MRConfig.MAPREDUCE_APP_SUBMISSION_CROSS_PLATFORM, true);
  conf.setBoolean(MRJobConfig.MAPREDUCE_JOB_CLASSLOADER, true);
  Map<String, String> env = new HashMap<String, String>();
  MRApps.setClasspath(env, conf);
  String cp = env.get("CLASSPATH");
  String appCp = env.get("APP_CLASSPATH");
  assertFalse("MAPREDUCE_JOB_CLASSLOADER true, but job.jar is in the"
    + " classpath!", cp.contains("jar" + ApplicationConstants.CLASS_PATH_SEPARATOR + "job"));
  assertFalse("MAPREDUCE_JOB_CLASSLOADER true, but PWD is in the classpath!",
    cp.contains("PWD"));
  String expectedAppClasspath = StringUtils.join(ApplicationConstants.CLASS_PATH_SEPARATOR,
    Arrays.asList(ApplicationConstants.Environment.PWD.$$(), "job.jar/job.jar",
      "job.jar/classes/", "job.jar/lib/*",
      ApplicationConstants.Environment.PWD.$$() + "/*"));
  assertEquals("MAPREDUCE_JOB_CLASSLOADER true, but job.jar is not in the app"
    + " classpath!", expectedAppClasspath, appCp);
}
 

public ContainerInfo getContainerInfo()
{
  ContainerInfo ci = new ContainerInfo();
  ci.id = container.getExternalId();
  ci.host = container.host;
  ci.state = container.getState().name();
  ci.jvmName = this.jvmName;
  ci.numOperators = container.getOperators().size();
  ci.operators = new TreeMap<>();

  for (PTOperator ptOperator : container.getOperators()) {
    ci.operators.put(ptOperator.getId(), ptOperator.getName());
  }

  ci.memoryMBAllocated = container.getAllocatedMemoryMB();
  ci.lastHeartbeat = lastHeartbeatMillis;
  ci.memoryMBFree = this.memoryMBFree;
  ci.gcCollectionCount = this.gcCollectionCount;
  ci.gcCollectionTime = this.gcCollectionTime;
  ci.startedTime = container.getStartedTime();
  ci.finishedTime = container.getFinishedTime();
  if (this.container.nodeHttpAddress != null) {
    YarnConfiguration conf = new YarnConfiguration();
    ci.containerLogsUrl = ConfigUtils
        .getSchemePrefix(conf) + this.container.nodeHttpAddress + "/node/containerlogs/" + ci.id + "/" + System
        .getenv(ApplicationConstants.Environment.USER.toString());
    ci.rawContainerLogsUrl = ConfigUtils
        .getRawContainerLogsUrl(conf, container.nodeHttpAddress, container.getPlan().getLogicalPlan().getAttributes()
            .get(LogicalPlan.APPLICATION_ID), ci.id);
  }
  return ci;
}
 

@GET
@Path(PATH_PHYSICAL_PLAN_CONTAINERS + "/{containerId}/" + PATH_STACKTRACE)
@Produces(MediaType.APPLICATION_JSON)
public JSONObject getContainerStackTrace(@PathParam("containerId") String containerId) throws Exception
{
  init();

  if (containerId.equals(System.getenv(ApplicationConstants.Environment.CONTAINER_ID.toString()))) {
    return StramUtils.getStackTrace();
  }

  StreamingContainerAgent sca = dagManager.getContainerAgent(containerId);

  if (sca == null) {
    throw new NotFoundException("Container not found.");
  }

  if (!sca.getContainerInfo().state.equals("ACTIVE")) {
    throw new NotFoundException("Container is not active.");
  }

  for (int i = 0; i < STACK_TRACE_ATTEMPTS; ++i) {
    String result = sca.getStackTrace();

    if (result != null) {
      return new JSONObject(result);
    }

    Thread.sleep(STACK_TRACE_WAIT_TIME);
  }

  throw new TimeoutException("Not able to get the stack trace");
}
 

@Nonnull
private static Configuration loadConfiguration(Configuration initialConfiguration, Map<String, String> env) throws IOException {
	final File workingDirectory = TEMPORARY_FOLDER.newFolder();
	env.put(ApplicationConstants.Environment.NM_HOST.key(), "foobar");
	BootstrapTools.writeConfiguration(initialConfiguration, new File(workingDirectory, "flink-conf.yaml"));
	return YarnEntrypointUtils.loadConfiguration(workingDirectory.getAbsolutePath(), env);
}
 

/**
 * Assembles the command to execute the Driver in list form.
 */
private List<String> getCommandList(
    final JobSubmissionEvent jobSubmissionEvent) {

  return new JavaLaunchCommandBuilder()
      .setJavaPath("%JAVA_HOME%/bin/java")
      .setConfigurationFilePaths(Collections.singletonList(this.filenames.getDriverConfigurationPath()))
      .setClassPath(this.classpath.getDriverClasspath())
      .setMemory(jobSubmissionEvent.getDriverMemory().get())
      .setStandardErr(ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/" + this.filenames.getDriverStderrFileName())
      .setStandardOut(ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/" + this.filenames.getDriverStdoutFileName())
      .build();
}
 

@Test(timeout = 5000)
public void testTaskLoggingOptsPerLogger() {

  Configuration conf = new TezConfiguration();
  conf.set(TezConfiguration.TEZ_TASK_LOG_LEVEL, "DEBUG;org.apache.hadoop.ipc=INFO;org.apache.hadoop.server=INFO");

  LogTestInfoHolder testInfo = new LogTestInfoHolder();
  VertexWrapper vertexWrapper = createVertexWrapperForLogTests(testInfo, conf);

  List<String> expectedCommands = new LinkedList<String>();
  expectedCommands.add("-Dlog4j.configuratorClass=org.apache.tez.common.TezLog4jConfigurator");
  expectedCommands.add("-Dlog4j.configuration=" + TezConstants.TEZ_CONTAINER_LOG4J_PROPERTIES_FILE);
  expectedCommands.add("-D" + YarnConfiguration.YARN_APP_CONTAINER_LOG_DIR + "=" +
      ApplicationConstants.LOG_DIR_EXPANSION_VAR);
  expectedCommands.add("-D" + TezConstants.TEZ_ROOT_LOGGER_NAME + "=" + "DEBUG" + "," +
      TezConstants.TEZ_CONTAINER_LOGGER_NAME);

  for (int i = 0 ; i < testInfo.numTasks ; i++) {
    ContainerContext containerContext = vertexWrapper
        .vertex.getContainerContext(i);
    String javaOpts = containerContext.getJavaOpts();
    assertTrue(javaOpts.contains(testInfo.initialJavaOpts));
    for (String expectedCmd : expectedCommands) {
      assertTrue(javaOpts.contains(expectedCmd));
    }

    Map<String, String> env = containerContext.getEnvironment();
    String val = env.get(testInfo.envKey);
    assertEquals(testInfo.envVal, val);
    String logEnvVal = env.get(TezConstants.TEZ_CONTAINER_LOG_PARAMS);
    assertEquals("org.apache.hadoop.ipc=INFO;org.apache.hadoop.server=INFO", logEnvVal);
  }
}
 

@Test (timeout = 120000)
public void testSetClasspathWithArchives () throws IOException {
  File testTGZ = new File(testWorkDir, "test.tgz");
  FileOutputStream out = new FileOutputStream(testTGZ);
  out.write(0);
  out.close();
  Configuration conf = new Configuration();
  conf.setBoolean(MRConfig.MAPREDUCE_APP_SUBMISSION_CROSS_PLATFORM, true);
  Job job = Job.getInstance(conf);
  conf = job.getConfiguration();
  String testTGZQualifiedPath = FileSystem.getLocal(conf).makeQualified(new Path(
    testTGZ.getAbsolutePath())).toString();
  conf.set(MRJobConfig.CLASSPATH_ARCHIVES, testTGZQualifiedPath);
  conf.set(MRJobConfig.CACHE_ARCHIVES, testTGZQualifiedPath + "#testTGZ");
  Map<String, String> environment = new HashMap<String, String>();
  MRApps.setClasspath(environment, conf);
  assertTrue(environment.get("CLASSPATH").startsWith(
    ApplicationConstants.Environment.PWD.$$() + ApplicationConstants.CLASS_PATH_SEPARATOR));
  String confClasspath = job.getConfiguration().get(
      YarnConfiguration.YARN_APPLICATION_CLASSPATH,
      StringUtils.join(",",
          YarnConfiguration.DEFAULT_YARN_CROSS_PLATFORM_APPLICATION_CLASSPATH));
  if (confClasspath != null) {
    confClasspath = confClasspath.replaceAll(",\\s*", ApplicationConstants.CLASS_PATH_SEPARATOR)
      .trim();
  }
  assertTrue(environment.get("CLASSPATH").contains(confClasspath));
  assertTrue(environment.get("CLASSPATH").contains("testTGZ"));
}
 

@Private
@VisibleForTesting
public static void addLog4jSystemProperties(String logLevel,
    List<String> vargs) {
  vargs.add("-Dlog4j.configuration="
      + TezConfiguration.TEZ_CONTAINER_LOG4J_PROPERTIES_FILE);
  vargs.add("-D" + YarnConfiguration.YARN_APP_CONTAINER_LOG_DIR + "="
      + ApplicationConstants.LOG_DIR_EXPANSION_VAR);
  vargs.add("-D" + TezConfiguration.TEZ_ROOT_LOGGER_NAME + "=" + logLevel
      + "," + TezConfiguration.TEZ_CONTAINER_LOGGER_NAME);
}
 

@POST // not supported by WebAppProxyServlet, can only be called directly
@Path(PATH_PHYSICAL_PLAN_CONTAINERS + "/{containerId}/kill")
@Produces(MediaType.APPLICATION_JSON)
public JSONObject killContainer(@PathParam("containerId") String containerId)
{
  init();
  JSONObject response = new JSONObject();
  if (containerId.equals(System.getenv(ApplicationConstants.Environment.CONTAINER_ID.toString()))) {
    LOG.info("Received a kill request on application master container. Exiting.");
    new Thread()
    {
      @Override
      public void run()
      {
        try {
          Thread.sleep(3000);
          System.exit(1);
        } catch (InterruptedException ex) {
          LOG.info("Received interrupt, aborting exit.");
        }
      }

    }.start();
  } else {
    dagManager.stopContainer(containerId);
  }
  return response;
}
 

public ContainerInfo getContainerInfo()
{
  ContainerInfo ci = new ContainerInfo();
  ci.id = container.getExternalId();
  ci.host = container.host;
  ci.state = container.getState().name();
  ci.jvmName = this.jvmName;
  ci.numOperators = container.getOperators().size();
  ci.operators = new TreeMap<>();

  for (PTOperator ptOperator : container.getOperators()) {
    ci.operators.put(ptOperator.getId(), ptOperator.getName());
  }

  ci.memoryMBAllocated = container.getAllocatedMemoryMB();
  ci.lastHeartbeat = lastHeartbeatMillis;
  ci.memoryMBFree = this.memoryMBFree;
  ci.gcCollectionCount = this.gcCollectionCount;
  ci.gcCollectionTime = this.gcCollectionTime;
  ci.startedTime = container.getStartedTime();
  ci.finishedTime = container.getFinishedTime();
  if (this.container.nodeHttpAddress != null) {
    YarnConfiguration conf = new YarnConfiguration();
    ci.containerLogsUrl = ConfigUtils
        .getSchemePrefix(conf) + this.container.nodeHttpAddress + "/node/containerlogs/" + ci.id + "/" + System
        .getenv(ApplicationConstants.Environment.USER.toString());
    ci.rawContainerLogsUrl = ConfigUtils
        .getRawContainerLogsUrl(conf, container.nodeHttpAddress, container.getPlan().getLogicalPlan().getAttributes()
            .get(LogicalPlan.APPLICATION_ID), ci.id);
  }
  return ci;
}
 

@Test (timeout = 120000)
public void testSetClasspath() throws IOException {
  Configuration conf = new Configuration();
  conf.setBoolean(MRConfig.MAPREDUCE_APP_SUBMISSION_CROSS_PLATFORM, true);
  Job job = Job.getInstance(conf);
  Map<String, String> environment = new HashMap<String, String>();
  MRApps.setClasspath(environment, job.getConfiguration());
  assertTrue(environment.get("CLASSPATH").startsWith(
    ApplicationConstants.Environment.PWD.$$()
        + ApplicationConstants.CLASS_PATH_SEPARATOR));
  String yarnAppClasspath = job.getConfiguration().get(
      YarnConfiguration.YARN_APPLICATION_CLASSPATH,
      StringUtils.join(",",
          YarnConfiguration.DEFAULT_YARN_CROSS_PLATFORM_APPLICATION_CLASSPATH));
  if (yarnAppClasspath != null) {
    yarnAppClasspath =
        yarnAppClasspath.replaceAll(",\\s*",
          ApplicationConstants.CLASS_PATH_SEPARATOR).trim();
  }
  assertTrue(environment.get("CLASSPATH").contains(yarnAppClasspath));
  String mrAppClasspath = 
      job.getConfiguration().get(
          MRJobConfig.MAPREDUCE_APPLICATION_CLASSPATH,
          MRJobConfig.DEFAULT_MAPREDUCE_CROSS_PLATFORM_APPLICATION_CLASSPATH);
  if (mrAppClasspath != null) {
    mrAppClasspath =
        mrAppClasspath.replaceAll(",\\s*",
          ApplicationConstants.CLASS_PATH_SEPARATOR).trim();
  }
  assertTrue(environment.get("CLASSPATH").contains(mrAppClasspath));
}
 

@Test (timeout = 3000000)
public void testSetClasspathWithFramework() throws IOException {
  final String FRAMEWORK_NAME = "some-framework-name";
  final String FRAMEWORK_PATH = "some-framework-path#" + FRAMEWORK_NAME;
  Configuration conf = new Configuration();
  conf.setBoolean(MRConfig.MAPREDUCE_APP_SUBMISSION_CROSS_PLATFORM, true);
  conf.set(MRJobConfig.MAPREDUCE_APPLICATION_FRAMEWORK_PATH, FRAMEWORK_PATH);
  Map<String, String> env = new HashMap<String, String>();
  try {
    MRApps.setClasspath(env, conf);
    fail("Failed to catch framework path set without classpath change");
  } catch (IllegalArgumentException e) {
    assertTrue("Unexpected IllegalArgumentException",
        e.getMessage().contains("Could not locate MapReduce framework name '"
            + FRAMEWORK_NAME + "'"));
  }

  env.clear();
  final String FRAMEWORK_CLASSPATH = FRAMEWORK_NAME + "/*.jar";
  conf.set(MRJobConfig.MAPREDUCE_APPLICATION_CLASSPATH, FRAMEWORK_CLASSPATH);
  MRApps.setClasspath(env, conf);
  final String stdClasspath = StringUtils.join(ApplicationConstants.CLASS_PATH_SEPARATOR,
      Arrays.asList("job.jar/job.jar", "job.jar/classes/", "job.jar/lib/*",
          ApplicationConstants.Environment.PWD.$$() + "/*"));
  String expectedClasspath = StringUtils.join(ApplicationConstants.CLASS_PATH_SEPARATOR,
      Arrays.asList(ApplicationConstants.Environment.PWD.$$(),
          FRAMEWORK_CLASSPATH, stdClasspath));
  assertEquals("Incorrect classpath with framework and no user precedence",
      expectedClasspath, env.get("CLASSPATH"));

  env.clear();
  conf.setBoolean(MRJobConfig.MAPREDUCE_JOB_USER_CLASSPATH_FIRST, true);
  MRApps.setClasspath(env, conf);
  expectedClasspath = StringUtils.join(ApplicationConstants.CLASS_PATH_SEPARATOR,
      Arrays.asList(ApplicationConstants.Environment.PWD.$$(),
          stdClasspath, FRAMEWORK_CLASSPATH));
  assertEquals("Incorrect classpath with framework and user precedence",
      expectedClasspath, env.get("CLASSPATH"));
}
 

@Test (timeout = 120000)
public void testSetClasspathWithArchives () throws IOException {
  File testTGZ = new File(testWorkDir, "test.tgz");
  FileOutputStream out = new FileOutputStream(testTGZ);
  out.write(0);
  out.close();
  Configuration conf = new Configuration();
  conf.setBoolean(MRConfig.MAPREDUCE_APP_SUBMISSION_CROSS_PLATFORM, true);
  Job job = Job.getInstance(conf);
  conf = job.getConfiguration();
  String testTGZQualifiedPath = FileSystem.getLocal(conf).makeQualified(new Path(
    testTGZ.getAbsolutePath())).toString();
  conf.set(MRJobConfig.CLASSPATH_ARCHIVES, testTGZQualifiedPath);
  conf.set(MRJobConfig.CACHE_ARCHIVES, testTGZQualifiedPath + "#testTGZ");
  Map<String, String> environment = new HashMap<String, String>();
  MRApps.setClasspath(environment, conf);
  assertTrue(environment.get("CLASSPATH").startsWith(
    ApplicationConstants.Environment.PWD.$$() + ApplicationConstants.CLASS_PATH_SEPARATOR));
  String confClasspath = job.getConfiguration().get(
      YarnConfiguration.YARN_APPLICATION_CLASSPATH,
      StringUtils.join(",",
          YarnConfiguration.DEFAULT_YARN_CROSS_PLATFORM_APPLICATION_CLASSPATH));
  if (confClasspath != null) {
    confClasspath = confClasspath.replaceAll(",\\s*", ApplicationConstants.CLASS_PATH_SEPARATOR)
      .trim();
  }
  assertTrue(environment.get("CLASSPATH").contains(confClasspath));
  assertTrue(environment.get("CLASSPATH").contains("testTGZ"));
}
 

@Test (timeout = 120000)
public void testSetClasspath() throws IOException {
  Configuration conf = new Configuration();
  conf.setBoolean(MRConfig.MAPREDUCE_APP_SUBMISSION_CROSS_PLATFORM, true);
  Job job = Job.getInstance(conf);
  Map<String, String> environment = new HashMap<String, String>();
  MRApps.setClasspath(environment, job.getConfiguration());
  assertTrue(environment.get("CLASSPATH").startsWith(
    ApplicationConstants.Environment.PWD.$$()
        + ApplicationConstants.CLASS_PATH_SEPARATOR));
  String yarnAppClasspath = job.getConfiguration().get(
      YarnConfiguration.YARN_APPLICATION_CLASSPATH,
      StringUtils.join(",",
          YarnConfiguration.DEFAULT_YARN_CROSS_PLATFORM_APPLICATION_CLASSPATH));
  if (yarnAppClasspath != null) {
    yarnAppClasspath =
        yarnAppClasspath.replaceAll(",\\s*",
          ApplicationConstants.CLASS_PATH_SEPARATOR).trim();
  }
  assertTrue(environment.get("CLASSPATH").contains(yarnAppClasspath));
  String mrAppClasspath = 
      job.getConfiguration().get(
          MRJobConfig.MAPREDUCE_APPLICATION_CLASSPATH,
          MRJobConfig.DEFAULT_MAPREDUCE_CROSS_PLATFORM_APPLICATION_CLASSPATH);
  if (mrAppClasspath != null) {
    mrAppClasspath =
        mrAppClasspath.replaceAll(",\\s*",
          ApplicationConstants.CLASS_PATH_SEPARATOR).trim();
  }
  assertTrue(environment.get("CLASSPATH").contains(mrAppClasspath));
}
 

private ContainerLaunchContext createTaskExecutorLaunchContext(
	String containerId,
	String host,
	TaskExecutorProcessSpec taskExecutorProcessSpec) throws Exception {

	// init the ContainerLaunchContext
	final String currDir = env.get(ApplicationConstants.Environment.PWD.key());

	final ContaineredTaskManagerParameters taskManagerParameters =
			ContaineredTaskManagerParameters.create(flinkConfig, taskExecutorProcessSpec);

	log.info("TaskExecutor {} will be started on {} with {}.",
		containerId,
		host,
		taskExecutorProcessSpec);

	final Configuration taskManagerConfig = BootstrapTools.cloneConfiguration(flinkConfig);
	taskManagerConfig.set(TaskManagerOptions.TASK_MANAGER_RESOURCE_ID, containerId);

	final String taskManagerDynamicProperties =
		BootstrapTools.getDynamicPropertiesAsString(flinkClientConfig, taskManagerConfig);

	log.debug("TaskManager configuration: {}", taskManagerConfig);

	ContainerLaunchContext taskExecutorLaunchContext = Utils.createTaskExecutorContext(
		flinkConfig,
		yarnConfig,
		env,
		taskManagerParameters,
		taskManagerDynamicProperties,
		currDir,
		YarnTaskExecutorRunner.class,
		log);

	taskExecutorLaunchContext.getEnvironment()
			.put(ENV_FLINK_NODE_ID, host);
	return taskExecutorLaunchContext;
}