Java Code Examples for org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext#getAMContainerSpec()

The following examples show how to use org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext#getAMContainerSpec() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: AMLauncher.java    From hadoop with Apache License 2.0 6 votes vote down vote up 
private ContainerLaunchContext createAMContainerLaunchContext(
    ApplicationSubmissionContext applicationMasterContext,
    ContainerId containerID) throws IOException {

  // Construct the actual Container
  ContainerLaunchContext container = 
      applicationMasterContext.getAMContainerSpec();
  LOG.info("Command to launch container "
      + containerID
      + " : "
      + StringUtils.arrayToString(container.getCommands().toArray(
          new String[0])));
  
  // Finalize the container
  setupTokens(container, containerID);
  
  return container;
}
Example 2
Source File: TestYARNRunner.java    From hadoop with Apache License 2.0 6 votes vote down vote up 
@Test(timeout=20000)
public void testAMProfiler() throws Exception {
  JobConf jobConf = new JobConf();

  jobConf.setBoolean(MRJobConfig.MR_AM_PROFILE, true);

  YARNRunner yarnRunner = new YARNRunner(jobConf);

  ApplicationSubmissionContext submissionContext =
      buildSubmitContext(yarnRunner, jobConf);

  ContainerLaunchContext containerSpec = submissionContext.getAMContainerSpec();
  List<String> commands = containerSpec.getCommands();

  for(String command : commands) {
    if (command != null) {
      if (command.contains(PROFILE_PARAMS)) {
        return;
      }
    }
  }
  throw new IllegalStateException("Profiler opts not found!");
}
Example 3
Source File: AMLauncher.java    From big-c with Apache License 2.0 6 votes vote down vote up 
private ContainerLaunchContext createAMContainerLaunchContext(
    ApplicationSubmissionContext applicationMasterContext,
    ContainerId containerID) throws IOException {

  // Construct the actual Container
  ContainerLaunchContext container = 
      applicationMasterContext.getAMContainerSpec();
  LOG.info("Command to launch container "
      + containerID
      + " : "
      + StringUtils.arrayToString(container.getCommands().toArray(
          new String[0])));
  
  // Finalize the container
  setupTokens(container, containerID);
  
  return container;
}
Example 4
Source File: TestYARNRunner.java    From big-c with Apache License 2.0 6 votes vote down vote up 
@Test(timeout=20000)
public void testAMProfiler() throws Exception {
  JobConf jobConf = new JobConf();

  jobConf.setBoolean(MRJobConfig.MR_AM_PROFILE, true);

  YARNRunner yarnRunner = new YARNRunner(jobConf);

  ApplicationSubmissionContext submissionContext =
      buildSubmitContext(yarnRunner, jobConf);

  ContainerLaunchContext containerSpec = submissionContext.getAMContainerSpec();
  List<String> commands = containerSpec.getCommands();

  for(String command : commands) {
    if (command != null) {
      if (command.contains(PROFILE_PARAMS)) {
        return;
      }
    }
  }
  throw new IllegalStateException("Profiler opts not found!");
}
Example 5
Source File: TestYARNRunner.java    From hadoop with Apache License 2.0 5 votes vote down vote up 
@Test
public void testAMStandardEnv() throws Exception {
  final String ADMIN_LIB_PATH = "foo";
  final String USER_LIB_PATH = "bar";
  final String USER_SHELL = "shell";
  JobConf jobConf = new JobConf();

  jobConf.set(MRJobConfig.MR_AM_ADMIN_USER_ENV, "LD_LIBRARY_PATH=" +
      ADMIN_LIB_PATH);
  jobConf.set(MRJobConfig.MR_AM_ENV, "LD_LIBRARY_PATH="
      + USER_LIB_PATH);
  jobConf.set(MRJobConfig.MAPRED_ADMIN_USER_SHELL, USER_SHELL);

  YARNRunner yarnRunner = new YARNRunner(jobConf);
  ApplicationSubmissionContext appSubCtx =
      buildSubmitContext(yarnRunner, jobConf);

  // make sure PWD is first in the lib path
  ContainerLaunchContext clc = appSubCtx.getAMContainerSpec();
  Map<String, String> env = clc.getEnvironment();
  String libPath = env.get(Environment.LD_LIBRARY_PATH.name());
  assertNotNull("LD_LIBRARY_PATH not set", libPath);
  String cps = jobConf.getBoolean(
      MRConfig.MAPREDUCE_APP_SUBMISSION_CROSS_PLATFORM,
      MRConfig.DEFAULT_MAPREDUCE_APP_SUBMISSION_CROSS_PLATFORM)
      ? ApplicationConstants.CLASS_PATH_SEPARATOR : File.pathSeparator;
  assertEquals("Bad AM LD_LIBRARY_PATH setting",
      MRApps.crossPlatformifyMREnv(conf, Environment.PWD)
      + cps + ADMIN_LIB_PATH + cps + USER_LIB_PATH, libPath);

  // make sure SHELL is set
  String shell = env.get(Environment.SHELL.name());
  assertNotNull("SHELL not set", shell);
  assertEquals("Bad SHELL setting", USER_SHELL, shell);
}
Example 6
Source File: TestYARNRunner.java    From big-c with Apache License 2.0 5 votes vote down vote up 
@Test
public void testAMStandardEnv() throws Exception {
  final String ADMIN_LIB_PATH = "foo";
  final String USER_LIB_PATH = "bar";
  final String USER_SHELL = "shell";
  JobConf jobConf = new JobConf();

  jobConf.set(MRJobConfig.MR_AM_ADMIN_USER_ENV, "LD_LIBRARY_PATH=" +
      ADMIN_LIB_PATH);
  jobConf.set(MRJobConfig.MR_AM_ENV, "LD_LIBRARY_PATH="
      + USER_LIB_PATH);
  jobConf.set(MRJobConfig.MAPRED_ADMIN_USER_SHELL, USER_SHELL);

  YARNRunner yarnRunner = new YARNRunner(jobConf);
  ApplicationSubmissionContext appSubCtx =
      buildSubmitContext(yarnRunner, jobConf);

  // make sure PWD is first in the lib path
  ContainerLaunchContext clc = appSubCtx.getAMContainerSpec();
  Map<String, String> env = clc.getEnvironment();
  String libPath = env.get(Environment.LD_LIBRARY_PATH.name());
  assertNotNull("LD_LIBRARY_PATH not set", libPath);
  String cps = jobConf.getBoolean(
      MRConfig.MAPREDUCE_APP_SUBMISSION_CROSS_PLATFORM,
      MRConfig.DEFAULT_MAPREDUCE_APP_SUBMISSION_CROSS_PLATFORM)
      ? ApplicationConstants.CLASS_PATH_SEPARATOR : File.pathSeparator;
  assertEquals("Bad AM LD_LIBRARY_PATH setting",
      MRApps.crossPlatformifyMREnv(conf, Environment.PWD)
      + cps + ADMIN_LIB_PATH + cps + USER_LIB_PATH, libPath);

  // make sure SHELL is set
  String shell = env.get(Environment.SHELL.name());
  assertNotNull("SHELL not set", shell);
  assertEquals("Bad SHELL setting", USER_SHELL, shell);
}
Example 7
Source File: TestTezClientUtils.java    From tez with Apache License 2.0 5 votes vote down vote up 
@Test(timeout = 5000)
public void testSessionTokenInAmClc() throws IOException, YarnException {

  TezConfiguration tezConf = new TezConfiguration();
  tezConf.set(TezConfiguration.TEZ_AM_STAGING_DIR, STAGING_DIR.getAbsolutePath());

  ApplicationId appId = ApplicationId.newInstance(1000, 1);
  DAG dag = DAG.create("testdag");
  dag.addVertex(Vertex.create("testVertex", ProcessorDescriptor.create("processorClassname"), 1)
      .setTaskLaunchCmdOpts("initialLaunchOpts"));

  Credentials credentials = new Credentials();
  JobTokenSecretManager jobTokenSecretManager = new JobTokenSecretManager();
  TezClientUtils.createSessionToken(appId.toString(), jobTokenSecretManager, credentials);
  Token<JobTokenIdentifier> jobToken = TokenCache.getSessionToken(credentials);
  assertNotNull(jobToken);

  AMConfiguration amConf =
      new AMConfiguration(tezConf, new HashMap<String, LocalResource>(), credentials);
  ApplicationSubmissionContext appSubmissionContext =
      TezClientUtils.createApplicationSubmissionContext(appId, dag, "amName", amConf,
          new HashMap<String, LocalResource>(), credentials, false, new TezApiVersionInfo(),
          null, null);

  ContainerLaunchContext amClc = appSubmissionContext.getAMContainerSpec();
  Map<String, ByteBuffer> amServiceData = amClc.getServiceData();
  assertNotNull(amServiceData);
  assertEquals(1, amServiceData.size());

  DataInputByteBuffer dibb = new DataInputByteBuffer();
  dibb.reset(amServiceData.values().iterator().next());
  Token<JobTokenIdentifier> jtSent = new Token<JobTokenIdentifier>();
  jtSent.readFields(dibb);

  assertTrue(Arrays.equals(jobToken.getIdentifier(), jtSent.getIdentifier()));
}
Example 8
Source File: TestYARNRunner.java    From hadoop with Apache License 2.0 4 votes vote down vote up 
@Test(timeout=20000)
public void testAMAdminCommandOpts() throws Exception {
  JobConf jobConf = new JobConf();
  
  jobConf.set(MRJobConfig.MR_AM_ADMIN_COMMAND_OPTS, "-Djava.net.preferIPv4Stack=true");
  jobConf.set(MRJobConfig.MR_AM_COMMAND_OPTS, "-Xmx1024m");
  
  YARNRunner yarnRunner = new YARNRunner(jobConf);
  
  ApplicationSubmissionContext submissionContext =
      buildSubmitContext(yarnRunner, jobConf);
  
  ContainerLaunchContext containerSpec = submissionContext.getAMContainerSpec();
  List<String> commands = containerSpec.getCommands();
  
  int index = 0;
  int adminIndex = 0;
  int adminPos = -1;
  int userIndex = 0;
  int userPos = -1;
  
  for(String command : commands) {
    if(command != null) {
      assertFalse("Profiler should be disabled by default",
          command.contains(PROFILE_PARAMS));
      adminPos = command.indexOf("-Djava.net.preferIPv4Stack=true");
      if(adminPos >= 0)
        adminIndex = index;
      
      userPos = command.indexOf("-Xmx1024m");
      if(userPos >= 0)
        userIndex = index;
    }
    
    index++;
  }
  
  // Check both admin java opts and user java opts are in the commands
  assertTrue("AM admin command opts not in the commands.", adminPos > 0);
  assertTrue("AM user command opts not in the commands.", userPos > 0);
  
  // Check the admin java opts is before user java opts in the commands
  if(adminIndex == userIndex) {
    assertTrue("AM admin command opts is after user command opts.", adminPos < userPos);
  } else {
    assertTrue("AM admin command opts is after user command opts.", adminIndex < userIndex);
  }
}
Example 9
Source File: TestYARNRunner.java    From big-c with Apache License 2.0 4 votes vote down vote up 
@Test(timeout=20000)
public void testAMAdminCommandOpts() throws Exception {
  JobConf jobConf = new JobConf();
  
  jobConf.set(MRJobConfig.MR_AM_ADMIN_COMMAND_OPTS, "-Djava.net.preferIPv4Stack=true");
  jobConf.set(MRJobConfig.MR_AM_COMMAND_OPTS, "-Xmx1024m");
  
  YARNRunner yarnRunner = new YARNRunner(jobConf);
  
  ApplicationSubmissionContext submissionContext =
      buildSubmitContext(yarnRunner, jobConf);
  
  ContainerLaunchContext containerSpec = submissionContext.getAMContainerSpec();
  List<String> commands = containerSpec.getCommands();
  
  int index = 0;
  int adminIndex = 0;
  int adminPos = -1;
  int userIndex = 0;
  int userPos = -1;
  
  for(String command : commands) {
    if(command != null) {
      assertFalse("Profiler should be disabled by default",
          command.contains(PROFILE_PARAMS));
      adminPos = command.indexOf("-Djava.net.preferIPv4Stack=true");
      if(adminPos >= 0)
        adminIndex = index;
      
      userPos = command.indexOf("-Xmx1024m");
      if(userPos >= 0)
        userIndex = index;
    }
    
    index++;
  }
  
  // Check both admin java opts and user java opts are in the commands
  assertTrue("AM admin command opts not in the commands.", adminPos > 0);
  assertTrue("AM user command opts not in the commands.", userPos > 0);
  
  // Check the admin java opts is before user java opts in the commands
  if(adminIndex == userIndex) {
    assertTrue("AM admin command opts is after user command opts.", adminPos < userPos);
  } else {
    assertTrue("AM admin command opts is after user command opts.", adminIndex < userIndex);
  }
}