Java Code Examples for org.apache.hadoop.yarn.util.ConverterUtils#toApplicationAttemptId()

private static CommitterEventHandler createCommitterEventHandler(
    Dispatcher dispatcher, OutputCommitter committer) {
  final SystemClock clock = new SystemClock();
  AppContext appContext = mock(AppContext.class);
  when(appContext.getEventHandler()).thenReturn(
      dispatcher.getEventHandler());
  when(appContext.getClock()).thenReturn(clock);
  RMHeartbeatHandler heartbeatHandler = new RMHeartbeatHandler() {
    @Override
    public long getLastHeartbeatTime() {
      return clock.getTime();
    }
    @Override
    public void runOnNextHeartbeat(Runnable callback) {
      callback.run();
    }
  };
  ApplicationAttemptId id = 
    ConverterUtils.toApplicationAttemptId("appattempt_1234567890000_0001_0");
  when(appContext.getApplicationID()).thenReturn(id.getApplicationId());
  when(appContext.getApplicationAttemptId()).thenReturn(id);
  CommitterEventHandler handler =
      new CommitterEventHandler(appContext, committer, heartbeatHandler);
  dispatcher.register(CommitterEventType.class, handler);
  return handler;
}
 

@Override
protected void modifyAppState() throws Exception {
  // imitate appAttemptFile1 is still .new, but old one is deleted
  String appAttemptIdStr1 = "appattempt_1352994193343_0001_000001";
  ApplicationAttemptId attemptId1 =
      ConverterUtils.toApplicationAttemptId(appAttemptIdStr1);
  Path appDir =
          fsTester.store.getAppDir(attemptId1.getApplicationId().toString());
  Path appAttemptFile1 =
      new Path(appDir, attemptId1.toString() + ".new");
  FileSystemRMStateStore fileSystemRMStateStore =
      (FileSystemRMStateStore) fsTester.getRMStateStore();
  fileSystemRMStateStore.renameFile(appAttemptFile1,
          new Path(appAttemptFile1.getParent(),
                  appAttemptFile1.getName() + ".new"));
}
 

@Override
protected void modifyAppState() throws Exception {
  // imitate appAttemptFile1 is still .new, but old one is deleted
  String appAttemptIdStr1 = "appattempt_1352994193343_0001_000001";
  ApplicationAttemptId attemptId1 =
      ConverterUtils.toApplicationAttemptId(appAttemptIdStr1);
  Path appDir =
          fsTester.store.getAppDir(attemptId1.getApplicationId().toString());
  Path appAttemptFile1 =
      new Path(appDir, attemptId1.toString() + ".new");
  FileSystemRMStateStore fileSystemRMStateStore =
      (FileSystemRMStateStore) fsTester.getRMStateStore();
  fileSystemRMStateStore.renameFile(appAttemptFile1,
          new Path(appAttemptFile1.getParent(),
                  appAttemptFile1.getName() + ".new"));
}
 

@Test
public void testMRAppMasterForDifferentUser() throws IOException,
    InterruptedException {
  String applicationAttemptIdStr = "appattempt_1317529182569_0004_000001";
  String containerIdStr = "container_1317529182569_0004_000001_1";
  
  String userName = "TestAppMasterUser";
  ApplicationAttemptId applicationAttemptId = ConverterUtils
      .toApplicationAttemptId(applicationAttemptIdStr);
  ContainerId containerId = ConverterUtils.toContainerId(containerIdStr);
  MRAppMasterTest appMaster =
      new MRAppMasterTest(applicationAttemptId, containerId, "host", -1, -1,
          System.currentTimeMillis());
  JobConf conf = new JobConf();
  conf.set(MRJobConfig.MR_AM_STAGING_DIR, stagingDir);
  MRAppMaster.initAndStartAppMaster(appMaster, conf, userName);
  Path userPath = new Path(stagingDir, userName);
  Path userStagingPath = new Path(userPath, ".staging");
  assertEquals(userStagingPath.toString(),
    appMaster.stagingDirPath.toString());
}
 

@Test(timeout = 60000)
public void testFSRMStateStore() throws Exception {
  HdfsConfiguration conf = new HdfsConfiguration();
  MiniDFSCluster cluster =
          new MiniDFSCluster.Builder(conf).numDataNodes(1).build();
  try {
    fsTester = new TestFSRMStateStoreTester(cluster, false);
    // If the state store is FileSystemRMStateStore then add corrupted entry.
    // It should discard the entry and remove it from file system.
    FSDataOutputStream fsOut = null;
    FileSystemRMStateStore fileSystemRMStateStore =
            (FileSystemRMStateStore) fsTester.getRMStateStore();
    String appAttemptIdStr3 = "appattempt_1352994193343_0001_000003";
    ApplicationAttemptId attemptId3 =
            ConverterUtils.toApplicationAttemptId(appAttemptIdStr3);
    Path appDir =
            fsTester.store.getAppDir(attemptId3.getApplicationId().toString());
    Path tempAppAttemptFile =
            new Path(appDir, attemptId3.toString() + ".tmp");
    fsOut = fileSystemRMStateStore.fs.create(tempAppAttemptFile, false);
    fsOut.write("Some random data ".getBytes());
    fsOut.close();

    testRMAppStateStore(fsTester);
    Assert.assertFalse(fsTester.workingDirPathURI
            .getFileSystem(conf).exists(tempAppAttemptFile));
    testRMDTSecretManagerStateStore(fsTester);
    testCheckVersion(fsTester);
    testEpoch(fsTester);
    testAppDeletion(fsTester);
    testDeleteStore(fsTester);
    testAMRMTokenSecretManagerStateStore(fsTester);
  } finally {
    cluster.shutdown();
  }
}
 

@Test
public void testDuplicateRMAppDeletion() throws Exception {
  TestZKRMStateStoreTester zkTester = new TestZKRMStateStoreTester();
  long submitTime = System.currentTimeMillis();
  long startTime = System.currentTimeMillis() + 1234;
  RMStateStore store = zkTester.getRMStateStore();
  TestDispatcher dispatcher = new TestDispatcher();
  store.setRMDispatcher(dispatcher);

  ApplicationAttemptId attemptIdRemoved = ConverterUtils
      .toApplicationAttemptId("appattempt_1352994193343_0002_000001");
  ApplicationId appIdRemoved = attemptIdRemoved.getApplicationId();
  storeApp(store, appIdRemoved, submitTime, startTime);
  storeAttempt(store, attemptIdRemoved,
      "container_1352994193343_0002_01_000001", null, null, dispatcher);

  ApplicationSubmissionContext context =
      new ApplicationSubmissionContextPBImpl();
  context.setApplicationId(appIdRemoved);
  ApplicationStateData appStateRemoved =
      ApplicationStateData.newInstance(
          submitTime, startTime, context, "user1");
  appStateRemoved.attempts.put(attemptIdRemoved, null);
  store.removeApplicationStateInternal(appStateRemoved);
  try {
    store.removeApplicationStateInternal(appStateRemoved);
  } catch (KeeperException.NoNodeException nne) {
    Assert.fail("NoNodeException should not happen.");
  }
  store.close();
}
 

public void fromProto(ContainerStoppedProto proto) {
  this.containerId = ConverterUtils.toContainerId(proto.getContainerId());
  stopTime = proto.getStopTime();
  exitStatus = proto.getExitStatus();
  this.applicationAttemptId = ConverterUtils.toApplicationAttemptId(
      proto.getApplicationAttemptId());
}
 

public void fromProto(DAGSubmittedProto proto) {
  this.dagID = TezDAGID.fromString(proto.getDagId());
  this.dagPlan = proto.getDagPlan();
  this.dagName = this.dagPlan.getName();
  this.submitTime = proto.getSubmitTime();
  this.applicationAttemptId = ConverterUtils.toApplicationAttemptId(
      proto.getApplicationAttemptId());
  if (proto.hasQueueName()) {
    this.queueName = proto.getQueueName();
  }
  if (proto.hasCumulativeAdditionalAmResources()) {
    this.cumulativeAdditionalLocalResources = DagTypeConverters.convertFromPlanLocalResources(proto
        .getCumulativeAdditionalAmResources());
  }
}
 

@Test
public void testMRAppMasterSuccessLock() throws IOException,
    InterruptedException {
  String applicationAttemptIdStr = "appattempt_1317529182569_0004_000002";
  String containerIdStr = "container_1317529182569_0004_000002_1";
  String userName = "TestAppMasterUser";
  JobConf conf = new JobConf();
  conf.set(MRJobConfig.MR_AM_STAGING_DIR, stagingDir);
  ApplicationAttemptId applicationAttemptId = ConverterUtils
      .toApplicationAttemptId(applicationAttemptIdStr);
  JobId jobId =  TypeConverter.toYarn(
      TypeConverter.fromYarn(applicationAttemptId.getApplicationId()));
  Path start = MRApps.getStartJobCommitFile(conf, userName, jobId);
  Path end = MRApps.getEndJobCommitSuccessFile(conf, userName, jobId);
  FileSystem fs = FileSystem.get(conf);
  fs.create(start).close();
  fs.create(end).close();
  ContainerId containerId = ConverterUtils.toContainerId(containerIdStr);
  MRAppMaster appMaster =
      new MRAppMasterTest(applicationAttemptId, containerId, "host", -1, -1,
          System.currentTimeMillis(), false, false);
  boolean caught = false;
  try {
    MRAppMaster.initAndStartAppMaster(appMaster, conf, userName);
  } catch (IOException e) {
    //The IO Exception is expected
    LOG.info("Caught expected Exception", e);
    caught = true;
  }
  assertTrue(caught);
  assertTrue(appMaster.errorHappenedShutDown);
  assertEquals(JobStateInternal.SUCCEEDED, appMaster.forcedState);
  appMaster.stop();

  // verify the final status is SUCCEEDED
  verifyFailedStatus((MRAppMasterTest)appMaster, "SUCCEEDED");
}
 

private ApplicationAttemptStateData createAttemptState(String itemName,
    byte[] data) throws IOException {
  ApplicationAttemptId attemptId =
      ConverterUtils.toApplicationAttemptId(itemName);
  ApplicationAttemptStateDataPBImpl attemptState =
      new ApplicationAttemptStateDataPBImpl(
          ApplicationAttemptStateDataProto.parseFrom(data));
  if (!attemptId.equals(attemptState.getAttemptId())) {
    throw new YarnRuntimeException("The database entry for " + attemptId
        + " contains data for " + attemptState.getAttemptId());
  }
  return attemptState;
}
 

protected static ApplicationAttemptId parseApplicationAttemptId(
    String appAttemptId) {
  if (appAttemptId == null || appAttemptId.isEmpty()) {
    throw new NotFoundException("appAttemptId, " + appAttemptId
        + ", is empty or null");
  }
  ApplicationAttemptId aaid =
      ConverterUtils.toApplicationAttemptId(appAttemptId);
  if (aaid == null) {
    throw new NotFoundException("appAttemptId is null");
  }
  return aaid;
}
 

public void fromProto(AMStartedProto proto) {
  this.applicationAttemptId =
      ConverterUtils.toApplicationAttemptId(proto.getApplicationAttemptId());
  this.startTime = proto.getStartTime();
}
 

/**
 * @return the ApplicationAttemptId
 */
public ApplicationAttemptId getAppAttemptId() {
  return ConverterUtils.toApplicationAttemptId(datum.applicationAttemptId
      .toString());
}
 

public void fromProto(AMStartedProto proto) {
  this.applicationAttemptId =
      ConverterUtils.toApplicationAttemptId(proto.getApplicationAttemptId());
  this.startTime = proto.getStartTime();
}
 

/**
 * Parse command line options
 *
 * @param args Command line args
 * @return Whether init successful and run should be invoked
 * @throws ParseException
 * @throws IOException
 */
public boolean init(String[] args) throws ParseException, IOException {
  CommandLine cliParser = AMOptions.parse(new GnuParser(), args);

  //Check whether customer log4j.properties file exists
  if (fileExist(log4jPath)) {
    try {
      Log4jPropertyHelper.updateLog4jConfiguration(ApplicationMaster.class,
              log4jPath);
    } catch (Exception e) {
      LOG.warn("Can not set up custom log4j properties. " + e);
    }
  }

  if (AMOptions.HELP.has(cliParser)) {
    AMOptions.printHelp();
    return false;
  }


  zkQuorum = AMOptions.ZK_QUORUM.get(cliParser);
  zkRoot = AMOptions.ZK_ROOT.get(cliParser);
  appJarPath = new Path(AMOptions.APP_JAR_PATH.get(cliParser));

  Map<String, String> envs = System.getenv();

  if (!envs.containsKey(Environment.CONTAINER_ID.name())) {
    if (AMOptions.APP_ATTEMPT_ID.has(cliParser)) {
      String appIdStr = AMOptions.APP_ATTEMPT_ID.get(cliParser, "");
      appAttemptID = ConverterUtils.toApplicationAttemptId(appIdStr);
    } else {
      throw new IllegalArgumentException(
              "Application Attempt Id not set in the environment");
    }
  } else {
    ContainerId containerId = ConverterUtils.toContainerId(envs
            .get(Environment.CONTAINER_ID.name()));
    appAttemptID = containerId.getApplicationAttemptId();
  }

  if (!envs.containsKey(ApplicationConstants.APP_SUBMIT_TIME_ENV)) {
    throw new RuntimeException(ApplicationConstants.APP_SUBMIT_TIME_ENV
            + " not set in the environment");
  }
  if (!envs.containsKey(Environment.NM_HOST.name())) {
    throw new RuntimeException(Environment.NM_HOST.name()
            + " not set in the environment");
  }
  if (!envs.containsKey(Environment.NM_HTTP_PORT.name())) {
    throw new RuntimeException(Environment.NM_HTTP_PORT
            + " not set in the environment");
  }
  if (!envs.containsKey(Environment.NM_PORT.name())) {
    throw new RuntimeException(Environment.NM_PORT.name()
            + " not set in the environment");
  }

  LOG.info("Application master for app" + ", appId="
          + appAttemptID.getApplicationId().getId() + ", clustertimestamp="
          + appAttemptID.getApplicationId().getClusterTimestamp()
          + ", attemptId=" + appAttemptID.getAttemptId());


  if (cliParser.hasOption("shell_env")) {
    String shellEnvs[] = cliParser.getOptionValues("shell_env");
    for (String env : shellEnvs) {
      env = env.trim();
      int index = env.indexOf('=');
      if (index == -1) {
        shellEnv.put(env, "");
        continue;
      }
      String key = env.substring(0, index);
      String val = "";
      if (index < (env.length() - 1)) {
        val = env.substring(index + 1);
      }
      shellEnv.put(key, val);
    }
  }


  if (envs.containsKey(Constants.TIMELINEDOMAIN)) {
    domainId = envs.get(Constants.TIMELINEDOMAIN);
  }
  return true;
}
 

@Test
public void testFailure() throws Exception {
  AppContext mockContext = mock(AppContext.class);
  OutputCommitter mockCommitter = mock(OutputCommitter.class);
  Clock mockClock = mock(Clock.class);
  
  CommitterEventHandler handler = new CommitterEventHandler(mockContext, 
      mockCommitter, new TestingRMHeartbeatHandler());
  YarnConfiguration conf = new YarnConfiguration();
  conf.set(MRJobConfig.MR_AM_STAGING_DIR, stagingDir);
  JobContext mockJobContext = mock(JobContext.class);
  ApplicationAttemptId attemptid = 
    ConverterUtils.toApplicationAttemptId("appattempt_1234567890000_0001_0");
  JobId jobId =  TypeConverter.toYarn(
      TypeConverter.fromYarn(attemptid.getApplicationId()));
  
  WaitForItHandler waitForItHandler = new WaitForItHandler();
  
  when(mockContext.getApplicationID()).thenReturn(attemptid.getApplicationId());
  when(mockContext.getApplicationAttemptId()).thenReturn(attemptid);
  when(mockContext.getEventHandler()).thenReturn(waitForItHandler);
  when(mockContext.getClock()).thenReturn(mockClock);
  
  doThrow(new YarnRuntimeException("Intentional Failure")).when(mockCommitter)
    .commitJob(any(JobContext.class));
  
  handler.init(conf);
  handler.start();
  try {
    handler.handle(new CommitterJobCommitEvent(jobId, mockJobContext));

    String user = UserGroupInformation.getCurrentUser().getShortUserName();
    Path startCommitFile = MRApps.getStartJobCommitFile(conf, user, jobId);
    Path endCommitSuccessFile = MRApps.getEndJobCommitSuccessFile(conf, user, 
        jobId);
    Path endCommitFailureFile = MRApps.getEndJobCommitFailureFile(conf, user, 
        jobId);

    Event e = waitForItHandler.getAndClearEvent();
    assertNotNull(e);
    assertTrue(e instanceof JobCommitFailedEvent);
    FileSystem fs = FileSystem.get(conf);
    assertTrue(fs.exists(startCommitFile));
    assertFalse(fs.exists(endCommitSuccessFile));
    assertTrue(fs.exists(endCommitFailureFile));
    verify(mockCommitter).commitJob(any(JobContext.class));
  } finally {
    handler.stop();
  }
}
 

public void fromProto(ContainerLaunchedProto proto) {
  this.containerId = ConverterUtils.toContainerId(proto.getContainerId());
  launchTime = proto.getLaunchTime();
  this.applicationAttemptId = ConverterUtils.toApplicationAttemptId(
      proto.getApplicationAttemptId());
}
 

public void fromProto(AMLaunchedProto proto) {
  this.applicationAttemptId =
      ConverterUtils.toApplicationAttemptId(proto.getApplicationAttemptId());
  this.launchTime = proto.getLaunchTime();
  this.appSubmitTime = proto.getAppSubmitTime();
}
 

/**
 * Parse command line options
 *
 * @param args Command line args
 * @return Whether init successful and run should be invoked
 * @throws org.apache.commons.cli.ParseException
 * @throws java.io.IOException
 */
public boolean init(String[] args) throws Exception {
  Options opts = new Options();
  opts.addOption("app_attempt_id", true,
      "App Attempt ID. Not to be used unless for testing purposes");
  opts.addOption("shell_env", true,
      "Environment for shell script. Specified as env_key=env_val pairs");
  opts.addOption("container_memory", true,
      "Amount of memory in MB to be requested to run the shell command");
  opts.addOption("container_vcores", true,
      "Amount of virtual cores to be requested to run the shell command");
  opts.addOption("num_containers", true,
      "No. of containers on which the shell command needs to be executed");
  opts.addOption("priority", true, "Application Priority. Default 0");
  opts.addOption("help", false, "Print usage");

  CommandLine cliParser = new GnuParser().parse(opts, args);

  Map<String, String> envs = System.getenv();

  if (!envs.containsKey(ApplicationConstants.Environment.CONTAINER_ID.name())) {
    if (cliParser.hasOption("app_attempt_id")) {
      String appIdStr = cliParser.getOptionValue("app_attempt_id", "");
      appAttemptID = ConverterUtils.toApplicationAttemptId(appIdStr);
    } else {
      throw new IllegalArgumentException(
          "Application Attempt Id not set in the environment");
    }
  } else {
    ContainerId containerId = ConverterUtils.toContainerId(envs
        .get(ApplicationConstants.Environment.CONTAINER_ID.name()));
    appAttemptID = containerId.getApplicationAttemptId();
  }

  if (!envs.containsKey(ApplicationConstants.APP_SUBMIT_TIME_ENV)) {
    throw new RuntimeException(ApplicationConstants.APP_SUBMIT_TIME_ENV + " not set in the environment");
  }
  if (!envs.containsKey(ApplicationConstants.Environment.NM_HOST.name())) {
    throw new RuntimeException(ApplicationConstants.Environment.NM_HOST.name() + " not set in the environment");
  }
  if (!envs.containsKey(ApplicationConstants.Environment.NM_HTTP_PORT.name())) {
    throw new RuntimeException(ApplicationConstants.Environment.NM_HTTP_PORT + " not set in the environment");
  }
  if (!envs.containsKey(ApplicationConstants.Environment.NM_PORT.name())) {
    throw new RuntimeException(ApplicationConstants.Environment.NM_PORT.name() + " not set in the environment");
  }

  if (envs.containsKey(Constants.AM_JAR_PATH)) {
    appJarPath = envs.get(Constants.AM_JAR_PATH);

    if (envs.containsKey(Constants.AM_JAR_TIMESTAMP)) {
      appJarTimestamp = Long.valueOf(envs.get(Constants.AM_JAR_TIMESTAMP));
    }
    if (envs.containsKey(Constants.AM_JAR_LENGTH)) {
      appJarPathLen = Long.valueOf(envs.get(Constants.AM_JAR_LENGTH));
    }

    if (!appJarPath.isEmpty() && (appJarTimestamp <= 0 || appJarPathLen <= 0)) {
      LOG.error("Illegal values in env for shell script path" + ", path="
          + appJarPath + ", len=" + appJarPathLen + ", timestamp="+ appJarTimestamp);
      throw new IllegalArgumentException(
          "Illegal values in env for shell script path");
    }
  }

  LOG.info("Application master for app" + ", appId="
      + appAttemptID.getApplicationId().getId() + ", clusterTimestamp="
      + appAttemptID.getApplicationId().getClusterTimestamp()
      + ", attemptId=" + appAttemptID.getAttemptId());

  containerMemory = Integer.parseInt(cliParser.getOptionValue("container_memory", "10"));
  containerVirtualCores = Integer.parseInt(cliParser.getOptionValue("container_vcores", "1"));
  numTotalContainers = Integer.parseInt(cliParser.getOptionValue("num_containers", "1"));
  if (numTotalContainers == 0) {
    throw new IllegalArgumentException("Cannot run MyAppliCationMaster with no containers");
  }
  requestPriority = Integer.parseInt(cliParser.getOptionValue("priority", "0"));

  return true;
}
 

public void fromProto(AMLaunchedProto proto) {
  this.applicationAttemptId =
      ConverterUtils.toApplicationAttemptId(proto.getApplicationAttemptId());
  this.launchTime = proto.getLaunchTime();
  this.appSubmitTime = proto.getAppSubmitTime();
}