Java Code Examples for org.apache.hadoop.yarn.api.records.ApplicationId#newInstance()

@Test (timeout = 10000)
public void testAppsHelpCommand() throws Exception {
  ApplicationCLI cli = createAndGetAppCLI();
  ApplicationCLI spyCli = spy(cli);
  int result = spyCli.run(new String[] { "application", "-help" });
  Assert.assertTrue(result == 0);
  verify(spyCli).printUsage(any(String.class), any(Options.class));
  Assert.assertEquals(createApplicationCLIHelpMessage(),
      sysOutStream.toString());

  sysOutStream.reset();
  ApplicationId applicationId = ApplicationId.newInstance(1234, 5);
  result = cli.run(
      new String[] {"application", "-kill", applicationId.toString(), "args" });
  verify(spyCli).printUsage(any(String.class), any(Options.class));
  Assert.assertEquals(createApplicationCLIHelpMessage(),
      sysOutStream.toString());

  sysOutStream.reset();
  NodeId nodeId = NodeId.newInstance("host0", 0);
  result = cli.run(
      new String[] { "application", "-status", nodeId.toString(), "args" });
  verify(spyCli).printUsage(any(String.class), any(Options.class));
  Assert.assertEquals(createApplicationCLIHelpMessage(),
      sysOutStream.toString());
}
 

@Test(timeout = 5000)
public void testHeartbeatShouldDie() throws IOException, InterruptedException, TezException,
    ExecutionException {

  ListeningExecutorService executor = null;
  try {
    ExecutorService rawExecutor = Executors.newFixedThreadPool(1);
    executor = MoreExecutors.listeningDecorator(rawExecutor);
    ApplicationId appId = ApplicationId.newInstance(10000, 1);
    TaskExecutionTestHelpers.TezTaskUmbilicalForTest
        umbilical = new TaskExecutionTestHelpers.TezTaskUmbilicalForTest();
    TaskReporter taskReporter = createTaskReporter(appId, umbilical);

    TezTaskRunner2 taskRunner = createTaskRunner(appId, umbilical, taskReporter, executor,
        TestProcessor.CONF_EMPTY);
    // Setup the executor
    Future<TaskRunner2Result> taskRunnerFuture =
        taskExecutor.submit(new TaskRunnerCallable2ForTest(taskRunner));
    // Signal the processor to go through
    TestProcessor.awaitStart();
    umbilical.signalSendShouldDie();
    umbilical.awaitRegisteredEvent();
    // Not signaling an actual start to verify task interruption

    TaskRunner2Result result = taskRunnerFuture.get();
    verifyTaskRunnerResult(result, EndReason.CONTAINER_STOP_REQUESTED, null, true, null);


    TestProcessor.awaitCompletion();
    assertTrue(TestProcessor.wasInterrupted());
    assertNull(taskReporter.currentCallable);
    // TODO Is this statement correct ?
    // No completion events since shouldDie was requested by the AM, which should have killed the
    // task.
    umbilical.verifyNoCompletionEvents();
    assertTrue(TestProcessor.wasAborted());
  } finally {
    executor.shutdownNow();
  }
}
 

@Test
public void testInvalidApp() {
  ApplicationId appId = ApplicationId.newInstance(0, MAX_APPS + 1);
  WebResource r = resource();
  ClientResponse response =
      r.path("ws").path("v1").path("applicationhistory").path("apps")
        .path(appId.toString())
        .queryParam("user.name", USERS[round])
        .accept(MediaType.APPLICATION_JSON)
        .get(ClientResponse.class);
  assertEquals("404 not found expected", Status.NOT_FOUND,
      response.getClientResponseStatus());
}
 

@Test(timeout=5000)
public void testRecoverableSummary_VertexInCommitting() throws IOException {
  ApplicationId appId = ApplicationId.newInstance(System.currentTimeMillis(), 1);
  TezDAGID dagID = TezDAGID.getInstance(appId, 1);
  AppContext appContext = mock(AppContext.class);
  when(appContext.getCurrentRecoveryDir()).thenReturn(new Path(recoveryPath+"/1"));
  when(appContext.getClock()).thenReturn(new SystemClock());
  when(mockDAGImpl.getID()).thenReturn(dagID);

  RecoveryService rService = new RecoveryService(appContext);
  Configuration conf = new Configuration();
  conf.setBoolean(RecoveryService.TEZ_TEST_RECOVERY_DRAIN_EVENTS_WHEN_STOPPED, true);
  rService.init(conf);
  rService.start();

  DAGPlan dagPlan = TestDAGImpl.createTestDAGPlan();
  // write a DAGSubmittedEvent first to initialize summaryStream
  rService.handle(new DAGHistoryEvent(dagID,
      new DAGSubmittedEvent(dagID, 1L, dagPlan, ApplicationAttemptId.newInstance(appId, 1),
          null, "user", new Configuration(), null, null)));
  // It should be fine to skip other events, just for testing.
  rService.handle(new DAGHistoryEvent(dagID,
      new VertexCommitStartedEvent(TezVertexID.getInstance(dagID, 0), 0L)));
  rService.stop();

  DAGRecoveryData dagData = parser.parseRecoveryData();
  assertEquals(dagID, dagData.recoveredDagID);
  assertTrue(dagData.nonRecoverable);
  assertTrue(dagData.reason.contains("Vertex Commit was in progress"));
}
 

@Test
public void testGetContainerReport() throws Exception {
  ApplicationCLI cli = createAndGetAppCLI();
  ApplicationId applicationId = ApplicationId.newInstance(1234, 5);
  ApplicationAttemptId attemptId = ApplicationAttemptId.newInstance(
      applicationId, 1);
  ContainerId containerId = ContainerId.newContainerId(attemptId, 1);
  ContainerReport container = ContainerReport.newInstance(containerId, null,
      NodeId.newInstance("host", 1234), Priority.UNDEFINED, 1234, 5678,
      "diagnosticInfo", "logURL", 0, ContainerState.COMPLETE,
      "http://" + NodeId.newInstance("host", 2345).toString());
  when(client.getContainerReport(any(ContainerId.class))).thenReturn(
      container);
  int result = cli.run(new String[] { "container", "-status",
      containerId.toString() });
  assertEquals(0, result);
  verify(client).getContainerReport(containerId);
  ByteArrayOutputStream baos = new ByteArrayOutputStream();
  PrintWriter pw = new PrintWriter(baos);
  pw.println("Container Report : ");
  pw.println("\tContainer-Id : container_1234_0005_01_000001");
  pw.println("\tStart-Time : 1234");
  pw.println("\tFinish-Time : 5678");
  pw.println("\tState : COMPLETE");
  pw.println("\tLOG-URL : logURL");
  pw.println("\tHost : host:1234");
  pw.println("\tNodeHttpAddress : http://host:2345");
  pw.println("\tDiagnostics : diagnosticInfo");
  pw.close();
  String appReportStr = baos.toString("UTF-8");
  Assert.assertEquals(appReportStr, sysOutStream.toString());
  verify(sysOut, times(1)).println(isA(String.class));
}
 

@Test(timeout = 5000)
public void testIgnoreErrorsDuringFailure() throws IOException, InterruptedException, TezException,
    ExecutionException {

  ListeningExecutorService executor = null;
  try {
    ExecutorService rawExecutor = Executors.newFixedThreadPool(1);
    executor = MoreExecutors.listeningDecorator(rawExecutor);
    ApplicationId appId = ApplicationId.newInstance(10000, 1);
    TaskExecutionTestHelpers.TezTaskUmbilicalForTest
        umbilical = new TaskExecutionTestHelpers.TezTaskUmbilicalForTest();

    TaskReporter taskReporter = new TaskReporter(umbilical, 100, 1000, 100, new AtomicLong(0),
      createContainerId(appId).toString()) {
      @Override
      protected boolean isShuttingDown() {
        return true;
      }
    };

    TezTaskRunner2 taskRunner = createTaskRunner(appId, umbilical, taskReporter, executor,
        TestProcessor.CONF_THROW_IO_EXCEPTION);
    // Setup the executor

    taskExecutor.submit(new TaskRunnerCallable2ForTest(taskRunner));

    // Signal the processor to go through
    TestProcessor.awaitStart();
    TestProcessor.signal();

    umbilical.verifyNoCompletionEvents();
  } finally {
    executor.shutdownNow();
  }
}
 

@Test (timeout = 10000)
public void testContainersHelpCommand() throws Exception {
  ApplicationCLI cli = createAndGetAppCLI();
  ApplicationCLI spyCli = spy(cli);
  int result = spyCli.run(new String[] { "container", "-help" });
  Assert.assertTrue(result == 0);
  verify(spyCli).printUsage(any(String.class), any(Options.class));
  Assert.assertEquals(createContainerCLIHelpMessage(),
      sysOutStream.toString());

  sysOutStream.reset();
  ApplicationId applicationId = ApplicationId.newInstance(1234, 5);
  ApplicationAttemptId appAttemptId =
      ApplicationAttemptId.newInstance(applicationId, 6);
  result = cli.run(
      new String[] {"container", "-list", appAttemptId.toString(), "args" });
  verify(spyCli).printUsage(any(String.class), any(Options.class));
  Assert.assertEquals(createContainerCLIHelpMessage(),
      sysOutStream.toString());

  sysOutStream.reset();
  ContainerId containerId = ContainerId.newContainerId(appAttemptId, 7);
  result = cli.run(
      new String[] { "container", "-status", containerId.toString(), "args" });
  verify(spyCli).printUsage(any(String.class), any(Options.class));
  Assert.assertEquals(createContainerCLIHelpMessage(),
      sysOutStream.toString());
}
 

@Test(timeout = 10000)
public void testGetContainerReport() throws YarnException, IOException {
  Configuration conf = new Configuration();
  conf.setBoolean(YarnConfiguration.APPLICATION_HISTORY_ENABLED,
      true);
  final YarnClient client = new MockYarnClient();
  client.init(conf);
  client.start();

  List<ApplicationReport> expectedReports = ((MockYarnClient) client)
      .getReports();

  ApplicationId applicationId = ApplicationId.newInstance(1234, 5);
  ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(
      applicationId, 1);
  ContainerId containerId = ContainerId.newContainerId(appAttemptId, 1);
  ContainerReport report = client.getContainerReport(containerId);
  Assert.assertNotNull(report);
  Assert.assertEquals(report.getContainerId().toString(),
      (ContainerId.newContainerId(expectedReports.get(0)
          .getCurrentApplicationAttemptId(), 1)).toString());
  containerId = ContainerId.newContainerId(appAttemptId, 3);
  report = client.getContainerReport(containerId);
  Assert.assertNotNull(report);
  Assert.assertEquals(report.getContainerId().toString(),
      (ContainerId.newContainerId(expectedReports.get(0)
          .getCurrentApplicationAttemptId(), 3)).toString());
  client.stop();
}
 

@Test(timeout = 5000)
public void testEventSerializingHash() throws Exception {
  ApplicationId appId = ApplicationId.newInstance(1, 2);
  TezDAGID dagID = TezDAGID.getInstance(appId, 1);
  TezVertexID vertexID = TezVertexID.getInstance(dagID, 1);
  TezTaskID taskID1 = TezTaskID.getInstance(vertexID, 1);
  TezTaskID taskID2 = TezTaskID.getInstance(vertexID, 2);
  TezTaskAttemptID taID11 = TezTaskAttemptID.getInstance(taskID1, 0);
  TezTaskAttemptID taID12 = TezTaskAttemptID.getInstance(taskID1, 1);
  TezTaskAttemptID taID21 = TezTaskAttemptID.getInstance(taskID2, 1);
  
  TaskAttemptEvent taEventFail11 = new TaskAttemptEvent(taID11, TaskAttemptEventType.TA_FAILED);
  TaskAttemptEvent taEventKill11 = new TaskAttemptEvent(taID11, TaskAttemptEventType.TA_KILL_REQUEST);
  TaskAttemptEvent taEventKill12 = new TaskAttemptEvent(taID12, TaskAttemptEventType.TA_KILL_REQUEST);
  TaskAttemptEvent taEventKill21 = new TaskAttemptEvent(taID21, TaskAttemptEventType.TA_KILL_REQUEST);
  TaskEvent tEventKill1 = new TaskEvent(taskID1, TaskEventType.T_ATTEMPT_KILLED);
  TaskEvent tEventFail1 = new TaskEvent(taskID1, TaskEventType.T_ATTEMPT_FAILED);
  TaskEvent tEventFail2 = new TaskEvent(taskID2, TaskEventType.T_ATTEMPT_FAILED);
  
  // all of them should have the same value
  assertEquals(taEventFail11.getSerializingHash(), taEventKill11.getSerializingHash());
  assertEquals(taEventKill11.getSerializingHash(), taEventKill12.getSerializingHash());
  assertEquals(tEventFail1.getSerializingHash(), tEventKill1.getSerializingHash());
  assertEquals(taEventFail11.getSerializingHash(), tEventKill1.getSerializingHash());
  assertEquals(taEventKill21.getSerializingHash(), tEventFail2.getSerializingHash());
  // events from different tasks may not have the same value
  assertFalse(tEventFail1.getSerializingHash() == tEventFail2.getSerializingHash());
}
 

@Test
public void testAppDiognosticEventOnNewTask() throws Exception {
  ApplicationId appId = ApplicationId.newInstance(1, 2);
  ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(
      appId, 0);
  JobId jobId = MRBuilderUtils.newJobId(appId, 1);
  TaskId taskId = MRBuilderUtils.newTaskId(jobId, 1, TaskType.MAP);
  TaskAttemptId attemptId = MRBuilderUtils.newTaskAttemptId(taskId, 0);
  Path jobFile = mock(Path.class);

  MockEventHandler eventHandler = new MockEventHandler();
  TaskAttemptListener taListener = mock(TaskAttemptListener.class);
  when(taListener.getAddress()).thenReturn(
      new InetSocketAddress("localhost", 0));

  JobConf jobConf = new JobConf();
  jobConf.setClass("fs.file.impl", StubbedFS.class, FileSystem.class);
  jobConf.setBoolean("fs.file.impl.disable.cache", true);
  jobConf.set(JobConf.MAPRED_MAP_TASK_ENV, "");
  jobConf.set(MRJobConfig.APPLICATION_ATTEMPT_ID, "10");

  TaskSplitMetaInfo splits = mock(TaskSplitMetaInfo.class);
  when(splits.getLocations()).thenReturn(new String[] { "127.0.0.1" });

  AppContext appCtx = mock(AppContext.class);
  ClusterInfo clusterInfo = mock(ClusterInfo.class);
  Resource resource = mock(Resource.class);
  when(appCtx.getClusterInfo()).thenReturn(clusterInfo);
  when(resource.getMemory()).thenReturn(1024);

  TaskAttemptImpl taImpl = new MapTaskAttemptImpl(taskId, 1, eventHandler,
      jobFile, 1, splits, jobConf, taListener,
      new Token(), new Credentials(), new SystemClock(), appCtx);

  NodeId nid = NodeId.newInstance("127.0.0.1", 0);
  ContainerId contId = ContainerId.newContainerId(appAttemptId, 3);
  Container container = mock(Container.class);
  when(container.getId()).thenReturn(contId);
  when(container.getNodeId()).thenReturn(nid);
  when(container.getNodeHttpAddress()).thenReturn("localhost:0");
  taImpl.handle(new TaskAttemptDiagnosticsUpdateEvent(attemptId,
      "Task got killed"));
  assertFalse(
      "InternalError occurred trying to handle TA_DIAGNOSTICS_UPDATE on assigned task",
      eventHandler.internalError);
}
 

@Test(timeout = 5000)
// Ensure ContainerTerminating and ContainerTerminated is handled correctly by
// the TaskAttempt
public void testContainerTerminatedAfterSuccess() throws Exception {
  ApplicationId appId = ApplicationId.newInstance(1, 2);
  ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(
      appId, 0);
  TezDAGID dagID = TezDAGID.getInstance(appId, 1);
  TezVertexID vertexID = TezVertexID.getInstance(dagID, 1);
  TezTaskID taskID = TezTaskID.getInstance(vertexID, 1);
  TezTaskAttemptID taskAttemptID = TezTaskAttemptID.getInstance(taskID, 0);

  MockEventHandler eventHandler = spy(new MockEventHandler());
  TaskAttemptListener taListener = mock(TaskAttemptListener.class);
  when(taListener.getAddress()).thenReturn(
      new InetSocketAddress("localhost", 0));

  Configuration taskConf = new Configuration();
  taskConf.setClass("fs.file.impl", StubbedFS.class, FileSystem.class);
  taskConf.setBoolean("fs.file.impl.disable.cache", true);

  TaskLocationHint locationHint = new TaskLocationHint(
      new HashSet<String>(Arrays.asList(new String[] {"127.0.0.1"})), null);
  Resource resource = Resource.newInstance(1024, 1);

  NodeId nid = NodeId.newInstance("127.0.0.1", 0);
  ContainerId contId = ContainerId.newInstance(appAttemptId, 3);
  Container container = mock(Container.class);
  when(container.getId()).thenReturn(contId);
  when(container.getNodeId()).thenReturn(nid);
  when(container.getNodeHttpAddress()).thenReturn("localhost:0");

  AppContext appCtx = mock(AppContext.class);
  AMContainerMap containers = new AMContainerMap(
      mock(ContainerHeartbeatHandler.class), mock(TaskAttemptListener.class),
      new ContainerContextMatcher(), appCtx);
  containers.addContainerIfNew(container);

  doReturn(new ClusterInfo()).when(appCtx).getClusterInfo();
  doReturn(containers).when(appCtx).getAllContainers();

  TaskAttemptImpl taImpl = new MockTaskAttemptImpl(taskID, 1, eventHandler,
      taListener, taskConf, new SystemClock(),
      mock(TaskHeartbeatHandler.class), appCtx, locationHint, false,
      resource, createFakeContainerContext(), false);

  ArgumentCaptor<Event> arg = ArgumentCaptor.forClass(Event.class);

  taImpl.handle(new TaskAttemptEventSchedule(taskAttemptID, null));
  // At state STARTING.
  taImpl.handle(new TaskAttemptEventStartedRemotely(taskAttemptID, contId,
      null));
  assertEquals("Task attempt is not in the RUNNING state", taImpl.getState(),
      TaskAttemptState.RUNNING);

  int expectedEventsAtRunning = 3;
  verify(eventHandler, times(expectedEventsAtRunning)).handle(arg.capture());

  taImpl.handle(new TaskAttemptEvent(taskAttemptID, TaskAttemptEventType.TA_DONE));

  assertEquals("Task attempt is not in the  SUCCEEDED state", taImpl.getState(),
      TaskAttemptState.SUCCEEDED);

  assertEquals(0, taImpl.getDiagnostics().size());

  int expectedEvenstAfterTerminating = expectedEventsAtRunning + 3;
  arg = ArgumentCaptor.forClass(Event.class);
  verify(eventHandler, times(expectedEvenstAfterTerminating)).handle(arg.capture());

  verifyEventType(
      arg.getAllValues().subList(expectedEventsAtRunning,
          expectedEvenstAfterTerminating), TaskEventTAUpdate.class, 1);
  verifyEventType(
      arg.getAllValues().subList(expectedEventsAtRunning,
          expectedEvenstAfterTerminating), AMSchedulerEventTAEnded.class, 1);

  taImpl.handle(new TaskAttemptEventContainerTerminated(taskAttemptID,
      "Terminated"));
  int expectedEventAfterTerminated = expectedEvenstAfterTerminating + 0;
  arg = ArgumentCaptor.forClass(Event.class);
  verify(eventHandler, times(expectedEventAfterTerminated)).handle(arg.capture());

  // Verify that the diagnostic message included in the Terminated event is not
  // captured - TA already succeeded.
  assertEquals(0, taImpl.getDiagnostics().size());
}
 

@Test(timeout = 20000)
public void testNMProxyRetry() throws Exception {
  containerManager.start();
  containerManager.setBlockNewContainerRequests(false);
  StartContainersRequest allRequests =
      Records.newRecord(StartContainersRequest.class);
  ApplicationId appId = ApplicationId.newInstance(1, 1);
  ApplicationAttemptId attemptId = ApplicationAttemptId.newInstance(appId, 1);

  org.apache.hadoop.yarn.api.records.Token nmToken =
      context.getNMTokenSecretManager().createNMToken(attemptId,
        context.getNodeId(), user);
  final InetSocketAddress address =
      conf.getSocketAddr(YarnConfiguration.NM_BIND_HOST,
        YarnConfiguration.NM_ADDRESS, YarnConfiguration.DEFAULT_NM_ADDRESS,
        YarnConfiguration.DEFAULT_NM_PORT);
  Token<NMTokenIdentifier> token =
      ConverterUtils.convertFromYarn(nmToken,
        SecurityUtil.buildTokenService(address));
  UserGroupInformation ugi = UserGroupInformation.createRemoteUser(user);
  ugi.addToken(token);

  ContainerManagementProtocol proxy =
      NMProxy.createNMProxy(conf, ContainerManagementProtocol.class, ugi,
        YarnRPC.create(conf), address);

  retryCount = 0;
  shouldThrowNMNotYetReadyException = false;
  proxy.startContainers(allRequests);
  Assert.assertEquals(5, retryCount);

  retryCount = 0;
  shouldThrowNMNotYetReadyException = false;
  proxy.stopContainers(Records.newRecord(StopContainersRequest.class));
  Assert.assertEquals(5, retryCount);

  retryCount = 0;
  shouldThrowNMNotYetReadyException = false;
  proxy.getContainerStatuses(Records
    .newRecord(GetContainerStatusesRequest.class));
  Assert.assertEquals(5, retryCount);

  retryCount = 0;
  shouldThrowNMNotYetReadyException = true;
  proxy.startContainers(allRequests);
  Assert.assertEquals(5, retryCount);
}
 

@Test
public void testGetApplicationReport() throws Exception {
  for (int i = 0; i < 2; ++i) {
    ApplicationCLI cli = createAndGetAppCLI();
    ApplicationId applicationId = ApplicationId.newInstance(1234, 5);
    ApplicationResourceUsageReport usageReport = i == 0 ? null :
        ApplicationResourceUsageReport.newInstance(
            2, 0, null, null, null, 123456, 4567);
    ApplicationReport newApplicationReport = ApplicationReport.newInstance(
        applicationId, ApplicationAttemptId.newInstance(applicationId, 1),
        "user", "queue", "appname", "host", 124, null,
        YarnApplicationState.FINISHED, "diagnostics", "url", 0, 0,
        FinalApplicationStatus.SUCCEEDED, usageReport, "N/A", 0.53789f, "YARN",
        null);
    when(client.getApplicationReport(any(ApplicationId.class))).thenReturn(
        newApplicationReport);
    int result = cli.run(new String[] { "application", "-status", applicationId.toString() });
    assertEquals(0, result);
    verify(client, times(1 + i)).getApplicationReport(applicationId);
    ByteArrayOutputStream baos = new ByteArrayOutputStream();
    PrintWriter pw = new PrintWriter(baos);
    pw.println("Application Report : ");
    pw.println("\tApplication-Id : application_1234_0005");
    pw.println("\tApplication-Name : appname");
    pw.println("\tApplication-Type : YARN");
    pw.println("\tUser : user");
    pw.println("\tQueue : queue");
    pw.println("\tStart-Time : 0");
    pw.println("\tFinish-Time : 0");
    pw.println("\tProgress : 53.79%");
    pw.println("\tState : FINISHED");
    pw.println("\tFinal-State : SUCCEEDED");
    pw.println("\tTracking-URL : N/A");
    pw.println("\tRPC Port : 124");
    pw.println("\tAM Host : host");
    pw.println("\tAggregate Resource Allocation : " +
        (i == 0 ? "N/A" : "123456 MB-seconds, 4567 vcore-seconds"));
    pw.println("\tDiagnostics : diagnostics");
    pw.close();
    String appReportStr = baos.toString("UTF-8");
    Assert.assertEquals(appReportStr, sysOutStream.toString());
    sysOutStream.reset();
    verify(sysOut, times(1 + i)).println(isA(String.class));
  }
}
 

public static ApplicationId newApplicationId(RecordFactory recordFactory,
    long clusterTimeStamp, int id) {
  return ApplicationId.newInstance(clusterTimeStamp, id);
}
 

@Test(timeout = 5000)
public void testAMLoggingOptsPerLogger() throws IOException, YarnException {

  TezConfiguration tezConf = new TezConfiguration();
  tezConf.set(TezConfiguration.TEZ_AM_LOG_LEVEL,
      "WARN;org.apache.hadoop.ipc=DEBUG;org.apache.hadoop.security=DEBUG");
  tezConf.set(TezConfiguration.TEZ_AM_STAGING_DIR, STAGING_DIR.getAbsolutePath());

  ApplicationId appId = ApplicationId.newInstance(1000, 1);
  Credentials credentials = new Credentials();
  JobTokenSecretManager jobTokenSecretManager = new JobTokenSecretManager();
  TezClientUtils.createSessionToken(appId.toString(), jobTokenSecretManager, credentials);
  DAG dag = DAG.create("testdag");
  dag.addVertex(Vertex.create("testVertex", ProcessorDescriptor.create("processorClassname"), 1)
      .setTaskLaunchCmdOpts("initialLaunchOpts"));
  AMConfiguration amConf =
      new AMConfiguration(tezConf, new HashMap<String, LocalResource>(), credentials);
  ApplicationSubmissionContext appSubmissionContext =
      TezClientUtils.createApplicationSubmissionContext(appId, dag, "amName", amConf,
          new HashMap<String, LocalResource>(), credentials, false, new TezApiVersionInfo(),
          null, null);

  List<String> expectedCommands = new LinkedList<String>();
  expectedCommands.add("-Dlog4j.configuratorClass=org.apache.tez.common.TezLog4jConfigurator");
  expectedCommands.add(
      "-Dlog4j.configuration=" + TezConstants.TEZ_CONTAINER_LOG4J_PROPERTIES_FILE);
  expectedCommands.add("-D" + YarnConfiguration.YARN_APP_CONTAINER_LOG_DIR + "=" +
      ApplicationConstants.LOG_DIR_EXPANSION_VAR);
  expectedCommands.add("-D" + TezConstants.TEZ_ROOT_LOGGER_NAME + "=" + "WARN" + "," +
      TezConstants.TEZ_CONTAINER_LOGGER_NAME);

  List<String> commands = appSubmissionContext.getAMContainerSpec().getCommands();
  assertEquals(1, commands.size());
  for (String expectedCmd : expectedCommands) {
    assertTrue(commands.get(0).contains(expectedCmd));
  }

  Map<String, String> environment = appSubmissionContext.getAMContainerSpec().getEnvironment();
  String logEnv = environment.get(TezConstants.TEZ_CONTAINER_LOG_PARAMS);
  assertEquals("org.apache.hadoop.ipc=DEBUG;org.apache.hadoop.security=DEBUG", logEnv);
}
 

@Test(timeout = 5000)
public void testLastDataEventRecording() throws Exception {
  ApplicationId appId = ApplicationId.newInstance(1, 2);
  ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(
      appId, 0);
  TezDAGID dagID = TezDAGID.getInstance(appId, 1);
  TezVertexID vertexID = TezVertexID.getInstance(dagID, 1);
  TezTaskID taskID = TezTaskID.getInstance(vertexID, 1);

  MockEventHandler eventHandler = spy(new MockEventHandler());
  TaskCommunicatorManagerInterface taListener = createMockTaskAttemptListener();

  Configuration taskConf = new Configuration();
  taskConf.setClass("fs.file.impl", StubbedFS.class, FileSystem.class);
  taskConf.setBoolean("fs.file.impl.disable.cache", true);
  taskConf.setBoolean(TezConfiguration.TEZ_AM_SPECULATION_ENABLED, true);

  locationHint = TaskLocationHint.createTaskLocationHint(
      new HashSet<String>(Arrays.asList(new String[]{"127.0.0.1"})), null);
  Resource resource = Resource.newInstance(1024, 1);

  NodeId nid = NodeId.newInstance("127.0.0.1", 0);
  @SuppressWarnings("deprecation")
  ContainerId contId = ContainerId.newInstance(appAttemptId, 3);
  Container container = mock(Container.class);
  when(container.getId()).thenReturn(contId);
  when(container.getNodeId()).thenReturn(nid);
  when(container.getNodeHttpAddress()).thenReturn("localhost:0");

  AMContainerMap containers = new AMContainerMap(
      mock(ContainerHeartbeatHandler.class), mock(TaskCommunicatorManagerInterface.class),
      new ContainerContextMatcher(), appCtx);
  containers.addContainerIfNew(container, 0, 0, 0);

  doReturn(new ClusterInfo()).when(appCtx).getClusterInfo();
  doReturn(containers).when(appCtx).getAllContainers();

  TaskHeartbeatHandler mockHeartbeatHandler = mock(TaskHeartbeatHandler.class);
  TaskAttemptImpl taImpl = new MockTaskAttemptImpl(taskID, 1, eventHandler,
      taListener, taskConf, new SystemClock(),
      mockHeartbeatHandler, appCtx, false,
      resource, createFakeContainerContext(), false);
  TezTaskAttemptID taskAttemptID = taImpl.getID();

  taImpl.handle(new TaskAttemptEventSchedule(taskAttemptID, 0, 0));
  taImpl.handle(new TaskAttemptEventSubmitted(taskAttemptID, contId));
  taImpl.handle(new TaskAttemptEventStartedRemotely(taskAttemptID));
  assertEquals("Task attempt is not in the RUNNING state", taImpl.getState(),
      TaskAttemptState.RUNNING);
  
  long ts1 = 1024;
  long ts2 = 2048;
  TezTaskAttemptID mockId1 = mock(TezTaskAttemptID.class);
  TezTaskAttemptID mockId2 = mock(TezTaskAttemptID.class);
  TezEvent mockTezEvent1 = mock(TezEvent.class, RETURNS_DEEP_STUBS);
  when(mockTezEvent1.getEventReceivedTime()).thenReturn(ts1);
  when(mockTezEvent1.getSourceInfo().getTaskAttemptID()).thenReturn(mockId1);
  TezEvent mockTezEvent2 = mock(TezEvent.class, RETURNS_DEEP_STUBS);
  when(mockTezEvent2.getEventReceivedTime()).thenReturn(ts2);
  when(mockTezEvent2.getSourceInfo().getTaskAttemptID()).thenReturn(mockId2);
  TaskAttemptEventStatusUpdate statusEvent =
      new TaskAttemptEventStatusUpdate(taskAttemptID, new TaskStatusUpdateEvent(null, 0.1f, null, false));

  assertEquals(0, taImpl.lastDataEvents.size());
  taImpl.setLastEventSent(mockTezEvent1);
  assertEquals(1, taImpl.lastDataEvents.size());
  assertEquals(ts1, taImpl.lastDataEvents.get(0).getTimestamp());
  assertEquals(mockId1, taImpl.lastDataEvents.get(0).getTaskAttemptId());
  taImpl.handle(statusEvent);
  taImpl.setLastEventSent(mockTezEvent2);
  assertEquals(1, taImpl.lastDataEvents.size());
  assertEquals(ts2, taImpl.lastDataEvents.get(0).getTimestamp());
  assertEquals(mockId2, taImpl.lastDataEvents.get(0).getTaskAttemptId()); // over-write earlier value
  statusEvent.setReadErrorReported(true);
  taImpl.handle(statusEvent);
  taImpl.setLastEventSent(mockTezEvent1);
  assertEquals(2, taImpl.lastDataEvents.size());
  assertEquals(ts1, taImpl.lastDataEvents.get(1).getTimestamp());
  assertEquals(mockId1, taImpl.lastDataEvents.get(1).getTaskAttemptId()); // add new event
  taImpl.setLastEventSent(mockTezEvent2);
  assertEquals(2, taImpl.lastDataEvents.size());
  assertEquals(ts2, taImpl.lastDataEvents.get(1).getTimestamp());
  assertEquals(mockId2, taImpl.lastDataEvents.get(1).getTaskAttemptId()); // over-write earlier value
}
 

@Test
public void testContainerCleanedWhileCommitting() throws Exception {
  ApplicationId appId = ApplicationId.newInstance(1, 2);
  ApplicationAttemptId appAttemptId =
    ApplicationAttemptId.newInstance(appId, 0);
  JobId jobId = MRBuilderUtils.newJobId(appId, 1);
  TaskId taskId = MRBuilderUtils.newTaskId(jobId, 1, TaskType.MAP);
  TaskAttemptId attemptId = MRBuilderUtils.newTaskAttemptId(taskId, 0);
  Path jobFile = mock(Path.class);

  MockEventHandler eventHandler = new MockEventHandler();
  TaskAttemptListener taListener = mock(TaskAttemptListener.class);
  when(taListener.getAddress()).thenReturn(new InetSocketAddress("localhost", 0));

  JobConf jobConf = new JobConf();
  jobConf.setClass("fs.file.impl", StubbedFS.class, FileSystem.class);
  jobConf.setBoolean("fs.file.impl.disable.cache", true);
  jobConf.set(JobConf.MAPRED_MAP_TASK_ENV, "");
  jobConf.set(MRJobConfig.APPLICATION_ATTEMPT_ID, "10");

  TaskSplitMetaInfo splits = mock(TaskSplitMetaInfo.class);
  when(splits.getLocations()).thenReturn(new String[] {});

  AppContext appCtx = mock(AppContext.class);
  ClusterInfo clusterInfo = mock(ClusterInfo.class);
  Resource resource = mock(Resource.class);
  when(appCtx.getClusterInfo()).thenReturn(clusterInfo);
  when(resource.getMemory()).thenReturn(1024);

  TaskAttemptImpl taImpl =
    new MapTaskAttemptImpl(taskId, 1, eventHandler, jobFile, 1,
        splits, jobConf, taListener,
        new Token(), new Credentials(),
        new SystemClock(), appCtx);

  NodeId nid = NodeId.newInstance("127.0.0.1", 0);
  ContainerId contId = ContainerId.newContainerId(appAttemptId, 3);
  Container container = mock(Container.class);
  when(container.getId()).thenReturn(contId);
  when(container.getNodeId()).thenReturn(nid);
  when(container.getNodeHttpAddress()).thenReturn("localhost:0");

  taImpl.handle(new TaskAttemptEvent(attemptId,
      TaskAttemptEventType.TA_SCHEDULE));
  taImpl.handle(new TaskAttemptContainerAssignedEvent(attemptId,
      container, mock(Map.class)));
  taImpl.handle(new TaskAttemptContainerLaunchedEvent(attemptId, 0));
  taImpl.handle(new TaskAttemptEvent(attemptId,
      TaskAttemptEventType.TA_COMMIT_PENDING));

  assertEquals("Task attempt is not in commit pending state", taImpl.getState(),
      TaskAttemptState.COMMIT_PENDING);
  taImpl.handle(new TaskAttemptEvent(attemptId,
      TaskAttemptEventType.TA_CONTAINER_CLEANED));
  assertFalse("InternalError occurred trying to handle TA_CONTAINER_CLEANED",
      eventHandler.internalError);
  assertEquals("Task attempt is assigned locally", Locality.OFF_SWITCH,
      taImpl.getLocality());
}
 

@Test(timeout = 1800000)
public void testTwoRoundsRecoverying() throws Exception {
  ApplicationId appId = ApplicationId.newInstance(System.currentTimeMillis(),
          1);
  TezDAGID dagId = TezDAGID.getInstance(appId, 1);
  TezVertexID vertexId0 = TezVertexID.getInstance(dagId, 0);
  TezVertexID vertexId1 = TezVertexID.getInstance(dagId, 1);
  TezVertexID vertexId2 = TezVertexID.getInstance(dagId, 2);
  ContainerId containerId = ContainerId.newInstance(
          ApplicationAttemptId.newInstance(appId, 1), 1);
  NodeId nodeId = NodeId.newInstance("localhost", 10);
  List<TezEvent> initGeneratedEvents = Lists.newArrayList(
          new TezEvent(InputDataInformationEvent.createWithObjectPayload(0, new Object()), null));


  List<SimpleShutdownCondition> shutdownConditions = Lists.newArrayList(

          new SimpleShutdownCondition(TIMING.POST, new DAGInitializedEvent(
                  dagId, 0L, "username", "dagName", null)),
          new SimpleShutdownCondition(TIMING.POST, new DAGStartedEvent(dagId,
                  0L, "username", "dagName")),
          new SimpleShutdownCondition(TIMING.POST,
                  new VertexInitializedEvent(vertexId0, "Tokenizer", 0L, 0L, 0,
                          "", null, initGeneratedEvents, null)),
          new SimpleShutdownCondition(TIMING.POST, new VertexStartedEvent(
                  vertexId0, 0L, 0L)),
          new SimpleShutdownCondition(TIMING.POST,
                  new VertexConfigurationDoneEvent(vertexId0, 0L, 2, null, null,
                          null, true)),
          new SimpleShutdownCondition(TIMING.POST, new TaskStartedEvent(
                  TezTaskID.getInstance(vertexId0, 0), "vertexName", 0L, 0L)),
          new SimpleShutdownCondition(TIMING.POST,
                  new TaskAttemptStartedEvent(TezTaskAttemptID.getInstance(
                          TezTaskID.getInstance(vertexId0, 0), 0), "vertexName", 0L,
                          containerId, nodeId, "", "", "")),
          new SimpleShutdownCondition(TIMING.POST, new TaskFinishedEvent(
                  TezTaskID.getInstance(vertexId0, 0), "vertexName", 0L, 0L,
                  null, TaskState.SUCCEEDED, "", new TezCounters(), 0)),
          new SimpleShutdownCondition(TIMING.POST, new VertexFinishedEvent(
                  vertexId0, "vertexName", 1, 0L, 0L, 0L, 0L, 0L,
                  VertexState.SUCCEEDED, "", new TezCounters(),
                  new VertexStats(), new HashMap<String, Integer>(), null)),
          new SimpleShutdownCondition(TIMING.POST, new VertexFinishedEvent(
                  vertexId1, "vertexName", 1, 0L, 0L, 0L, 0L, 0L,
                  VertexState.SUCCEEDED, "", new TezCounters(),
                  new VertexStats(), new HashMap<String, Integer>(), null)),
          new SimpleShutdownCondition(TIMING.POST, new VertexFinishedEvent(
                  vertexId2, "vertexName", 1, 0L, 0L, 0L, 0L, 0L,
                  VertexState.SUCCEEDED, "", new TezCounters(),
                  new VertexStats(), new HashMap<String, Integer>(), null)),
          new SimpleShutdownCondition(TIMING.POST, new DAGFinishedEvent(
                  dagId, 0L, 0L, DAGState.SUCCEEDED, "", new TezCounters(),
                  "username", "dagName", new HashMap<String, Integer>(),
                  ApplicationAttemptId.newInstance(appId, 1), null))

  );

  Random rand = new Random();
  for (int i = 0; i < shutdownConditions.size() - 1; i++) {
    // randomly choose half of the test scenario to avoid
    // timeout.
    if (rand.nextDouble()<0.5) {
      int nextSimpleConditionIndex = i + 1 + rand.nextInt(shutdownConditions.size() - i - 1);
      if (nextSimpleConditionIndex == shutdownConditions.size() - 1) {
        testOrderedWordCountMultipleRoundRecoverying(
                new RecoveryServiceWithEventHandlingHook.MultipleRoundShutdownCondition(
                        Lists.newArrayList(shutdownConditions.get(i), shutdownConditions.get(nextSimpleConditionIndex)))
                , true,
                shutdownConditions.get(i).getHistoryEvent().getEventType() == HistoryEventType.VERTEX_STARTED);
      }
    }
  }
}
 

private static ApplicationId toApplicationId(RecordFactory recordFactory,
    Iterator<String> it) {
  ApplicationId appId = ApplicationId.newInstance(Long.parseLong(it.next()),
      Integer.parseInt(it.next()));
  return appId;
}
 

private ContainerId mockContainerId(int cId) {
  ApplicationId appId = ApplicationId.newInstance(1000, 1);
  ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(appId, 1);
  ContainerId containerId = ContainerId.newInstance(appAttemptId, cId);
  return containerId;
}