org.apache.hadoop.yarn.api.records.ApplicationAttemptId Java Examples

@Override
public synchronized Allocation allocate(
    ApplicationAttemptId applicationAttemptId, List<ResourceRequest> ask,
    List<ContainerId> release, 
    List<String> blacklistAdditions, List<String> blacklistRemovals) {
  List<ResourceRequest> askCopy = new ArrayList<ResourceRequest>();
  for (ResourceRequest req : ask) {
    ResourceRequest reqCopy = ResourceRequest.newInstance(req
        .getPriority(), req.getResourceName(), req.getCapability(), req
        .getNumContainers(), req.getRelaxLocality());
    askCopy.add(reqCopy);
  }
  SecurityUtil.setTokenServiceUseIp(false);
  lastAsk = ask;
  lastRelease = release;
  lastBlacklistAdditions = blacklistAdditions;
  lastBlacklistRemovals = blacklistRemovals;
  return super.allocate(
      applicationAttemptId, askCopy, release, 
      blacklistAdditions, blacklistRemovals);
}
 

@SuppressWarnings("rawtypes")
@Test (timeout = 10000)
public void testCallFailureWithNullLocalizedResources() {
  Container container = mock(Container.class);
  when(container.getContainerId()).thenReturn(ContainerId.newContainerId(
      ApplicationAttemptId.newInstance(ApplicationId.newInstance(
          System.currentTimeMillis(), 1), 1), 1));
  ContainerLaunchContext clc = mock(ContainerLaunchContext.class);
  when(clc.getCommands()).thenReturn(Collections.<String>emptyList());
  when(container.getLaunchContext()).thenReturn(clc);
  when(container.getLocalizedResources()).thenReturn(null);
  Dispatcher dispatcher = mock(Dispatcher.class);
  EventHandler eventHandler = new EventHandler() {
    public void handle(Event event) {
      Assert.assertTrue(event instanceof ContainerExitEvent);
      ContainerExitEvent exitEvent = (ContainerExitEvent) event;
      Assert.assertEquals(ContainerEventType.CONTAINER_EXITED_WITH_FAILURE,
          exitEvent.getType());
    }
  };
  when(dispatcher.getEventHandler()).thenReturn(eventHandler);
  ContainerLaunch launch = new ContainerLaunch(context, new Configuration(),
      dispatcher, exec, null, container, dirsHandler, containerManager);
  launch.call();
}
 

/**
 * {@link RMAppAttemptState#SCHEDULED}
 */
@SuppressWarnings("unchecked")
private void testAppAttemptScheduledState() {
  RMAppAttemptState expectedState;
  int expectedAllocateCount;
  if(unmanagedAM) {
    expectedState = RMAppAttemptState.LAUNCHED;
    expectedAllocateCount = 0;
  } else {
    expectedState = RMAppAttemptState.SCHEDULED;
    expectedAllocateCount = 1;
  }

  assertEquals(expectedState, 
      applicationAttempt.getAppAttemptState());
  verify(scheduler, times(expectedAllocateCount)).
  allocate(any(ApplicationAttemptId.class), 
      any(List.class), any(List.class), any(List.class), any(List.class));

  assertEquals(0,applicationAttempt.getJustFinishedContainers().size());
  assertNull(applicationAttempt.getMasterContainer());
  assertEquals(0.0, (double)applicationAttempt.getProgress(), 0.0001);
  assertEquals(0, application.getRanNodes().size());
  assertNull(applicationAttempt.getFinalApplicationStatus());
}
 

@Test
public void testInvalidContainer() throws Exception {
  ApplicationId appId = ApplicationId.newInstance(0, 1);
  ApplicationAttemptId appAttemptId =
      ApplicationAttemptId.newInstance(appId, 1);
  ContainerId containerId = ContainerId.newContainerId(appAttemptId,
      MAX_APPS + 1);
  WebResource r = resource();
  ClientResponse response =
      r.path("ws").path("v1").path("applicationhistory").path("apps")
        .path(appId.toString()).path("appattempts")
        .path(appAttemptId.toString()).path("containers")
        .path(containerId.toString())
        .queryParam("user.name", USERS[round])
        .accept(MediaType.APPLICATION_JSON)
        .get(ClientResponse.class);
  if (round == 1) {
    assertEquals(
        Status.FORBIDDEN, response.getClientResponseStatus());
    return;
  }
  assertEquals("404 not found expected", Status.NOT_FOUND,
          response.getClientResponseStatus());
}
 

@Override
public synchronized Allocation allocate(
    ApplicationAttemptId applicationAttemptId, List<ResourceRequest> ask,
    List<ContainerId> release, List<String> blacklistAdditions,
    List<String> blacklistRemovals) {
  List<ResourceRequest> askCopy = new ArrayList<ResourceRequest>();
  for (ResourceRequest req : ask) {
    ResourceRequest reqCopy =
        ResourceRequest.newInstance(req.getPriority(),
            req.getResourceName(), req.getCapability(),
            req.getNumContainers(), req.getRelaxLocality());
    askCopy.add(reqCopy);
  }
  lastAsk = ask;
  lastRelease = release;
  lastBlacklistAdditions = blacklistAdditions;
  lastBlacklistRemovals = blacklistRemovals;
  return super.allocate(applicationAttemptId, askCopy, release,
      blacklistAdditions, blacklistRemovals);
}
 

public static void waitForNumContainersToRecover(int num, MockRM rm,
    ApplicationAttemptId attemptId) throws Exception {
  AbstractYarnScheduler scheduler =
      (AbstractYarnScheduler) rm.getResourceScheduler();
  SchedulerApplicationAttempt attempt =
      scheduler.getApplicationAttempt(attemptId);
  while (attempt == null) {
    System.out.println("Wait for scheduler attempt " + attemptId
        + " to be created");
    Thread.sleep(200);
    attempt = scheduler.getApplicationAttempt(attemptId);
  }
  while (attempt.getLiveContainers().size() < num) {
    System.out.println("Wait for " + num
        + " containers to recover. currently: "
        + attempt.getLiveContainers().size());
    Thread.sleep(200);
  }
}
 

@Test(timeout = 10000)
public void testGetApplicationAttempt() throws YarnException, IOException {
  Configuration conf = new Configuration();
  final AHSClient client = new MockAHSClient();
  client.init(conf);
  client.start();

  List<ApplicationReport> expectedReports =
      ((MockAHSClient) client).getReports();

  ApplicationId applicationId = ApplicationId.newInstance(1234, 5);
  ApplicationAttemptId appAttemptId =
      ApplicationAttemptId.newInstance(applicationId, 1);
  ApplicationAttemptReport report =
      client.getApplicationAttemptReport(appAttemptId);
  Assert.assertNotNull(report);
  Assert.assertEquals(report.getApplicationAttemptId().toString(),
    expectedReports.get(0).getCurrentApplicationAttemptId().toString());
  client.stop();
}
 

@Test
/**
 * Ensure that when negative paramaters are given (signaling delay scheduling
 * no tin use), the least restrictive locality level is returned.
 */
public void testLocalityLevelWithoutDelays() {
  FSLeafQueue queue = Mockito.mock(FSLeafQueue.class);
  Priority prio = Mockito.mock(Priority.class);
  Mockito.when(prio.getPriority()).thenReturn(1);

  RMContext rmContext = resourceManager.getRMContext();
  ApplicationAttemptId applicationAttemptId = createAppAttemptId(1, 1);
  FSAppAttempt schedulerApp =
      new FSAppAttempt(scheduler, applicationAttemptId, "user1", queue ,
          null, rmContext);
  assertEquals(NodeType.OFF_SWITCH, schedulerApp.getAllowedLocalityLevel(
      prio, 10, -1.0, -1.0));
}
 

@Test
public void testMassiveWriteContainerHistoryData() throws IOException {
  LOG.info("Starting testMassiveWriteContainerHistoryData");
  long mb = 1024 * 1024;
  long usedDiskBefore = fs.getContentSummary(fsWorkingPath).getLength() / mb;
  ApplicationId appId = ApplicationId.newInstance(0, 1);
  writeApplicationStartData(appId);
  ApplicationAttemptId appAttemptId =
      ApplicationAttemptId.newInstance(appId, 1);
  for (int i = 1; i <= 100000; ++i) {
    ContainerId containerId = ContainerId.newContainerId(appAttemptId, i);
    writeContainerStartData(containerId);
    writeContainerFinishData(containerId);
  }
  writeApplicationFinishData(appId);
  long usedDiskAfter = fs.getContentSummary(fsWorkingPath).getLength() / mb;
  Assert.assertTrue((usedDiskAfter - usedDiskBefore) < 20);
}
 

@Override
public RMAppAttempt getRMAppAttempt(ApplicationAttemptId appAttemptId) {
  this.readLock.lock();

  try {
    return this.attempts.get(appAttemptId);
  } finally {
    this.readLock.unlock();
  }
}
 

@Private
public ApplicationAttemptId getApplicationAttemptId() {
  if (!proto.hasAppAttemptId()) {
    return null;
  }
  return new ApplicationAttemptIdPBImpl(proto.getAppAttemptId());
}
 

private RMAppImpl createMockRMApp(ApplicationAttemptId attemptId,
    RMContext context) {
  RMAppImpl app = mock(RMAppImpl.class);
  when(app.getApplicationId()).thenReturn(attemptId.getApplicationId());
  RMAppAttemptImpl attempt = mock(RMAppAttemptImpl.class);
  when(attempt.getAppAttemptId()).thenReturn(attemptId);
  RMAppAttemptMetrics attemptMetric = mock(RMAppAttemptMetrics.class);
  when(attempt.getRMAppAttemptMetrics()).thenReturn(attemptMetric);
  when(app.getCurrentAppAttempt()).thenReturn(attempt);
  context.getRMApps().putIfAbsent(attemptId.getApplicationId(), app);
  return app;
}
 

public void registerApplicationAttempt(ApplicationAttemptId appAttemptId) {
  try {
    this.writeLock.lock();
    this.appAttemptToNodeKeyMap.put(appAttemptId, new HashSet<NodeId>());
  } finally {
    this.writeLock.unlock();
  }
}
 

public ApplicationAttemptId validateRunningAttemptId(ApplicationId appId) throws Exception {
  ApplicationAttemptId attemptId = this.client.getApplicationReport(appId).getCurrentApplicationAttemptId();
  ApplicationAttemptReport attemptReport = this.client.getApplicationAttemptReport(attemptId);
  if (attemptReport.getYarnApplicationAttemptState() == YarnApplicationAttemptState.RUNNING) {
    log.info("Job is running. AttempId " + attemptId.toString());
    return attemptId;
  } else {
    throw new SamzaException("Job not running " + this.jobName);
  }
}
 

/**
 * Two apps on one queue, one app on another
 */
@Test
public void testBasicDRFWithQueues() throws Exception {
  scheduler.init(conf);
  scheduler.start();
  scheduler.reinitialize(conf, resourceManager.getRMContext());

  RMNode node = MockNodes.newNodeInfo(1, BuilderUtils.newResource(8192, 7),
      1, "127.0.0.1");
  NodeAddedSchedulerEvent nodeEvent = new NodeAddedSchedulerEvent(node);
  scheduler.handle(nodeEvent);

  ApplicationAttemptId appAttId1 = createSchedulingRequest(3072, 1, "queue1",
      "user1", 2);
  FSAppAttempt app1 = scheduler.getSchedulerApp(appAttId1);
  ApplicationAttemptId appAttId2 = createSchedulingRequest(2048, 2, "queue1",
      "user1", 2);
  FSAppAttempt app2 = scheduler.getSchedulerApp(appAttId2);
  ApplicationAttemptId appAttId3 = createSchedulingRequest(1024, 2, "queue2",
      "user1", 2);
  FSAppAttempt app3 = scheduler.getSchedulerApp(appAttId3);
  
  DominantResourceFairnessPolicy drfPolicy = new DominantResourceFairnessPolicy();
  drfPolicy.initialize(scheduler.getClusterResource());
  scheduler.getQueueManager().getQueue("root").setPolicy(drfPolicy);
  scheduler.getQueueManager().getQueue("queue1").setPolicy(drfPolicy);
  scheduler.update();

  NodeUpdateSchedulerEvent updateEvent = new NodeUpdateSchedulerEvent(node);
  scheduler.handle(updateEvent);
  Assert.assertEquals(1, app1.getLiveContainers().size());
  scheduler.handle(updateEvent);
  Assert.assertEquals(1, app3.getLiveContainers().size());
  scheduler.handle(updateEvent);
  Assert.assertEquals(2, app3.getLiveContainers().size());
  scheduler.handle(updateEvent);
  Assert.assertEquals(1, app2.getLiveContainers().size());
}
 

@Override
public synchronized void storeNMTokenApplicationMasterKey(
    ApplicationAttemptId attempt, MasterKey key) throws IOException {
  MasterKeyPBImpl keypb = (MasterKeyPBImpl) key;
  nmTokenState.applicationMasterKeys.put(attempt,
      new MasterKeyPBImpl(keypb.getProto()));
}
 

private ContainerId createContainerId(int id) {
  ApplicationId appId = ApplicationId.newInstance(0, 0);
  ApplicationAttemptId appAttemptId =
      ApplicationAttemptId.newInstance(appId, 1);
  ContainerId containerId = ContainerId.newContainerId(appAttemptId, id);
  return containerId;
}
 

public static TaskAttemptReport newTaskAttemptReport(TaskAttemptId id) {
  ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(
      id.getTaskId().getJobId().getAppId(), 0);
  ContainerId containerId = ContainerId.newContainerId(appAttemptId, 0);
  TaskAttemptReport report = Records.newRecord(TaskAttemptReport.class);
  report.setTaskAttemptId(id);
  report
      .setStartTime(System.currentTimeMillis() - (int) (Math.random() * DT));
  report.setFinishTime(System.currentTimeMillis()
      + (int) (Math.random() * DT) + 1);

  if (id.getTaskId().getTaskType() == TaskType.REDUCE) {
    report.setShuffleFinishTime(
        (report.getFinishTime() + report.getStartTime()) / 2);
    report.setSortFinishTime(
        (report.getFinishTime() + report.getShuffleFinishTime()) / 2);
  }

  report.setPhase(PHASES.next());
  report.setTaskAttemptState(TASK_ATTEMPT_STATES.next());
  report.setProgress((float) Math.random());
  report.setCounters(TypeConverter.toYarn(newCounters()));
  report.setContainerId(containerId);
  report.setDiagnosticInfo(DIAGS.next());
  report.setStateString("Moving average " + Math.random());
  return report;
}
 

@Test(timeout=5000)
public void testRecoverableSummary_VertexInCommitting() throws IOException {
  ApplicationId appId = ApplicationId.newInstance(System.currentTimeMillis(), 1);
  TezDAGID dagID = TezDAGID.getInstance(appId, 1);
  AppContext appContext = mock(AppContext.class);
  when(appContext.getCurrentRecoveryDir()).thenReturn(new Path(recoveryPath+"/1"));
  when(appContext.getClock()).thenReturn(new SystemClock());
  when(mockDAGImpl.getID()).thenReturn(dagID);

  RecoveryService rService = new RecoveryService(appContext);
  Configuration conf = new Configuration();
  conf.setBoolean(RecoveryService.TEZ_TEST_RECOVERY_DRAIN_EVENTS_WHEN_STOPPED, true);
  rService.init(conf);
  rService.start();

  DAGPlan dagPlan = TestDAGImpl.createTestDAGPlan();
  // write a DAGSubmittedEvent first to initialize summaryStream
  rService.handle(new DAGHistoryEvent(dagID,
      new DAGSubmittedEvent(dagID, 1L, dagPlan, ApplicationAttemptId.newInstance(appId, 1),
          null, "user", new Configuration(), null, null)));
  // It should be fine to skip other events, just for testing.
  rService.handle(new DAGHistoryEvent(dagID,
      new VertexCommitStartedEvent(TezVertexID.getInstance(dagID, 0), 0L)));
  rService.stop();

  DAGRecoveryData dagData = parser.parseRecoveryData();
  assertEquals(dagID, dagData.recoveredDagID);
  assertTrue(dagData.nonRecoverable);
  assertTrue(dagData.reason.contains("Vertex Commit was in progress"));
}
 

private void createNewAttempt() {
  ApplicationAttemptId appAttemptId =
      ApplicationAttemptId.newInstance(applicationId, attempts.size() + 1);
  RMAppAttempt attempt =
      new RMAppAttemptImpl(appAttemptId, rmContext, scheduler, masterService,
        submissionContext, conf,
        // The newly created attempt maybe last attempt if (number of
        // previously failed attempts(which should not include Preempted,
        // hardware error and NM resync) + 1) equal to the max-attempt
        // limit.
        maxAppAttempts == (getNumFailedAppAttempts() + 1), amReq);
  attempts.put(appAttemptId, attempt);
  currentAttempt = attempt;
}
 

@Test
public void testApplicationAttemptReport() throws IOException, YarnException {
  ApplicationId appId = ApplicationId.newInstance(0, 1);
  ApplicationAttemptId appAttemptId =
      ApplicationAttemptId.newInstance(appId, 1);
  GetApplicationAttemptReportRequest request =
      GetApplicationAttemptReportRequest.newInstance(appAttemptId);
  GetApplicationAttemptReportResponse response =
      clientService.getApplicationAttemptReport(request);
  ApplicationAttemptReport attemptReport =
      response.getApplicationAttemptReport();
  Assert.assertNotNull(attemptReport);
  Assert.assertEquals("appattempt_0_0001_000001", attemptReport
    .getApplicationAttemptId().toString());
}
 

@Test (expected = YarnException.class)
public void testMoveWouldViolateMaxAppsConstraints() throws Exception {
  scheduler.init(conf);
  scheduler.start();
  scheduler.reinitialize(conf, resourceManager.getRMContext());

  QueueManager queueMgr = scheduler.getQueueManager();
  queueMgr.getLeafQueue("queue2", true);
  scheduler.getAllocationConfiguration().queueMaxApps.put("root.queue2", 0);
  
  ApplicationAttemptId appAttId =
      createSchedulingRequest(1024, 1, 1, "queue1", "user1", 3);
  
  scheduler.moveApplication(appAttId.getApplicationId(), "queue2");
}
 

@Override
public Map<ApplicationAttemptId, RMAppAttempt> getAppAttempts() {
  Map<ApplicationAttemptId, RMAppAttempt> attempts =
    new LinkedHashMap<ApplicationAttemptId, RMAppAttempt>();
  if(attempt != null) {
    attempts.put(attempt.getAppAttemptId(), attempt);
  }
  return attempts;
}
 

private void
    getContainerStatus(YarnRPC rpc,
        org.apache.hadoop.yarn.api.records.Token nmToken,
        ContainerId containerId,
        ApplicationAttemptId appAttemptId, NodeId nodeId,
        boolean isExceptionExpected) throws Exception {
  List<ContainerId> containerIds = new ArrayList<ContainerId>();
  containerIds.add(containerId);
  GetContainerStatusesRequest request =
      GetContainerStatusesRequest.newInstance(containerIds);
  ContainerManagementProtocol proxy = null;
  try {
    proxy =
        getContainerManagementProtocolProxy(rpc, nmToken, nodeId,
            appAttemptId.toString());
    GetContainerStatusesResponse statuses = proxy.getContainerStatuses(request);
    if (statuses.getFailedRequests() != null
        && statuses.getFailedRequests().containsKey(containerId)) {
      parseAndThrowException(statuses.getFailedRequests().get(containerId)
        .deSerialize());
    }
  } finally {
    if (proxy != null) {
      rpc.stopProxy(proxy, conf);
    }
  }
}
 

public void sendAMLaunchFailed(ApplicationAttemptId appAttemptId)
    throws Exception {
  MockAM am = new MockAM(getRMContext(), masterService, appAttemptId);
  am.waitForState(RMAppAttemptState.ALLOCATED);
  getRMContext().getDispatcher().getEventHandler()
      .handle(new RMAppAttemptLaunchFailedEvent(appAttemptId, "Failed"));
}
 

@Test(timeout = 10000)
public void testGetContainerReport() throws YarnException, IOException {
  Configuration conf = new Configuration();
  conf.setBoolean(YarnConfiguration.APPLICATION_HISTORY_ENABLED,
      true);
  final YarnClient client = new MockYarnClient();
  client.init(conf);
  client.start();

  List<ApplicationReport> expectedReports = ((MockYarnClient) client)
      .getReports();

  ApplicationId applicationId = ApplicationId.newInstance(1234, 5);
  ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(
      applicationId, 1);
  ContainerId containerId = ContainerId.newContainerId(appAttemptId, 1);
  ContainerReport report = client.getContainerReport(containerId);
  Assert.assertNotNull(report);
  Assert.assertEquals(report.getContainerId().toString(),
      (ContainerId.newContainerId(expectedReports.get(0)
          .getCurrentApplicationAttemptId(), 1)).toString());
  containerId = ContainerId.newContainerId(appAttemptId, 3);
  report = client.getContainerReport(containerId);
  Assert.assertNotNull(report);
  Assert.assertEquals(report.getContainerId().toString(),
      (ContainerId.newContainerId(expectedReports.get(0)
          .getCurrentApplicationAttemptId(), 3)).toString());
  client.stop();
}
 

@Override
public void preemptContainer(ApplicationAttemptId aid, RMContainer cont) {
  if(LOG.isDebugEnabled()){
    LOG.debug("PREEMPT_CONTAINER: application:" + aid.toString() +
        " container: " + cont.toString());
  }
  FiCaSchedulerApp app = getApplicationAttempt(aid);
  if (app != null) {
    app.addPreemptContainer(cont.getContainerId());
  }
}
 

@Override
public List<ContainerReport>
    getContainers(ApplicationAttemptId appAttemptId) throws YarnException,
        IOException {
  when(mockContainersResponse.getContainerList()).thenReturn(
    getContainersReport(appAttemptId));
  return super.getContainers(appAttemptId);
}
 

MyAppContext(int numberMaps, int numberReduces) {
  myApplicationID = ApplicationId.newInstance(clock.getTime(), 1);

  myAppAttemptID = ApplicationAttemptId.newInstance(myApplicationID, 0);
  myJobID = recordFactory.newRecordInstance(JobId.class);
  myJobID.setAppId(myApplicationID);

  Job myJob
      = new MyJobImpl(myJobID, numberMaps, numberReduces);

  allJobs = Collections.singletonMap(myJobID, myJob);
}
 

private void killApplication(MockRM rm, ApplicationId appId,
    ApplicationAttemptId appAttemptId, RMAppState rmAppState)
    throws Exception {
  KillApplicationResponse response = rm.killApp(appId);
  Assert
      .assertTrue(response.getIsKillCompleted() == isFinalState(rmAppState));
  RMApp loadedApp0 =
      rm.getRMContext().getRMApps().get(appId);
  rm.waitForState(appId, RMAppState.KILLED);
  if (appAttemptId != null) {
    rm.waitForState(appAttemptId, RMAppAttemptState.KILLED);
  }
  // no new attempt is created.
  Assert.assertEquals(1, loadedApp0.getAppAttempts().size());
}