Java Code Examples for org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState#FINISHED

The following examples show how to use org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState#FINISHED . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: TestSchedulerUtils.java    From hadoop with Apache License 2.0 6 votes vote down vote up 
public static SchedulerApplication<SchedulerApplicationAttempt>
    verifyAppAddedAndRemovedFromScheduler(
        Map<ApplicationId, SchedulerApplication<SchedulerApplicationAttempt>> applications,
        EventHandler<SchedulerEvent> handler, String queueName)
        throws Exception {
  ApplicationId appId =
      ApplicationId.newInstance(System.currentTimeMillis(), 1);
  AppAddedSchedulerEvent appAddedEvent =
      new AppAddedSchedulerEvent(appId, queueName, "user");
  handler.handle(appAddedEvent);
  SchedulerApplication<SchedulerApplicationAttempt> app =
      applications.get(appId);
  // verify application is added.
  Assert.assertNotNull(app);
  Assert.assertEquals("user", app.getUser());

  AppRemovedSchedulerEvent appRemoveEvent =
      new AppRemovedSchedulerEvent(appId, RMAppState.FINISHED);
  handler.handle(appRemoveEvent);
  Assert.assertNull(applications.get(appId));
  return app;
}
Example 2
Source File: TestSchedulerUtils.java    From big-c with Apache License 2.0 6 votes vote down vote up 
public static SchedulerApplication<SchedulerApplicationAttempt>
    verifyAppAddedAndRemovedFromScheduler(
        Map<ApplicationId, SchedulerApplication<SchedulerApplicationAttempt>> applications,
        EventHandler<SchedulerEvent> handler, String queueName)
        throws Exception {
  ApplicationId appId =
      ApplicationId.newInstance(System.currentTimeMillis(), 1);
  AppAddedSchedulerEvent appAddedEvent =
      new AppAddedSchedulerEvent(appId, queueName, "user");
  handler.handle(appAddedEvent);
  SchedulerApplication<SchedulerApplicationAttempt> app =
      applications.get(appId);
  // verify application is added.
  Assert.assertNotNull(app);
  Assert.assertEquals("user", app.getUser());

  AppRemovedSchedulerEvent appRemoveEvent =
      new AppRemovedSchedulerEvent(appId, RMAppState.FINISHED);
  handler.handle(appRemoveEvent);
  Assert.assertNull(applications.get(appId));
  return app;
}
Example 3
Source File: TestAppManager.java    From hadoop with Apache License 2.0 5 votes vote down vote up 
protected void addToCompletedApps(TestRMAppManager appMonitor, RMContext rmContext) {
  for (RMApp app : rmContext.getRMApps().values()) {
    if (app.getState() == RMAppState.FINISHED
        || app.getState() == RMAppState.KILLED 
        || app.getState() == RMAppState.FAILED) {
      appMonitor.finishApplication(app.getApplicationId());
    }
  }
}
Example 4
Source File: TestAppManager.java    From big-c with Apache License 2.0 5 votes vote down vote up 
protected void addToCompletedApps(TestRMAppManager appMonitor, RMContext rmContext) {
  for (RMApp app : rmContext.getRMApps().values()) {
    if (app.getState() == RMAppState.FINISHED
        || app.getState() == RMAppState.KILLED 
        || app.getState() == RMAppState.FAILED) {
      appMonitor.finishApplication(app.getApplicationId());
    }
  }
}
Example 5
Source File: RMContextImplEventRunnable.java    From garmadon with Apache License 2.0 4 votes vote down vote up 
public void sendAppEvent(ApplicationId applicationId, RMApp rmApp) {
    if (cacheFinishedApp.getIfPresent(applicationId.toString()) == null) {
        Header.Builder headerBuilder = Header.newBuilder()
            .withId(applicationId.toString())
            .withApplicationID(applicationId.toString())
            .withUser(rmApp.getUser())
            .withApplicationName(rmApp.getName())
            .withFramework(rmApp.getApplicationType().toUpperCase());

        ApplicationEvent.Builder eventBuilder = ApplicationEvent.newBuilder()
            .setState(rmApp.getState().name())
            .setQueue(rmApp.getQueue());

        rmApp.getApplicationTags().stream()
            .filter(tag -> YARN_TAGS_TO_EXTRACT.stream().noneMatch(tag::startsWith) && !tag.contains(":"))
            .forEach(eventBuilder::addYarnTags);

        rmApp.getApplicationTags().stream()
            .filter(tag -> tag.contains(":") && YARN_TAGS_TO_EXTRACT.stream().anyMatch(tag::startsWith))
            .map(tag -> {
                int idx = tag.indexOf(':');
                String key = tag.substring(0, idx);
                String value = tag.substring(idx + 1);
                return new String[] {key, value};
            })
            .forEach(splitTag -> BUILDERS.get(splitTag[0]).accept(splitTag[1], eventBuilder));

        eventBuilder.setFinalStatus(rmApp.getFinalApplicationStatus().name());

        eventBuilder.setStartTime(rmApp.getStartTime());
        eventBuilder.setFinishTime(rmApp.getFinishTime());

        RMAppMetrics rmAppMetrics = rmApp.getRMAppMetrics();
        if (rmAppMetrics != null) {
            eventBuilder.setMemorySeconds(rmAppMetrics.getMemorySeconds());
            eventBuilder.setVcoreSeconds(rmAppMetrics.getVcoreSeconds());
        }

        RMAppAttempt rmAppAttempt = rmApp.getCurrentAppAttempt();
        if (rmAppAttempt != null) {
            headerBuilder.withAttemptID(rmAppAttempt.getAppAttemptId().toString());

            Container container = rmAppAttempt.getMasterContainer();
            if (container != null) {
                eventBuilder.setAmContainerId(container.getId().toString());
            }
        }

        if (rmApp.getTrackingUrl() != null) {
            eventBuilder.setTrackingUrl(normalizeTrackingUrl(rmApp.getTrackingUrl()));
        }

        if (rmApp.getOriginalTrackingUrl() != null && !"N/A".equals(rmApp.getOriginalTrackingUrl())) {
            eventBuilder.setOriginalTrackingUrl(normalizeTrackingUrl(rmApp.getOriginalTrackingUrl()));
        }

        eventHandler.accept(System.currentTimeMillis(), headerBuilder.build(), eventBuilder.build());

        if (rmApp.getState() == RMAppState.FINISHED || rmApp.getState() == RMAppState.KILLED || rmApp.getState() == RMAppState.FAILED) {
            cacheFinishedApp.put(applicationId.toString(), rmApp.getState().name());
        }
    }
}
Example 6
Source File: TestAppManager.java    From hadoop with Apache License 2.0 4 votes vote down vote up 
@Test
public void testRMAppRetireSomeDifferentStates() throws Exception {
  long now = System.currentTimeMillis();

  // these parameters don't matter, override applications below
  RMContext rmContext = mockRMContext(10, now - 20000);
  Configuration conf = new YarnConfiguration();
  conf.setInt(YarnConfiguration.RM_STATE_STORE_MAX_COMPLETED_APPLICATIONS, 2);
  conf.setInt(YarnConfiguration.RM_MAX_COMPLETED_APPLICATIONS, 2);

  TestRMAppManager appMonitor = new TestRMAppManager(rmContext, conf);

  // clear out applications map
  rmContext.getRMApps().clear();
  Assert.assertEquals("map isn't empty", 0, rmContext.getRMApps().size());

  // 6 applications are in final state, 4 are not in final state.
  // / set with various finished states
  RMApp app = new MockRMApp(0, now - 20000, RMAppState.KILLED);
  rmContext.getRMApps().put(app.getApplicationId(), app);
  app = new MockRMApp(1, now - 200000, RMAppState.FAILED);
  rmContext.getRMApps().put(app.getApplicationId(), app);
  app = new MockRMApp(2, now - 30000, RMAppState.FINISHED);
  rmContext.getRMApps().put(app.getApplicationId(), app);
  app = new MockRMApp(3, now - 20000, RMAppState.RUNNING);
  rmContext.getRMApps().put(app.getApplicationId(), app);
  app = new MockRMApp(4, now - 20000, RMAppState.NEW);
  rmContext.getRMApps().put(app.getApplicationId(), app);

  // make sure it doesn't expire these since still running
  app = new MockRMApp(5, now - 10001, RMAppState.KILLED);
  rmContext.getRMApps().put(app.getApplicationId(), app);
  app = new MockRMApp(6, now - 30000, RMAppState.ACCEPTED);
  rmContext.getRMApps().put(app.getApplicationId(), app);
  app = new MockRMApp(7, now - 20000, RMAppState.SUBMITTED);
  rmContext.getRMApps().put(app.getApplicationId(), app);
  app = new MockRMApp(8, now - 10001, RMAppState.FAILED);
  rmContext.getRMApps().put(app.getApplicationId(), app);
  app = new MockRMApp(9, now - 20000, RMAppState.FAILED);
  rmContext.getRMApps().put(app.getApplicationId(), app);

  Assert.assertEquals("Number of apps incorrect before", 10, rmContext
      .getRMApps().size());

  // add them to completed apps list
  addToCompletedApps(appMonitor, rmContext);

  // shouldn't  have to many apps
  appMonitor.checkAppNumCompletedLimit();
  Assert.assertEquals("Number of apps incorrect after # completed check", 6,
      rmContext.getRMApps().size());
  Assert.assertEquals("Number of completed apps incorrect after check", 2,
      appMonitor.getCompletedAppsListSize());
  // 6 applications in final state, 4 of them are removed
  verify(rmContext.getStateStore(), times(4)).removeApplication(
    isA(RMApp.class));
}
Example 7
Source File: TestLeafQueue.java    From hadoop with Apache License 2.0 4 votes vote down vote up 
@Test
public void testAppAttemptMetrics() throws Exception {

  // Manipulate queue 'a'
  LeafQueue a = stubLeafQueue((LeafQueue) queues.get(B));

  // Users
  final String user_0 = "user_0";

  // Submit applications
  final ApplicationAttemptId appAttemptId_0 = TestUtils
      .getMockApplicationAttemptId(0, 1);

  AppAddedSchedulerEvent addAppEvent =
      new AppAddedSchedulerEvent(appAttemptId_0.getApplicationId(),
        a.getQueueName(), user_0);
  cs.handle(addAppEvent);
  AppAttemptAddedSchedulerEvent addAttemptEvent = 
      new AppAttemptAddedSchedulerEvent(appAttemptId_0, false);
  cs.handle(addAttemptEvent);

  AppAttemptRemovedSchedulerEvent event = new AppAttemptRemovedSchedulerEvent(
      appAttemptId_0, RMAppAttemptState.FAILED, false);
  cs.handle(event);
  
  assertEquals(0, a.getMetrics().getAppsPending());
  assertEquals(0, a.getMetrics().getAppsFailed());

  // Attempt the same application again
  final ApplicationAttemptId appAttemptId_1 = TestUtils
      .getMockApplicationAttemptId(0, 2);
  FiCaSchedulerApp app_1 = new FiCaSchedulerApp(appAttemptId_1, user_0, a, null,
      spyRMContext);
  a.submitApplicationAttempt(app_1, user_0); // same user

  assertEquals(1, a.getMetrics().getAppsSubmitted());
  assertEquals(1, a.getMetrics().getAppsPending());
  
  event = new AppAttemptRemovedSchedulerEvent(appAttemptId_0,
      RMAppAttemptState.FINISHED, false);
  cs.handle(event);
  AppRemovedSchedulerEvent rEvent = new AppRemovedSchedulerEvent(
      appAttemptId_0.getApplicationId(), RMAppState.FINISHED);
  cs.handle(rEvent);
  
  assertEquals(1, a.getMetrics().getAppsSubmitted());
  assertEquals(0, a.getMetrics().getAppsPending());
  assertEquals(0, a.getMetrics().getAppsFailed());
  assertEquals(1, a.getMetrics().getAppsCompleted());

  QueueMetrics userMetrics = a.getMetrics().getUserMetrics(user_0);
  assertEquals(1, userMetrics.getAppsSubmitted());
}
Example 8
Source File: TestAppManager.java    From big-c with Apache License 2.0 4 votes vote down vote up 
@Test
public void testRMAppRetireSomeDifferentStates() throws Exception {
  long now = System.currentTimeMillis();

  // these parameters don't matter, override applications below
  RMContext rmContext = mockRMContext(10, now - 20000);
  Configuration conf = new YarnConfiguration();
  conf.setInt(YarnConfiguration.RM_STATE_STORE_MAX_COMPLETED_APPLICATIONS, 2);
  conf.setInt(YarnConfiguration.RM_MAX_COMPLETED_APPLICATIONS, 2);

  TestRMAppManager appMonitor = new TestRMAppManager(rmContext, conf);

  // clear out applications map
  rmContext.getRMApps().clear();
  Assert.assertEquals("map isn't empty", 0, rmContext.getRMApps().size());

  // 6 applications are in final state, 4 are not in final state.
  // / set with various finished states
  RMApp app = new MockRMApp(0, now - 20000, RMAppState.KILLED);
  rmContext.getRMApps().put(app.getApplicationId(), app);
  app = new MockRMApp(1, now - 200000, RMAppState.FAILED);
  rmContext.getRMApps().put(app.getApplicationId(), app);
  app = new MockRMApp(2, now - 30000, RMAppState.FINISHED);
  rmContext.getRMApps().put(app.getApplicationId(), app);
  app = new MockRMApp(3, now - 20000, RMAppState.RUNNING);
  rmContext.getRMApps().put(app.getApplicationId(), app);
  app = new MockRMApp(4, now - 20000, RMAppState.NEW);
  rmContext.getRMApps().put(app.getApplicationId(), app);

  // make sure it doesn't expire these since still running
  app = new MockRMApp(5, now - 10001, RMAppState.KILLED);
  rmContext.getRMApps().put(app.getApplicationId(), app);
  app = new MockRMApp(6, now - 30000, RMAppState.ACCEPTED);
  rmContext.getRMApps().put(app.getApplicationId(), app);
  app = new MockRMApp(7, now - 20000, RMAppState.SUBMITTED);
  rmContext.getRMApps().put(app.getApplicationId(), app);
  app = new MockRMApp(8, now - 10001, RMAppState.FAILED);
  rmContext.getRMApps().put(app.getApplicationId(), app);
  app = new MockRMApp(9, now - 20000, RMAppState.FAILED);
  rmContext.getRMApps().put(app.getApplicationId(), app);

  Assert.assertEquals("Number of apps incorrect before", 10, rmContext
      .getRMApps().size());

  // add them to completed apps list
  addToCompletedApps(appMonitor, rmContext);

  // shouldn't  have to many apps
  appMonitor.checkAppNumCompletedLimit();
  Assert.assertEquals("Number of apps incorrect after # completed check", 6,
      rmContext.getRMApps().size());
  Assert.assertEquals("Number of completed apps incorrect after check", 2,
      appMonitor.getCompletedAppsListSize());
  // 6 applications in final state, 4 of them are removed
  verify(rmContext.getStateStore(), times(4)).removeApplication(
    isA(RMApp.class));
}
Example 9
Source File: TestLeafQueue.java    From big-c with Apache License 2.0 4 votes vote down vote up 
@Test
public void testAppAttemptMetrics() throws Exception {

  // Manipulate queue 'a'
  LeafQueue a = stubLeafQueue((LeafQueue) queues.get(B));

  // Users
  final String user_0 = "user_0";

  // Submit applications
  final ApplicationAttemptId appAttemptId_0 = TestUtils
      .getMockApplicationAttemptId(0, 1);

  AppAddedSchedulerEvent addAppEvent =
      new AppAddedSchedulerEvent(appAttemptId_0.getApplicationId(),
        a.getQueueName(), user_0);
  cs.handle(addAppEvent);
  AppAttemptAddedSchedulerEvent addAttemptEvent = 
      new AppAttemptAddedSchedulerEvent(appAttemptId_0, false);
  cs.handle(addAttemptEvent);

  AppAttemptRemovedSchedulerEvent event = new AppAttemptRemovedSchedulerEvent(
      appAttemptId_0, RMAppAttemptState.FAILED, false);
  cs.handle(event);
  
  assertEquals(0, a.getMetrics().getAppsPending());
  assertEquals(0, a.getMetrics().getAppsFailed());

  // Attempt the same application again
  final ApplicationAttemptId appAttemptId_1 = TestUtils
      .getMockApplicationAttemptId(0, 2);
  FiCaSchedulerApp app_1 = new FiCaSchedulerApp(appAttemptId_1, user_0, a, null,
      spyRMContext);
  a.submitApplicationAttempt(app_1, user_0); // same user

  assertEquals(1, a.getMetrics().getAppsSubmitted());
  assertEquals(1, a.getMetrics().getAppsPending());
  
  event = new AppAttemptRemovedSchedulerEvent(appAttemptId_0,
      RMAppAttemptState.FINISHED, false);
  cs.handle(event);
  AppRemovedSchedulerEvent rEvent = new AppRemovedSchedulerEvent(
      appAttemptId_0.getApplicationId(), RMAppState.FINISHED);
  cs.handle(rEvent);
  
  assertEquals(1, a.getMetrics().getAppsSubmitted());
  assertEquals(0, a.getMetrics().getAppsPending());
  assertEquals(0, a.getMetrics().getAppsFailed());
  assertEquals(1, a.getMetrics().getAppsCompleted());

  QueueMetrics userMetrics = a.getMetrics().getUserMetrics(user_0);
  assertEquals(1, userMetrics.getAppsSubmitted());
}