org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppMetrics Java Examples

private void waitForAppPreemptionInfo(RMApp app, Resource preempted,
    int numAMPreempted, int numTaskPreempted,
    Resource currentAttemptPreempted, boolean currentAttemptAMPreempted,
    int numLatestAttemptTaskPreempted) throws InterruptedException {
  while (true) {
    RMAppMetrics appPM = app.getRMAppMetrics();
    RMAppAttemptMetrics attemptPM =
        app.getCurrentAppAttempt().getRMAppAttemptMetrics();

    if (appPM.getResourcePreempted().equals(preempted)
        && appPM.getNumAMContainersPreempted() == numAMPreempted
        && appPM.getNumNonAMContainersPreempted() == numTaskPreempted
        && attemptPM.getResourcePreempted().equals(currentAttemptPreempted)
        && app.getCurrentAppAttempt().getRMAppAttemptMetrics()
          .getIsPreempted() == currentAttemptAMPreempted
        && attemptPM.getNumNonAMContainersPreempted() == 
           numLatestAttemptTaskPreempted) {
      return;
    }
    Thread.sleep(500);
  }
}
 

public ApplicationFinishedEvent(
    ApplicationId appId,
    String diagnosticsInfo,
    FinalApplicationStatus appStatus,
    YarnApplicationState state,
    ApplicationAttemptId latestAppAttemptId,
    long finishedTime,
    RMAppMetrics appMetrics) {
  super(SystemMetricsEventType.APP_FINISHED, finishedTime);
  this.appId = appId;
  this.diagnosticsInfo = diagnosticsInfo;
  this.appStatus = appStatus;
  this.latestAppAttemptId = latestAppAttemptId;
  this.state = state;
  this.appMetrics=appMetrics;
}
 

public ApplicationFinishedEvent(
    ApplicationId appId,
    String diagnosticsInfo,
    FinalApplicationStatus appStatus,
    YarnApplicationState state,
    ApplicationAttemptId latestAppAttemptId,
    long finishedTime,
    RMAppMetrics appMetrics) {
  super(SystemMetricsEventType.APP_FINISHED, finishedTime);
  this.appId = appId;
  this.diagnosticsInfo = diagnosticsInfo;
  this.appStatus = appStatus;
  this.latestAppAttemptId = latestAppAttemptId;
  this.state = state;
  this.appMetrics=appMetrics;
}
 

private void waitForAppPreemptionInfo(RMApp app, Resource preempted,
    int numAMPreempted, int numTaskPreempted,
    Resource currentAttemptPreempted, boolean currentAttemptAMPreempted,
    int numLatestAttemptTaskPreempted) throws InterruptedException {
  while (true) {
    RMAppMetrics appPM = app.getRMAppMetrics();
    RMAppAttemptMetrics attemptPM =
        app.getCurrentAppAttempt().getRMAppAttemptMetrics();

    if (appPM.getResourcePreempted().equals(preempted)
        && appPM.getNumAMContainersPreempted() == numAMPreempted
        && appPM.getNumNonAMContainersPreempted() == numTaskPreempted
        && attemptPM.getResourcePreempted().equals(currentAttemptPreempted)
        && app.getCurrentAppAttempt().getRMAppAttemptMetrics()
          .getIsPreempted() == currentAttemptAMPreempted
        && attemptPM.getNumNonAMContainersPreempted() == 
           numLatestAttemptTaskPreempted) {
      return;
    }
    Thread.sleep(500);
  }
}
 

private static RMApp createRMApp(ApplicationId appId) {
  RMApp app = mock(RMApp.class);
  when(app.getApplicationId()).thenReturn(appId);
  when(app.getName()).thenReturn("test app");
  when(app.getApplicationType()).thenReturn("test app type");
  when(app.getUser()).thenReturn("test user");
  when(app.getQueue()).thenReturn("test queue");
  when(app.getSubmitTime()).thenReturn(Integer.MAX_VALUE + 1L);
  when(app.getStartTime()).thenReturn(Integer.MAX_VALUE + 2L);
  when(app.getFinishTime()).thenReturn(Integer.MAX_VALUE + 3L);
  when(app.getDiagnostics()).thenReturn(
      new StringBuilder("test diagnostics info"));
  RMAppAttempt appAttempt = mock(RMAppAttempt.class);
  when(appAttempt.getAppAttemptId()).thenReturn(
      ApplicationAttemptId.newInstance(appId, 1));
  when(app.getCurrentAppAttempt()).thenReturn(appAttempt);
  when(app.getFinalApplicationStatus()).thenReturn(
      FinalApplicationStatus.UNDEFINED);
  when(app.getRMAppMetrics()).thenReturn(
      new RMAppMetrics(null, 0, 0, Integer.MAX_VALUE, Long.MAX_VALUE));
  return app;
}
 

private static RMApp createRMApp(ApplicationId appId) {
  RMApp app = mock(RMApp.class);
  when(app.getApplicationId()).thenReturn(appId);
  when(app.getName()).thenReturn("test app");
  when(app.getApplicationType()).thenReturn("test app type");
  when(app.getUser()).thenReturn("test user");
  when(app.getQueue()).thenReturn("test queue");
  when(app.getSubmitTime()).thenReturn(Integer.MAX_VALUE + 1L);
  when(app.getStartTime()).thenReturn(Integer.MAX_VALUE + 2L);
  when(app.getFinishTime()).thenReturn(Integer.MAX_VALUE + 3L);
  when(app.getDiagnostics()).thenReturn(
      new StringBuilder("test diagnostics info"));
  RMAppAttempt appAttempt = mock(RMAppAttempt.class);
  when(appAttempt.getAppAttemptId()).thenReturn(
      ApplicationAttemptId.newInstance(appId, 1));
  when(app.getCurrentAppAttempt()).thenReturn(appAttempt);
  when(app.getFinalApplicationStatus()).thenReturn(
      FinalApplicationStatus.UNDEFINED);
  when(app.getRMAppMetrics()).thenReturn(
      new RMAppMetrics(null, 0, 0, Integer.MAX_VALUE, Long.MAX_VALUE, Long.MAX_VALUE));
  return app;
}
 

private static RMContext mockRMContext(List<RMAppState> states) {
  final ConcurrentMap<ApplicationId, RMApp> applicationsMaps = Maps
      .newConcurrentMap();
  int i = 0;
  for (RMAppState state : states) {
    MockRMApp app = new MockRMApp(i, i, state) {
      @Override
      public RMAppMetrics getRMAppMetrics() {
        return new RMAppMetrics(Resource.newInstance(0, 0, 0), 0, 0, 0, 0, 0);
      }
      @Override
      public YarnApplicationState createApplicationState() {
        return YarnApplicationState.ACCEPTED;
      }
    };
    RMAppAttempt attempt = mock(RMAppAttempt.class);
    app.setCurrentAppAttempt(attempt);
    applicationsMaps.put(app.getApplicationId(), app);
    i++;
  }

  RMContextImpl rmContext =  new RMContextImpl(null, null, null, null,
      null, null, null, null, null, null) {
    @Override
    public ConcurrentMap<ApplicationId, RMApp> getRMApps() {
      return applicationsMaps;
    }
    @Override
    public ResourceScheduler getScheduler() {
      return mock(AbstractYarnScheduler.class);
    }
  };
  return rmContext;
}
 

private void publishApplicationFinishedEvent(ApplicationFinishedEvent event) {
  TimelineEntity entity =
      createApplicationEntity(event.getApplicationId());
  TimelineEvent tEvent = new TimelineEvent();
  tEvent.setEventType(
      ApplicationMetricsConstants.FINISHED_EVENT_TYPE);
  tEvent.setTimestamp(event.getTimestamp());
  Map<String, Object> eventInfo = new HashMap<String, Object>();
  eventInfo.put(ApplicationMetricsConstants.DIAGNOSTICS_INFO_EVENT_INFO,
      event.getDiagnosticsInfo());
  eventInfo.put(ApplicationMetricsConstants.FINAL_STATUS_EVENT_INFO,
      event.getFinalApplicationStatus().toString());
  eventInfo.put(ApplicationMetricsConstants.STATE_EVENT_INFO,
      event.getYarnApplicationState().toString());
  if (event.getLatestApplicationAttemptId() != null) {
    eventInfo.put(ApplicationMetricsConstants.LATEST_APP_ATTEMPT_EVENT_INFO,
        event.getLatestApplicationAttemptId().toString());
  }
  RMAppMetrics appMetrics = event.getAppMetrics();
  entity.addOtherInfo(ApplicationMetricsConstants.APP_CPU_METRICS,
      appMetrics.getVcoreSeconds());
  entity.addOtherInfo(ApplicationMetricsConstants.APP_MEM_METRICS,
      appMetrics.getMemorySeconds());
  
  tEvent.setEventInfo(eventInfo);
  entity.addEvent(tEvent);
  putEntity(entity);
}
 

/**
 * create a summary of the application's runtime.
 * 
 * @param app {@link RMApp} whose summary is to be created, cannot
 *            be <code>null</code>.
 */
public static SummaryBuilder createAppSummary(RMApp app) {
  String trackingUrl = "N/A";
  String host = "N/A";
  RMAppAttempt attempt = app.getCurrentAppAttempt();
  if (attempt != null) {
    trackingUrl = attempt.getTrackingUrl();
    host = attempt.getHost();
  }
  RMAppMetrics metrics = app.getRMAppMetrics();
  SummaryBuilder summary = new SummaryBuilder()
      .add("appId", app.getApplicationId())
      .add("name", app.getName())
      .add("user", app.getUser())
      .add("queue", app.getQueue())
      .add("state", app.getState())
      .add("trackingUrl", trackingUrl)
      .add("appMasterHost", host)
      .add("startTime", app.getStartTime())
      .add("finishTime", app.getFinishTime())
      .add("finalStatus", app.getFinalApplicationStatus())
      .add("memorySeconds", metrics.getMemorySeconds())
      .add("vcoreSeconds", metrics.getVcoreSeconds())
      .add("preemptedAMContainers", metrics.getNumAMContainersPreempted())
      .add("preemptedNonAMContainers", metrics.getNumNonAMContainersPreempted())
      .add("preemptedResources", metrics.getResourcePreempted())
      .add("applicationType", app.getApplicationType());
  return summary;
}
 

@Test (timeout = 30000)
 public void testEscapeApplicationSummary() {
   RMApp app = mock(RMAppImpl.class);
   when(app.getApplicationId()).thenReturn(
       ApplicationId.newInstance(100L, 1));
   when(app.getName()).thenReturn("Multiline\n\n\r\rAppName");
   when(app.getUser()).thenReturn("Multiline\n\n\r\rUserName");
   when(app.getQueue()).thenReturn("Multiline\n\n\r\rQueueName");
   when(app.getState()).thenReturn(RMAppState.RUNNING);
   when(app.getApplicationType()).thenReturn("MAPREDUCE");
   RMAppMetrics metrics =
       new RMAppMetrics(Resource.newInstance(1234, 56), 10, 1, 16384, 64);
   when(app.getRMAppMetrics()).thenReturn(metrics);

   RMAppManager.ApplicationSummary.SummaryBuilder summary =
       new RMAppManager.ApplicationSummary().createAppSummary(app);
   String msg = summary.toString();
   LOG.info("summary: " + msg);
   Assert.assertFalse(msg.contains("\n"));
   Assert.assertFalse(msg.contains("\r"));

   String escaped = "\\n\\n\\r\\r";
   Assert.assertTrue(msg.contains("Multiline" + escaped +"AppName"));
   Assert.assertTrue(msg.contains("Multiline" + escaped +"UserName"));
   Assert.assertTrue(msg.contains("Multiline" + escaped +"QueueName"));
   Assert.assertTrue(msg.contains("memorySeconds=16384"));
   Assert.assertTrue(msg.contains("vcoreSeconds=64"));
   Assert.assertTrue(msg.contains("preemptedAMContainers=1"));
   Assert.assertTrue(msg.contains("preemptedNonAMContainers=10"));
   Assert.assertTrue(msg.contains("preemptedResources=<memory:1234\\, vCores:56>"));
   Assert.assertTrue(msg.contains("applicationType=MAPREDUCE"));
}
 

private static RMContext mockRMContext(List<RMAppState> states) {
  final ConcurrentMap<ApplicationId, RMApp> applicationsMaps = Maps
      .newConcurrentMap();
  int i = 0;
  for (RMAppState state : states) {
    MockRMApp app = new MockRMApp(i, i, state) {
      @Override
      public RMAppMetrics getRMAppMetrics() {
        return new RMAppMetrics(Resource.newInstance(0, 0), 0, 0, 0, 0);
      }
      @Override
      public YarnApplicationState createApplicationState() {
        return YarnApplicationState.ACCEPTED;
      }
    };
    RMAppAttempt attempt = mock(RMAppAttempt.class);
    app.setCurrentAppAttempt(attempt);
    applicationsMaps.put(app.getApplicationId(), app);
    i++;
  }

  RMContextImpl rmContext =  new RMContextImpl(null, null, null, null,
      null, null, null, null, null, null) {
    @Override
    public ConcurrentMap<ApplicationId, RMApp> getRMApps() {
      return applicationsMaps;
    }
    @Override
    public ResourceScheduler getScheduler() {
      return mock(AbstractYarnScheduler.class);
    }
  };
  return rmContext;
}
 

@Test (timeout = 30000)
 public void testEscapeApplicationSummary() {
   RMApp app = mock(RMAppImpl.class);
   when(app.getApplicationId()).thenReturn(
       ApplicationId.newInstance(100L, 1));
   when(app.getName()).thenReturn("Multiline\n\n\r\rAppName");
   when(app.getUser()).thenReturn("Multiline\n\n\r\rUserName");
   when(app.getQueue()).thenReturn("Multiline\n\n\r\rQueueName");
   when(app.getState()).thenReturn(RMAppState.RUNNING);
   when(app.getApplicationType()).thenReturn("MAPREDUCE");
   RMAppMetrics metrics =
       new RMAppMetrics(Resource.newInstance(1234, 56, 56), 10, 1, 16384, 64, 64);
   when(app.getRMAppMetrics()).thenReturn(metrics);

   RMAppManager.ApplicationSummary.SummaryBuilder summary =
       new RMAppManager.ApplicationSummary().createAppSummary(app);
   String msg = summary.toString();
   LOG.info("summary: " + msg);
   Assert.assertFalse(msg.contains("\n"));
   Assert.assertFalse(msg.contains("\r"));

   String escaped = "\\n\\n\\r\\r";
   Assert.assertTrue(msg.contains("Multiline" + escaped +"AppName"));
   Assert.assertTrue(msg.contains("Multiline" + escaped +"UserName"));
   Assert.assertTrue(msg.contains("Multiline" + escaped +"QueueName"));
   Assert.assertTrue(msg.contains("memorySeconds=16384"));
   Assert.assertTrue(msg.contains("vcoreSeconds=64"));
   Assert.assertTrue(msg.contains("gcoreSeconds=64"));
   Assert.assertTrue(msg.contains("preemptedAMContainers=1"));
   Assert.assertTrue(msg.contains("preemptedNonAMContainers=10"));
   Assert.assertTrue(msg.contains("preemptedResources=<memory:1234\\, vCores:56\\, gCores:56>"));
   Assert.assertTrue(msg.contains("applicationType=MAPREDUCE"));
}
 

/**
 * create a summary of the application's runtime.
 * 
 * @param app {@link RMApp} whose summary is to be created, cannot
 *            be <code>null</code>.
 */
public static SummaryBuilder createAppSummary(RMApp app) {
  String trackingUrl = "N/A";
  String host = "N/A";
  RMAppAttempt attempt = app.getCurrentAppAttempt();
  if (attempt != null) {
    trackingUrl = attempt.getTrackingUrl();
    host = attempt.getHost();
  }
  RMAppMetrics metrics = app.getRMAppMetrics();
  SummaryBuilder summary = new SummaryBuilder()
      .add("appId", app.getApplicationId())
      .add("name", app.getName())
      .add("user", app.getUser())
      .add("queue", app.getQueue())
      .add("state", app.getState())
      .add("trackingUrl", trackingUrl)
      .add("appMasterHost", host)
      .add("startTime", app.getStartTime())
      .add("finishTime", app.getFinishTime())
      .add("finalStatus", app.getFinalApplicationStatus())
      .add("memorySeconds", metrics.getMemorySeconds())
      .add("vcoreSeconds", metrics.getVcoreSeconds())
      .add("gcoreSeconds", metrics.getGcoreSeconds())
      .add("preemptedAMContainers", metrics.getNumAMContainersPreempted())
      .add("preemptedNonAMContainers", metrics.getNumNonAMContainersPreempted())
      .add("preemptedResources", metrics.getResourcePreempted())
      .add("applicationType", app.getApplicationType());
  return summary;
}
 

private void publishApplicationFinishedEvent(ApplicationFinishedEvent event) {
  TimelineEntity entity =
      createApplicationEntity(event.getApplicationId());
  TimelineEvent tEvent = new TimelineEvent();
  tEvent.setEventType(
      ApplicationMetricsConstants.FINISHED_EVENT_TYPE);
  tEvent.setTimestamp(event.getTimestamp());
  Map<String, Object> eventInfo = new HashMap<String, Object>();
  eventInfo.put(ApplicationMetricsConstants.DIAGNOSTICS_INFO_EVENT_INFO,
      event.getDiagnosticsInfo());
  eventInfo.put(ApplicationMetricsConstants.FINAL_STATUS_EVENT_INFO,
      event.getFinalApplicationStatus().toString());
  eventInfo.put(ApplicationMetricsConstants.STATE_EVENT_INFO,
      event.getYarnApplicationState().toString());
  if (event.getLatestApplicationAttemptId() != null) {
    eventInfo.put(ApplicationMetricsConstants.LATEST_APP_ATTEMPT_EVENT_INFO,
        event.getLatestApplicationAttemptId().toString());
  }
  RMAppMetrics appMetrics = event.getAppMetrics();
  entity.addOtherInfo(ApplicationMetricsConstants.APP_CPU_METRICS,
      appMetrics.getVcoreSeconds());
  entity.addOtherInfo(ApplicationMetricsConstants.APP_MEM_METRICS,
      appMetrics.getMemorySeconds());
  entity.addOtherInfo(ApplicationMetricsConstants.APP_GPU_METRICS,
      appMetrics.getGcoreSeconds());

  tEvent.setEventInfo(eventInfo);
  entity.addEvent(tEvent);
  putEntity(entity);
}
 

@Test (timeout = 120000)
public void testUsageWithOneAttemptAndOneContainer() throws Exception {
  MockRM rm = new MockRM(conf);
  rm.start();
  
  MockNM nm =
      new MockNM("127.0.0.1:1234", 15120, rm.getResourceTrackerService());
  nm.registerNode();

  RMApp app0 = rm.submitApp(200);

  RMAppMetrics rmAppMetrics = app0.getRMAppMetrics();
  Assert.assertTrue(
      "Before app submittion, memory seconds should have been 0 but was "
                        + rmAppMetrics.getMemorySeconds(),
      rmAppMetrics.getMemorySeconds() == 0);
  Assert.assertTrue(
      "Before app submission, vcore seconds should have been 0 but was "
                        + rmAppMetrics.getVcoreSeconds(),
      rmAppMetrics.getVcoreSeconds() == 0);

  RMAppAttempt attempt0 = app0.getCurrentAppAttempt();

  nm.nodeHeartbeat(true);
  MockAM am0 = rm.sendAMLaunched(attempt0.getAppAttemptId());
  am0.registerAppAttempt();

  RMContainer rmContainer =
      rm.getResourceScheduler()
         .getRMContainer(attempt0.getMasterContainer().getId());

  // Allow metrics to accumulate.
  int sleepInterval = 1000;
  int cumulativeSleepTime = 0;
  while (rmAppMetrics.getMemorySeconds() <= 0 && cumulativeSleepTime < 5000) {
    Thread.sleep(sleepInterval);
    cumulativeSleepTime += sleepInterval;
  }

  rmAppMetrics = app0.getRMAppMetrics();
  Assert.assertTrue(
      "While app is running, memory seconds should be >0 but is "
          + rmAppMetrics.getMemorySeconds(),
      rmAppMetrics.getMemorySeconds() > 0);
  Assert.assertTrue(
      "While app is running, vcore seconds should be >0 but is "
          + rmAppMetrics.getVcoreSeconds(),
      rmAppMetrics.getVcoreSeconds() > 0);

  MockRM.finishAMAndVerifyAppState(app0, rm, nm, am0);

  AggregateAppResourceUsage ru = calculateContainerResourceMetrics(rmContainer);
  rmAppMetrics = app0.getRMAppMetrics();

  Assert.assertEquals("Unexcpected MemorySeconds value",
      ru.getMemorySeconds(), rmAppMetrics.getMemorySeconds());
  Assert.assertEquals("Unexpected VcoreSeconds value",
      ru.getVcoreSeconds(), rmAppMetrics.getVcoreSeconds());

  rm.stop();
}
 

@SuppressWarnings({ "rawtypes", "unchecked" })
public AppInfo(ResourceManager rm, RMApp app, Boolean hasAccess,
    String schemePrefix) {
  this.schemePrefix = schemePrefix;
  if (app != null) {
    String trackingUrl = app.getTrackingUrl();
    this.state = app.createApplicationState();
    this.trackingUrlIsNotReady = trackingUrl == null || trackingUrl.isEmpty()
        || YarnApplicationState.NEW == this.state
        || YarnApplicationState.NEW_SAVING == this.state
        || YarnApplicationState.SUBMITTED == this.state
        || YarnApplicationState.ACCEPTED == this.state;
    this.trackingUI = this.trackingUrlIsNotReady ? "UNASSIGNED" : (app
        .getFinishTime() == 0 ? "ApplicationMaster" : "History");
    if (!trackingUrlIsNotReady) {
      this.trackingUrl =
          WebAppUtils.getURLWithScheme(schemePrefix,
              trackingUrl);
      this.trackingUrlPretty = this.trackingUrl;
    } else {
      this.trackingUrlPretty = "UNASSIGNED";
    }
    this.applicationId = app.getApplicationId();
    this.applicationType = app.getApplicationType();
    this.appIdNum = String.valueOf(app.getApplicationId().getId());
    this.id = app.getApplicationId().toString();
    this.user = app.getUser().toString();
    this.name = app.getName().toString();
    this.queue = app.getQueue().toString();
    this.progress = app.getProgress() * 100;
    this.diagnostics = app.getDiagnostics().toString();
    if (diagnostics == null || diagnostics.isEmpty()) {
      this.diagnostics = "";
    }
    if (app.getApplicationTags() != null && !app.getApplicationTags().isEmpty()) {
      this.applicationTags = Joiner.on(',').join(app.getApplicationTags());
    }
    this.finalStatus = app.getFinalApplicationStatus();
    this.clusterId = ResourceManager.getClusterTimeStamp();
    if (hasAccess) {
      this.startedTime = app.getStartTime();
      this.finishedTime = app.getFinishTime();
      this.elapsedTime = Times.elapsed(app.getStartTime(),
          app.getFinishTime());

      RMAppAttempt attempt = app.getCurrentAppAttempt();
      if (attempt != null) {
        Container masterContainer = attempt.getMasterContainer();
        if (masterContainer != null) {
          this.amContainerLogsExist = true;
          this.amContainerLogs = WebAppUtils.getRunningLogURL(
              schemePrefix + masterContainer.getNodeHttpAddress(),
              ConverterUtils.toString(masterContainer.getId()),
              app.getUser());
          this.amHostHttpAddress = masterContainer.getNodeHttpAddress();
        }
        
        ApplicationResourceUsageReport resourceReport = attempt
            .getApplicationResourceUsageReport();
        if (resourceReport != null) {
          Resource usedResources = resourceReport.getUsedResources();
          allocatedMB = usedResources.getMemory();
          allocatedVCores = usedResources.getVirtualCores();
          runningContainers = resourceReport.getNumUsedContainers();
        }
        resourceRequests =
            ((AbstractYarnScheduler) rm.getRMContext().getScheduler())
              .getPendingResourceRequestsForAttempt(attempt.getAppAttemptId());
      }
    }

    // copy preemption info fields
    RMAppMetrics appMetrics = app.getRMAppMetrics();
    numAMContainerPreempted =
        appMetrics.getNumAMContainersPreempted();
    preemptedResourceMB =
        appMetrics.getResourcePreempted().getMemory();
    numNonAMContainerPreempted =
        appMetrics.getNumNonAMContainersPreempted();
    preemptedResourceVCores =
        appMetrics.getResourcePreempted().getVirtualCores();
    memorySeconds = appMetrics.getMemorySeconds();
    vcoreSeconds = appMetrics.getVcoreSeconds();
  }
}
 

@Test
public void testAppBlockRenderWithNullCurrentAppAttempt() throws Exception {
  final ApplicationId APP_ID = ApplicationId.newInstance(1234L, 0);
  Injector injector;
  
  // init app
  RMApp app = mock(RMApp.class);
  when(app.getTrackingUrl()).thenReturn("http://host:123");
  when(app.getState()).thenReturn(RMAppState.FAILED);
  when(app.getApplicationId()).thenReturn(APP_ID);
  when(app.getApplicationType()).thenReturn("Type");
  when(app.getUser()).thenReturn("user");
  when(app.getName()).thenReturn("Name");
  when(app.getQueue()).thenReturn("queue");
  when(app.getDiagnostics()).thenReturn(new StringBuilder());
  when(app.getFinalApplicationStatus()).thenReturn(FinalApplicationStatus.FAILED);
  when(app.getFinalApplicationStatus()).thenReturn(FinalApplicationStatus.FAILED);
  when(app.getStartTime()).thenReturn(0L);
  when(app.getFinishTime()).thenReturn(0L);
  when(app.createApplicationState()).thenReturn(YarnApplicationState.FAILED);
  
  RMAppMetrics appMetrics = new RMAppMetrics(Resource.newInstance(0, 0), 0, 0, 0, 0);
  when(app.getRMAppMetrics()).thenReturn(appMetrics);
  
  // initialize RM Context, and create RMApp, without creating RMAppAttempt
  final RMContext rmContext = TestRMWebApp.mockRMContext(15, 1, 2, 8);
  rmContext.getRMApps().put(APP_ID, app);
  
  injector =
      WebAppTests.createMockInjector(RMContext.class, rmContext,
          new Module() {
            @Override
            public void configure(Binder binder) {
              try {
                ResourceManager rm = TestRMWebApp.mockRm(rmContext);
                binder.bind(ResourceManager.class).toInstance(rm);
                binder.bind(ApplicationBaseProtocol.class).toInstance(
                  rm.getClientRMService());
              } catch (IOException e) {
                throw new IllegalStateException(e);
              }
            }
          });
  
  AppBlock instance = injector.getInstance(AppBlock.class);
  instance.set(YarnWebParams.APPLICATION_ID, APP_ID.toString());
  instance.render();
}
 

@Override
public RMAppMetrics getRMAppMetrics() {
  return new RMAppMetrics(Resource.newInstance(0, 0), 0, 0, 0, 0);
}
 

@Override
public RMAppMetrics getRMAppMetrics() {
  return null;
}
 

public RMAppMetrics getAppMetrics() {
  return appMetrics;
}
 

@Override
protected void createApplicationMetricsTable(Block html){
  RMApp rmApp = this.rm.getRMContext().getRMApps().get(appID);
  RMAppMetrics appMetrics = rmApp == null ? null : rmApp.getRMAppMetrics();
  // Get attempt metrics and fields, it is possible currentAttempt of RMApp is
  // null. In that case, we will assume resource preempted and number of Non
  // AM container preempted on that attempt is 0
  RMAppAttemptMetrics attemptMetrics;
  if (rmApp == null || null == rmApp.getCurrentAppAttempt()) {
    attemptMetrics = null;
  } else {
    attemptMetrics = rmApp.getCurrentAppAttempt().getRMAppAttemptMetrics();
  }
  Resource attemptResourcePreempted =
      attemptMetrics == null ? Resources.none() : attemptMetrics
        .getResourcePreempted();
  int attemptNumNonAMContainerPreempted =
      attemptMetrics == null ? 0 : attemptMetrics
        .getNumNonAMContainersPreempted();
  DIV<Hamlet> pdiv = html.
      _(InfoBlock.class).
      div(_INFO_WRAP);
  info("Application Overview").clear();
  info("Application Metrics")
      ._("Total Resource Preempted:",
        appMetrics == null ? "N/A" : appMetrics.getResourcePreempted())
      ._("Total Number of Non-AM Containers Preempted:",
        appMetrics == null ? "N/A"
            : appMetrics.getNumNonAMContainersPreempted())
      ._("Total Number of AM Containers Preempted:",
        appMetrics == null ? "N/A"
            : appMetrics.getNumAMContainersPreempted())
      ._("Resource Preempted from Current Attempt:",
        attemptResourcePreempted)
      ._("Number of Non-AM Containers Preempted from Current Attempt:",
        attemptNumNonAMContainerPreempted)
      ._("Aggregate Resource Allocation:",
        String.format("%d MB-seconds, %d vcore-seconds",
            appMetrics == null ? "N/A" : appMetrics.getMemorySeconds(),
            appMetrics == null ? "N/A" : appMetrics.getVcoreSeconds()));
  pdiv._();
}
 

public void sendAppEvent(ApplicationId applicationId, RMApp rmApp) {
    if (cacheFinishedApp.getIfPresent(applicationId.toString()) == null) {
        Header.Builder headerBuilder = Header.newBuilder()
            .withId(applicationId.toString())
            .withApplicationID(applicationId.toString())
            .withUser(rmApp.getUser())
            .withApplicationName(rmApp.getName())
            .withFramework(rmApp.getApplicationType().toUpperCase());

        ApplicationEvent.Builder eventBuilder = ApplicationEvent.newBuilder()
            .setState(rmApp.getState().name())
            .setQueue(rmApp.getQueue());

        rmApp.getApplicationTags().stream()
            .filter(tag -> YARN_TAGS_TO_EXTRACT.stream().noneMatch(tag::startsWith) && !tag.contains(":"))
            .forEach(eventBuilder::addYarnTags);

        rmApp.getApplicationTags().stream()
            .filter(tag -> tag.contains(":") && YARN_TAGS_TO_EXTRACT.stream().anyMatch(tag::startsWith))
            .map(tag -> {
                int idx = tag.indexOf(':');
                String key = tag.substring(0, idx);
                String value = tag.substring(idx + 1);
                return new String[] {key, value};
            })
            .forEach(splitTag -> BUILDERS.get(splitTag[0]).accept(splitTag[1], eventBuilder));

        eventBuilder.setFinalStatus(rmApp.getFinalApplicationStatus().name());

        eventBuilder.setStartTime(rmApp.getStartTime());
        eventBuilder.setFinishTime(rmApp.getFinishTime());

        RMAppMetrics rmAppMetrics = rmApp.getRMAppMetrics();
        if (rmAppMetrics != null) {
            eventBuilder.setMemorySeconds(rmAppMetrics.getMemorySeconds());
            eventBuilder.setVcoreSeconds(rmAppMetrics.getVcoreSeconds());
        }

        RMAppAttempt rmAppAttempt = rmApp.getCurrentAppAttempt();
        if (rmAppAttempt != null) {
            headerBuilder.withAttemptID(rmAppAttempt.getAppAttemptId().toString());

            Container container = rmAppAttempt.getMasterContainer();
            if (container != null) {
                eventBuilder.setAmContainerId(container.getId().toString());
            }
        }

        if (rmApp.getTrackingUrl() != null) {
            eventBuilder.setTrackingUrl(normalizeTrackingUrl(rmApp.getTrackingUrl()));
        }

        if (rmApp.getOriginalTrackingUrl() != null && !"N/A".equals(rmApp.getOriginalTrackingUrl())) {
            eventBuilder.setOriginalTrackingUrl(normalizeTrackingUrl(rmApp.getOriginalTrackingUrl()));
        }

        eventHandler.accept(System.currentTimeMillis(), headerBuilder.build(), eventBuilder.build());

        if (rmApp.getState() == RMAppState.FINISHED || rmApp.getState() == RMAppState.KILLED || rmApp.getState() == RMAppState.FAILED) {
            cacheFinishedApp.put(applicationId.toString(), rmApp.getState().name());
        }
    }
}
 

@Override
public RMAppMetrics getRMAppMetrics() {
  return new RMAppMetrics(Resource.newInstance(0, 0, 0), 0, 0, 0, 0, 0);
}
 

@Test (timeout = 120000)
public void testUsageWithOneAttemptAndOneContainer() throws Exception {
  MockRM rm = new MockRM(conf);
  rm.start();
  
  MockNM nm =
      new MockNM("127.0.0.1:1234", 15120, rm.getResourceTrackerService());
  nm.registerNode();

  RMApp app0 = rm.submitApp(200);

  RMAppMetrics rmAppMetrics = app0.getRMAppMetrics();
  Assert.assertTrue(
      "Before app submittion, memory seconds should have been 0 but was "
                        + rmAppMetrics.getMemorySeconds(),
      rmAppMetrics.getMemorySeconds() == 0);
  Assert.assertTrue(
      "Before app submission, vcore seconds should have been 0 but was "
                        + rmAppMetrics.getVcoreSeconds(),
      rmAppMetrics.getVcoreSeconds() == 0);

  Assert.assertTrue(
      "Before app submission, gcore seconds should have been 0 but was "
                        + rmAppMetrics.getGcoreSeconds(),
      rmAppMetrics.getGcoreSeconds() == 0);

  RMAppAttempt attempt0 = app0.getCurrentAppAttempt();

  nm.nodeHeartbeat(true);
  MockAM am0 = rm.sendAMLaunched(attempt0.getAppAttemptId());
  am0.registerAppAttempt();

  RMContainer rmContainer =
      rm.getResourceScheduler()
         .getRMContainer(attempt0.getMasterContainer().getId());

  // Allow metrics to accumulate.
  int sleepInterval = 1000;
  int cumulativeSleepTime = 0;
  while (rmAppMetrics.getMemorySeconds() <= 0 && cumulativeSleepTime < 5000) {
    Thread.sleep(sleepInterval);
    cumulativeSleepTime += sleepInterval;
  }

  rmAppMetrics = app0.getRMAppMetrics();
  Assert.assertTrue(
      "While app is running, memory seconds should be >0 but is "
          + rmAppMetrics.getMemorySeconds(),
      rmAppMetrics.getMemorySeconds() > 0);
  Assert.assertTrue(
      "While app is running, vcore seconds should be >0 but is "
          + rmAppMetrics.getVcoreSeconds(),
      rmAppMetrics.getVcoreSeconds() > 0);

  MockRM.finishAMAndVerifyAppState(app0, rm, nm, am0);

  AggregateAppResourceUsage ru = calculateContainerResourceMetrics(rmContainer);
  rmAppMetrics = app0.getRMAppMetrics();

  Assert.assertEquals("Unexcpected MemorySeconds value",
      ru.getMemorySeconds(), rmAppMetrics.getMemorySeconds());
  Assert.assertEquals("Unexpected VcoreSeconds value",
      ru.getVcoreSeconds(), rmAppMetrics.getVcoreSeconds());
  Assert.assertEquals("Unexpected GcoreSeconds value",
      ru.getGcoreSeconds(), rmAppMetrics.getGcoreSeconds());
  rm.stop();
}
 

@Test
public void testAppBlockRenderWithNullCurrentAppAttempt() throws Exception {
  final ApplicationId APP_ID = ApplicationId.newInstance(1234L, 0);
  Injector injector;
  
  // init app
  RMApp app = mock(RMApp.class);
  when(app.getTrackingUrl()).thenReturn("http://host:123");
  when(app.getState()).thenReturn(RMAppState.FAILED);
  when(app.getApplicationId()).thenReturn(APP_ID);
  when(app.getApplicationType()).thenReturn("Type");
  when(app.getUser()).thenReturn("user");
  when(app.getName()).thenReturn("Name");
  when(app.getQueue()).thenReturn("queue");
  when(app.getDiagnostics()).thenReturn(new StringBuilder());
  when(app.getFinalApplicationStatus()).thenReturn(FinalApplicationStatus.FAILED);
  when(app.getFinalApplicationStatus()).thenReturn(FinalApplicationStatus.FAILED);
  when(app.getStartTime()).thenReturn(0L);
  when(app.getFinishTime()).thenReturn(0L);
  when(app.createApplicationState()).thenReturn(YarnApplicationState.FAILED);
  
  RMAppMetrics appMetrics = new RMAppMetrics(Resource.newInstance(0, 0, 0), 0, 0, 0, 0, 0);
  when(app.getRMAppMetrics()).thenReturn(appMetrics);
  
  // initialize RM Context, and create RMApp, without creating RMAppAttempt
  final RMContext rmContext = TestRMWebApp.mockRMContext(15, 1, 2, 8);
  rmContext.getRMApps().put(APP_ID, app);
  
  injector =
      WebAppTests.createMockInjector(RMContext.class, rmContext,
          new Module() {
            @Override
            public void configure(Binder binder) {
              try {
                ResourceManager rm = TestRMWebApp.mockRm(rmContext);
                binder.bind(ResourceManager.class).toInstance(rm);
                binder.bind(ApplicationBaseProtocol.class).toInstance(
                  rm.getClientRMService());
              } catch (IOException e) {
                throw new IllegalStateException(e);
              }
            }
          });
  
  AppBlock instance = injector.getInstance(AppBlock.class);
  instance.set(YarnWebParams.APPLICATION_ID, APP_ID.toString());
  instance.render();
}
 

public RMAppMetrics getAppMetrics() {
  return appMetrics;
}
 

@Override
protected void createApplicationMetricsTable(Block html){
  RMApp rmApp = this.rm.getRMContext().getRMApps().get(appID);
  RMAppMetrics appMetrics = rmApp == null ? null : rmApp.getRMAppMetrics();
  // Get attempt metrics and fields, it is possible currentAttempt of RMApp is
  // null. In that case, we will assume resource preempted and number of Non
  // AM container preempted on that attempt is 0
  RMAppAttemptMetrics attemptMetrics;
  if (rmApp == null || null == rmApp.getCurrentAppAttempt()) {
    attemptMetrics = null;
  } else {
    attemptMetrics = rmApp.getCurrentAppAttempt().getRMAppAttemptMetrics();
  }
  Resource attemptResourcePreempted =
      attemptMetrics == null ? Resources.none() : attemptMetrics
        .getResourcePreempted();
  int attemptNumNonAMContainerPreempted =
      attemptMetrics == null ? 0 : attemptMetrics
        .getNumNonAMContainersPreempted();
  DIV<Hamlet> pdiv = html.
      _(InfoBlock.class).
      div(_INFO_WRAP);
  info("Application Overview").clear();
  info("Application Metrics")
      ._("Total Resource Preempted:",
        appMetrics == null ? "N/A" : appMetrics.getResourcePreempted())
      ._("Total Number of Non-AM Containers Preempted:",
        appMetrics == null ? "N/A"
            : appMetrics.getNumNonAMContainersPreempted())
      ._("Total Number of AM Containers Preempted:",
        appMetrics == null ? "N/A"
            : appMetrics.getNumAMContainersPreempted())
      ._("Resource Preempted from Current Attempt:",
        attemptResourcePreempted)
      ._("Number of Non-AM Containers Preempted from Current Attempt:",
        attemptNumNonAMContainerPreempted)
      ._("Aggregate Resource Allocation:",
        String.format("%d MB-seconds, %d vcore-seconds, %d gcore-seconds",
            appMetrics == null ? "N/A" : appMetrics.getMemorySeconds(),
            appMetrics == null ? "N/A" : appMetrics.getVcoreSeconds(),
            appMetrics == null ? "N/A" : appMetrics.getGcoreSeconds()));
  pdiv._();
}
 

@SuppressWarnings({ "rawtypes", "unchecked" })
public AppInfo(ResourceManager rm, RMApp app, Boolean hasAccess,
    String schemePrefix) {
  this.schemePrefix = schemePrefix;
  if (app != null) {
    String trackingUrl = app.getTrackingUrl();
    this.state = app.createApplicationState();
    this.trackingUrlIsNotReady = trackingUrl == null || trackingUrl.isEmpty()
        || YarnApplicationState.NEW == this.state
        || YarnApplicationState.NEW_SAVING == this.state
        || YarnApplicationState.SUBMITTED == this.state
        || YarnApplicationState.ACCEPTED == this.state;
    this.trackingUI = this.trackingUrlIsNotReady ? "UNASSIGNED" : (app
        .getFinishTime() == 0 ? "ApplicationMaster" : "History");
    if (!trackingUrlIsNotReady) {
      this.trackingUrl =
          WebAppUtils.getURLWithScheme(schemePrefix,
              trackingUrl);
      this.trackingUrlPretty = this.trackingUrl;
    } else {
      this.trackingUrlPretty = "UNASSIGNED";
    }
    this.applicationId = app.getApplicationId();
    this.applicationType = app.getApplicationType();
    this.appIdNum = String.valueOf(app.getApplicationId().getId());
    this.id = app.getApplicationId().toString();
    this.user = app.getUser().toString();
    this.name = app.getName().toString();
    this.queue = app.getQueue().toString();
    this.progress = app.getProgress() * 100;
    this.diagnostics = app.getDiagnostics().toString();
    if (diagnostics == null || diagnostics.isEmpty()) {
      this.diagnostics = "";
    }
    if (app.getApplicationTags() != null && !app.getApplicationTags().isEmpty()) {
      this.applicationTags = Joiner.on(',').join(app.getApplicationTags());
    }
    this.finalStatus = app.getFinalApplicationStatus();
    this.clusterId = ResourceManager.getClusterTimeStamp();
    if (hasAccess) {
      this.startedTime = app.getStartTime();
      this.finishedTime = app.getFinishTime();
      this.elapsedTime = Times.elapsed(app.getStartTime(),
          app.getFinishTime());

      RMAppAttempt attempt = app.getCurrentAppAttempt();
      if (attempt != null) {
        Container masterContainer = attempt.getMasterContainer();
        if (masterContainer != null) {
          this.amContainerLogsExist = true;
          this.amContainerLogs = WebAppUtils.getRunningLogURL(
              schemePrefix + masterContainer.getNodeHttpAddress(),
              ConverterUtils.toString(masterContainer.getId()),
              app.getUser());
          this.amHostHttpAddress = masterContainer.getNodeHttpAddress();
        }
        
        ApplicationResourceUsageReport resourceReport = attempt
            .getApplicationResourceUsageReport();
        if (resourceReport != null) {
          Resource usedResources = resourceReport.getUsedResources();
          allocatedMB = usedResources.getMemory();
          allocatedVCores = usedResources.getVirtualCores();
          allocatedGCores = usedResources.getGpuCores();
          runningContainers = resourceReport.getNumUsedContainers();
        }
        resourceRequests =
            ((AbstractYarnScheduler) rm.getRMContext().getScheduler())
              .getPendingResourceRequestsForAttempt(attempt.getAppAttemptId());
      }
    }

    // copy preemption info fields
    RMAppMetrics appMetrics = app.getRMAppMetrics();
    numAMContainerPreempted =
        appMetrics.getNumAMContainersPreempted();
    preemptedResourceMB =
        appMetrics.getResourcePreempted().getMemory();
    numNonAMContainerPreempted =
        appMetrics.getNumNonAMContainersPreempted();
    preemptedResourceVCores =
        appMetrics.getResourcePreempted().getVirtualCores();
    preemptedResourceGCores =
        appMetrics.getResourcePreempted().getGpuCores();
    memorySeconds = appMetrics.getMemorySeconds();
    vcoreSeconds = appMetrics.getVcoreSeconds();
    gcoreSeconds = appMetrics.getGcoreSeconds();
  }
}