org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport Java Exaples

Source File: AppReportConverter.java From cloudbreak with Apache License 2.0

6 votes

@Override
public AppReportJson convert(ApplicationReport source) {
    AppReportJson json = new AppReportJson();
    json.setAppId(source.getApplicationId().toString());
    json.setStart(source.getStartTime());
    json.setFinish(source.getFinishTime());
    json.setProgress(source.getProgress());
    json.setQueue(source.getQueue());
    json.setUrl(source.getTrackingUrl());
    json.setUser(source.getUser());
    json.setState(source.getYarnApplicationState().name());
    ApplicationResourceUsageReport usageReport = source.getApplicationResourceUsageReport();
    json.setReservedContainers(usageReport.getNumReservedContainers());
    json.setUsedContainers(usageReport.getNumUsedContainers());
    json.setUsedMemory(usageReport.getUsedResources().getMemory());
    json.setUsedVCores(usageReport.getUsedResources().getVirtualCores());
    return json;
}

Source File: RMAppAttemptImpl.java From big-c with Apache License 2.0

6 votes

@Override
public ApplicationResourceUsageReport getApplicationResourceUsageReport() {
  this.readLock.lock();
  try {
    ApplicationResourceUsageReport report =
        scheduler.getAppResourceUsageReport(this.getAppAttemptId());
    if (report == null) {
      report = RMServerUtils.DUMMY_APPLICATION_RESOURCE_USAGE_REPORT;
    }
    AggregateAppResourceUsage resUsage =
        this.attemptMetrics.getAggregateAppResourceUsage();
    report.setMemorySeconds(resUsage.getMemorySeconds());
    report.setVcoreSeconds(resUsage.getVcoreSeconds());
    return report;
  } finally {
    this.readLock.unlock();
  }
}

Source File: TestApplicationACLs.java From hadoop with Apache License 2.0

6 votes

private void verifyEnemyAppReport(ApplicationReport appReport) {
  Assert.assertEquals("Enemy should not see app host!",
      UNAVAILABLE, appReport.getHost());
  Assert.assertEquals("Enemy should not see app rpc port!",
      -1, appReport.getRpcPort());
  Assert.assertEquals("Enemy should not see app client token!",
      null, appReport.getClientToAMToken());
  Assert.assertEquals("Enemy should not see app diagnostics!",
      UNAVAILABLE, appReport.getDiagnostics());
  Assert.assertEquals("Enemy should not see app tracking url!",
      UNAVAILABLE, appReport.getTrackingUrl());
  Assert.assertEquals("Enemy should not see app original tracking url!",
      UNAVAILABLE, appReport.getOriginalTrackingUrl());
  ApplicationResourceUsageReport usageReport =
      appReport.getApplicationResourceUsageReport();
  Assert.assertEquals("Enemy should not see app used containers",
      -1, usageReport.getNumUsedContainers());
  Assert.assertEquals("Enemy should not see app reserved containers",
      -1, usageReport.getNumReservedContainers());
  Assert.assertEquals("Enemy should not see app used resources",
      -1, usageReport.getUsedResources().getMemory());
  Assert.assertEquals("Enemy should not see app reserved resources",
      -1, usageReport.getReservedResources().getMemory());
  Assert.assertEquals("Enemy should not see app needed resources",
      -1, usageReport.getNeededResources().getMemory());
}

Source File: RMAppAttemptMetrics.java From big-c with Apache License 2.0

6 votes

public AggregateAppResourceUsage getAggregateAppResourceUsage() {
  long memorySeconds = finishedMemorySeconds.get();
  long vcoreSeconds = finishedVcoreSeconds.get();

  // Only add in the running containers if this is the active attempt.
  RMAppAttempt currentAttempt = rmContext.getRMApps()
                 .get(attemptId.getApplicationId()).getCurrentAppAttempt();
  if (currentAttempt.getAppAttemptId().equals(attemptId)) {
    ApplicationResourceUsageReport appResUsageReport = rmContext
          .getScheduler().getAppResourceUsageReport(attemptId);
    if (appResUsageReport != null) {
      memorySeconds += appResUsageReport.getMemorySeconds();
      vcoreSeconds += appResUsageReport.getVcoreSeconds();
    }
  }
  return new AggregateAppResourceUsage(memorySeconds, vcoreSeconds);
}

Source File: TestClientRMService.java From hadoop with Apache License 2.0

6 votes

@Test
public void testGetApplicationResourceUsageReportDummy() throws YarnException,
    IOException {
  ApplicationAttemptId attemptId = getApplicationAttemptId(1);
  YarnScheduler yarnScheduler = mockYarnScheduler();
  RMContext rmContext = mock(RMContext.class);
  mockRMContext(yarnScheduler, rmContext);
  when(rmContext.getDispatcher().getEventHandler()).thenReturn(
      new EventHandler<Event>() {
        public void handle(Event event) {
        }
      });
  ApplicationSubmissionContext asContext = 
      mock(ApplicationSubmissionContext.class);
  YarnConfiguration config = new YarnConfiguration();
  RMAppAttemptImpl rmAppAttemptImpl = new RMAppAttemptImpl(attemptId,
      rmContext, yarnScheduler, null, asContext, config, false, null);
  ApplicationResourceUsageReport report = rmAppAttemptImpl
      .getApplicationResourceUsageReport();
  assertEquals(report, RMServerUtils.DUMMY_APPLICATION_RESOURCE_USAGE_REPORT);
}

Source File: TestApplicationACLs.java From big-c with Apache License 2.0

6 votes

private void verifyEnemyAppReport(ApplicationReport appReport) {
  Assert.assertEquals("Enemy should not see app host!",
      UNAVAILABLE, appReport.getHost());
  Assert.assertEquals("Enemy should not see app rpc port!",
      -1, appReport.getRpcPort());
  Assert.assertEquals("Enemy should not see app client token!",
      null, appReport.getClientToAMToken());
  Assert.assertEquals("Enemy should not see app diagnostics!",
      UNAVAILABLE, appReport.getDiagnostics());
  Assert.assertEquals("Enemy should not see app tracking url!",
      UNAVAILABLE, appReport.getTrackingUrl());
  Assert.assertEquals("Enemy should not see app original tracking url!",
      UNAVAILABLE, appReport.getOriginalTrackingUrl());
  ApplicationResourceUsageReport usageReport =
      appReport.getApplicationResourceUsageReport();
  Assert.assertEquals("Enemy should not see app used containers",
      -1, usageReport.getNumUsedContainers());
  Assert.assertEquals("Enemy should not see app reserved containers",
      -1, usageReport.getNumReservedContainers());
  Assert.assertEquals("Enemy should not see app used resources",
      -1, usageReport.getUsedResources().getMemory());
  Assert.assertEquals("Enemy should not see app reserved resources",
      -1, usageReport.getReservedResources().getMemory());
  Assert.assertEquals("Enemy should not see app needed resources",
      -1, usageReport.getNeededResources().getMemory());
}

Source File: RMAppAttemptMetrics.java From hadoop with Apache License 2.0

6 votes

public AggregateAppResourceUsage getAggregateAppResourceUsage() {
  long memorySeconds = finishedMemorySeconds.get();
  long vcoreSeconds = finishedVcoreSeconds.get();
  long gcoreSeconds = finishedGcoreSeconds.get();

  // Only add in the running containers if this is the active attempt.
  RMAppAttempt currentAttempt = rmContext.getRMApps()
                 .get(attemptId.getApplicationId()).getCurrentAppAttempt();
  if (currentAttempt.getAppAttemptId().equals(attemptId)) {
    ApplicationResourceUsageReport appResUsageReport = rmContext
          .getScheduler().getAppResourceUsageReport(attemptId);
    if (appResUsageReport != null) {
      memorySeconds += appResUsageReport.getMemorySeconds();
      vcoreSeconds += appResUsageReport.getVcoreSeconds();
      gcoreSeconds += appResUsageReport.getGcoreSeconds();
    }
  }
  return new AggregateAppResourceUsage(memorySeconds, vcoreSeconds, gcoreSeconds);
}

Source File: RMAppAttemptImpl.java From hadoop with Apache License 2.0

6 votes

@Override
public ApplicationResourceUsageReport getApplicationResourceUsageReport() {
  this.readLock.lock();
  try {
    ApplicationResourceUsageReport report =
        scheduler.getAppResourceUsageReport(this.getAppAttemptId());
    if (report == null) {
      report = RMServerUtils.DUMMY_APPLICATION_RESOURCE_USAGE_REPORT;
    }
    AggregateAppResourceUsage resUsage =
        this.attemptMetrics.getAggregateAppResourceUsage();
    report.setMemorySeconds(resUsage.getMemorySeconds());
    report.setVcoreSeconds(resUsage.getVcoreSeconds());
    report.setGcoreSeconds(resUsage.getGcoreSeconds());
    return report;
  } finally {
    this.readLock.unlock();
  }
}

Source File: Utils.java From AthenaX with Apache License 2.0

6 votes

static InstanceInfo extractInstanceInfo(String clusterName, ApplicationReport report) {
  InstanceMetadata md = getMetadata(report.getApplicationTags());
  if (md == null) {
    return null;
  }

  ApplicationResourceUsageReport usage = report.getApplicationResourceUsageReport();
  InstanceStatus stat = new InstanceStatus()
      .allocatedVCores((long) usage.getUsedResources().getVirtualCores())
      .allocatedMB((long) usage.getUsedResources().getMemory())
      .clusterId(clusterName)
      .applicationId(report.getApplicationId().toString())
      .startedTime(report.getStartTime())
      .runningContainers((long) usage.getNumUsedContainers())
      .trackingUrl(report.getTrackingUrl())
      .state(InstanceStatus.StateEnum.fromValue(report.getYarnApplicationState().toString()));
  return new InstanceInfo(clusterName, report.getApplicationId(), md, stat);
}

Source File: BuilderUtils.java From hadoop with Apache License 2.0

6 votes

public static ApplicationResourceUsageReport newApplicationResourceUsageReport(
    int numUsedContainers, int numReservedContainers, Resource usedResources,
    Resource reservedResources, Resource neededResources, long memorySeconds, 
    long vcoreSeconds, long gcoreSeconds) {
  ApplicationResourceUsageReport report =
      recordFactory.newRecordInstance(ApplicationResourceUsageReport.class);
  report.setNumUsedContainers(numUsedContainers);
  report.setNumReservedContainers(numReservedContainers);
  report.setUsedResources(usedResources);
  report.setReservedResources(reservedResources);
  report.setNeededResources(neededResources);
  report.setMemorySeconds(memorySeconds);
  report.setVcoreSeconds(vcoreSeconds);
  report.setGcoreSeconds(gcoreSeconds);
  return report;
}

Source File: TestClientRMService.java From big-c with Apache License 2.0

6 votes

@Test
public void testGetApplicationResourceUsageReportDummy() throws YarnException,
    IOException {
  ApplicationAttemptId attemptId = getApplicationAttemptId(1);
  YarnScheduler yarnScheduler = mockYarnScheduler();
  RMContext rmContext = mock(RMContext.class);
  mockRMContext(yarnScheduler, rmContext);
  when(rmContext.getDispatcher().getEventHandler()).thenReturn(
      new EventHandler<Event>() {
        public void handle(Event event) {
        }
      });
  ApplicationSubmissionContext asContext = 
      mock(ApplicationSubmissionContext.class);
  YarnConfiguration config = new YarnConfiguration();
  RMAppAttemptImpl rmAppAttemptImpl = new RMAppAttemptImpl(attemptId,
      rmContext, yarnScheduler, null, asContext, config, false, null);
  ApplicationResourceUsageReport report = rmAppAttemptImpl
      .getApplicationResourceUsageReport();
  assertEquals(report, RMServerUtils.DUMMY_APPLICATION_RESOURCE_USAGE_REPORT);
}

Source File: TestResourceMgrDelegate.java From big-c with Apache License 2.0

6 votes

private ApplicationReport getApplicationReport(
    YarnApplicationState yarnApplicationState,
    FinalApplicationStatus finalApplicationStatus) {
  ApplicationReport appReport = Mockito.mock(ApplicationReport.class);
  ApplicationResourceUsageReport appResources = Mockito
      .mock(ApplicationResourceUsageReport.class);
  Mockito.when(appReport.getApplicationId()).thenReturn(
      ApplicationId.newInstance(0, 0));
  Mockito.when(appResources.getNeededResources()).thenReturn(
      Records.newRecord(Resource.class));
  Mockito.when(appResources.getReservedResources()).thenReturn(
      Records.newRecord(Resource.class));
  Mockito.when(appResources.getUsedResources()).thenReturn(
      Records.newRecord(Resource.class));
  Mockito.when(appReport.getApplicationResourceUsageReport()).thenReturn(
      appResources);
  Mockito.when(appReport.getYarnApplicationState()).thenReturn(
      yarnApplicationState);
  Mockito.when(appReport.getFinalApplicationStatus()).thenReturn(
      finalApplicationStatus);

  return appReport;
}

Source File: TestResourceMgrDelegate.java From hadoop with Apache License 2.0

6 votes

private ApplicationReport getApplicationReport(
    YarnApplicationState yarnApplicationState,
    FinalApplicationStatus finalApplicationStatus) {
  ApplicationReport appReport = Mockito.mock(ApplicationReport.class);
  ApplicationResourceUsageReport appResources = Mockito
      .mock(ApplicationResourceUsageReport.class);
  Mockito.when(appReport.getApplicationId()).thenReturn(
      ApplicationId.newInstance(0, 0));
  Mockito.when(appResources.getNeededResources()).thenReturn(
      Records.newRecord(Resource.class));
  Mockito.when(appResources.getReservedResources()).thenReturn(
      Records.newRecord(Resource.class));
  Mockito.when(appResources.getUsedResources()).thenReturn(
      Records.newRecord(Resource.class));
  Mockito.when(appReport.getApplicationResourceUsageReport()).thenReturn(
      appResources);
  Mockito.when(appReport.getYarnApplicationState()).thenReturn(
      yarnApplicationState);
  Mockito.when(appReport.getFinalApplicationStatus()).thenReturn(
      finalApplicationStatus);

  return appReport;
}

Source File: ClusterAnalysisService.java From jumbune with GNU Lesser General Public License v3.0

6 votes

/**
 * It fetches used vcore and used containers for running applications
 *  from hadoop (resource manager) and persists into influxdb
 * @param cluster
 * @throws Exception
 */
private void persistRunningAppsDataToInfluxdb(Cluster cluster) throws Exception {
	
	List<ApplicationReport> list = sessionUtils.getRM(cluster, getSession()).getRunningApplications();
	
	InfluxDataWriter writer = new InfluxDataWriter(
			AdminConfigurationUtil.getInfluxdbConfiguration(cluster.getClusterName()));
	writer.setTableName(WebConstants.JOB_HISTORY_DETAILS_TABLE);
	writer.setTimeUnit(TimeUnit.SECONDS);
	
	ApplicationResourceUsageReport usage;
	
	for (ApplicationReport report: list) {
		usage = report.getApplicationResourceUsageReport();
		
		writer.addColumn(WebConstants.USED_V_CORES, usage.getUsedResources().getVirtualCores());
		
		writer.addColumn(WebConstants.USED_CONTAINERS, usage.getNumUsedContainers());
		
		writer.addTag(WebConstants.JOB_ID, report.getApplicationId()
				.toString().replace(WebConstants.APPLICATION, WebConstants.JOB));
		writer.writeData();
	}
}

Source File: TestTypeConverter.java From hadoop with Apache License 2.0

5 votes

@Test
public void testFromYarn() throws Exception {
  int appStartTime = 612354;
  int appFinishTime = 612355;
  YarnApplicationState state = YarnApplicationState.RUNNING;
  ApplicationId applicationId = ApplicationId.newInstance(0, 0);
  ApplicationReport applicationReport = Records
      .newRecord(ApplicationReport.class);
  applicationReport.setApplicationId(applicationId);
  applicationReport.setYarnApplicationState(state);
  applicationReport.setStartTime(appStartTime);
  applicationReport.setFinishTime(appFinishTime);
  applicationReport.setUser("TestTypeConverter-user");
  ApplicationResourceUsageReport appUsageRpt = Records
      .newRecord(ApplicationResourceUsageReport.class);
  Resource r = Records.newRecord(Resource.class);
  r.setMemory(2048);
  appUsageRpt.setNeededResources(r);
  appUsageRpt.setNumReservedContainers(1);
  appUsageRpt.setNumUsedContainers(3);
  appUsageRpt.setReservedResources(r);
  appUsageRpt.setUsedResources(r);
  applicationReport.setApplicationResourceUsageReport(appUsageRpt);
  JobStatus jobStatus = TypeConverter.fromYarn(applicationReport, "dummy-jobfile");
  Assert.assertEquals(appStartTime, jobStatus.getStartTime());
  Assert.assertEquals(appFinishTime, jobStatus.getFinishTime());    
  Assert.assertEquals(state.toString(), jobStatus.getState().toString());
}

Source File: TypeConverter.java From hadoop with Apache License 2.0

5 votes

public static JobStatus fromYarn(ApplicationReport application,
    String jobFile) {
  String trackingUrl = application.getTrackingUrl();
  trackingUrl = trackingUrl == null ? "" : trackingUrl;
  JobStatus jobStatus =
    new JobStatus(
        TypeConverter.fromYarn(application.getApplicationId()),
        0.0f, 0.0f, 0.0f, 0.0f,
        TypeConverter.fromYarn(application.getYarnApplicationState(), application.getFinalApplicationStatus()),
        org.apache.hadoop.mapreduce.JobPriority.NORMAL,
        application.getUser(), application.getName(),
        application.getQueue(), jobFile, trackingUrl, false
    );
  jobStatus.setSchedulingInfo(trackingUrl); // Set AM tracking url
  jobStatus.setStartTime(application.getStartTime());
  jobStatus.setFinishTime(application.getFinishTime());
  jobStatus.setFailureInfo(application.getDiagnostics());
  ApplicationResourceUsageReport resourceUsageReport =
      application.getApplicationResourceUsageReport();
  if (resourceUsageReport != null) {
    jobStatus.setNeededMem(
        resourceUsageReport.getNeededResources().getMemory());
    jobStatus.setNumReservedSlots(
        resourceUsageReport.getNumReservedContainers());
    jobStatus.setNumUsedSlots(resourceUsageReport.getNumUsedContainers());
    jobStatus.setReservedMem(
        resourceUsageReport.getReservedResources().getMemory());
    jobStatus.setUsedMem(resourceUsageReport.getUsedResources().getMemory());
  }
  return jobStatus;
}

Source File: ApplicationReportPBImpl.java From big-c with Apache License 2.0

5 votes

public void setApplicationResourceUsageReport(ApplicationResourceUsageReport appInfo) {
  maybeInitBuilder();
  if (appInfo == null) {
    builder.clearAppResourceUsage();
    return;
  }
  builder.setAppResourceUsage(convertToProtoFormat(appInfo));
}

Source File: ApplicationReportPBImpl.java From big-c with Apache License 2.0

5 votes

@Override
public ApplicationResourceUsageReport getApplicationResourceUsageReport() {
  ApplicationReportProtoOrBuilder p = viaProto ? proto : builder;
  if (!p.hasAppResourceUsage()) {
    return null;
  }
  return convertFromProtoFormat(p.getAppResourceUsage());
}

Source File: TwillTester.java From twill with Apache License 2.0

5 votes

public ApplicationResourceUsageReport getApplicationResourceReport(String appId) throws Exception {
  List<String> splits = Lists.newArrayList(Splitter.on('_').split(appId));
  Preconditions.checkArgument(splits.size() == 3, "Invalid application id - " + appId);
  ApplicationId applicationId = ApplicationId.newInstance(Long.parseLong(splits.get(1)),
                                                          Integer.parseInt(splits.get(2)));

  ClientRMService clientRMService = cluster.getResourceManager().getClientRMService();
  GetApplicationReportRequest request = Records.newRecord(GetApplicationReportRequest.class);
  request.setApplicationId(applicationId);
  return clientRMService.getApplicationReport(request)
    .getApplicationReport().getApplicationResourceUsageReport();
}

Source File: SchedulerApplicationAttempt.java From hadoop with Apache License 2.0

5 votes

public synchronized ApplicationResourceUsageReport getResourceUsageReport() {
  AggregateAppResourceUsage runningResourceUsage =
      getRunningAggregateAppResourceUsage();
  Resource usedResourceClone =
      Resources.clone(attemptResourceUsage.getUsed());
  Resource reservedResourceClone =
      Resources.clone(attemptResourceUsage.getReserved());
  return ApplicationResourceUsageReport.newInstance(liveContainers.size(),
      reservedContainers.size(), usedResourceClone, reservedResourceClone,
      Resources.add(usedResourceClone, reservedResourceClone),
      runningResourceUsage.getMemorySeconds(),
      runningResourceUsage.getVcoreSeconds(),
      runningResourceUsage.getGcoreSeconds());
}

Source File: BuilderUtils.java From big-c with Apache License 2.0

5 votes

public static ApplicationReport newApplicationReport(
    ApplicationId applicationId, ApplicationAttemptId applicationAttemptId,
    String user, String queue, String name, String host, int rpcPort,
    Token clientToAMToken, YarnApplicationState state, String diagnostics,
    String url, long startTime, long finishTime,
    FinalApplicationStatus finalStatus,
    ApplicationResourceUsageReport appResources, String origTrackingUrl,
    float progress, String appType, Token amRmToken, Set<String> tags) {
  ApplicationReport report = recordFactory
      .newRecordInstance(ApplicationReport.class);
  report.setApplicationId(applicationId);
  report.setCurrentApplicationAttemptId(applicationAttemptId);
  report.setUser(user);
  report.setQueue(queue);
  report.setName(name);
  report.setHost(host);
  report.setRpcPort(rpcPort);
  report.setClientToAMToken(clientToAMToken);
  report.setYarnApplicationState(state);
  report.setDiagnostics(diagnostics);
  report.setTrackingUrl(url);
  report.setStartTime(startTime);
  report.setFinishTime(finishTime);
  report.setFinalApplicationStatus(finalStatus);
  report.setApplicationResourceUsageReport(appResources);
  report.setOriginalTrackingUrl(origTrackingUrl);
  report.setProgress(progress);
  report.setApplicationType(appType);
  report.setAMRMToken(amRmToken);
  report.setApplicationTags(tags);
  return report;
}

Source File: BuilderUtils.java From big-c with Apache License 2.0

5 votes

public static ApplicationResourceUsageReport newApplicationResourceUsageReport(
    int numUsedContainers, int numReservedContainers, Resource usedResources,
    Resource reservedResources, Resource neededResources, long memorySeconds, 
    long vcoreSeconds) {
  ApplicationResourceUsageReport report =
      recordFactory.newRecordInstance(ApplicationResourceUsageReport.class);
  report.setNumUsedContainers(numUsedContainers);
  report.setNumReservedContainers(numReservedContainers);
  report.setUsedResources(usedResources);
  report.setReservedResources(reservedResources);
  report.setNeededResources(neededResources);
  report.setMemorySeconds(memorySeconds);
  report.setVcoreSeconds(vcoreSeconds);
  return report;
}

Source File: SchedulerApplicationAttempt.java From big-c with Apache License 2.0

5 votes

public synchronized ApplicationResourceUsageReport getResourceUsageReport() {
  AggregateAppResourceUsage resUsage = getRunningAggregateAppResourceUsage();
  return ApplicationResourceUsageReport.newInstance(liveContainers.size(),
             reservedContainers.size(), Resources.clone(currentConsumption),
             Resources.clone(currentReservation),
             Resources.add(currentConsumption, currentReservation),
             resUsage.getMemorySeconds(), resUsage.getVcoreSeconds());
}

Source File: AbstractYarnScheduler.java From big-c with Apache License 2.0

5 votes

@Override
public ApplicationResourceUsageReport getAppResourceUsageReport(
    ApplicationAttemptId appAttemptId) {
  SchedulerApplicationAttempt attempt = getApplicationAttempt(appAttemptId);
  if (attempt == null) {
    if (LOG.isDebugEnabled()) {
      LOG.debug("Request for appInfo of unknown attempt " + appAttemptId);
    }
    return null;
  }
  return attempt.getResourceUsageReport();
}

Source File: TestRMWebApp.java From big-c with Apache License 2.0

5 votes

public static ClientRMService mockClientRMService(RMContext rmContext) {
  ClientRMService clientRMService = mock(ClientRMService.class);
  List<ApplicationReport> appReports = new ArrayList<ApplicationReport>();
  for (RMApp app : rmContext.getRMApps().values()) {
    ApplicationReport appReport =
        ApplicationReport.newInstance(
            app.getApplicationId(), (ApplicationAttemptId) null,
            app.getUser(), app.getQueue(),
            app.getName(), (String) null, 0, (Token) null,
            app.createApplicationState(),
            app.getDiagnostics().toString(), (String) null,
            app.getStartTime(), app.getFinishTime(),
            app.getFinalApplicationStatus(),
            (ApplicationResourceUsageReport) null, app.getTrackingUrl(),
            app.getProgress(), app.getApplicationType(), (Token) null);
    appReports.add(appReport);
  }
  GetApplicationsResponse response = mock(GetApplicationsResponse.class);
  when(response.getApplicationList()).thenReturn(appReports);
  try {
    when(clientRMService.getApplications(any(GetApplicationsRequest.class)))
        .thenReturn(response);
  } catch (YarnException e) {
    Assert.fail("Exception is not expteced.");
  }
  return clientRMService;
}

Source File: TestRMAppAttemptTransitions.java From big-c with Apache License 2.0

5 votes

@Test
public void testUsageReport() {
  // scheduler has info on running apps
  ApplicationAttemptId attemptId = applicationAttempt.getAppAttemptId();
  ApplicationResourceUsageReport appResUsgRpt =
          mock(ApplicationResourceUsageReport.class);
  when(appResUsgRpt.getMemorySeconds()).thenReturn(123456L);
  when(appResUsgRpt.getVcoreSeconds()).thenReturn(55544L);
  when(scheduler.getAppResourceUsageReport(any(ApplicationAttemptId.class)))
  .thenReturn(appResUsgRpt);

  // start and finish the attempt
  Container amContainer = allocateApplicationAttempt();
  launchApplicationAttempt(amContainer);
  runApplicationAttempt(amContainer, "host", 8042, "oldtrackingurl", false);
  applicationAttempt.handle(new RMAppAttemptUnregistrationEvent(attemptId,
      "", FinalApplicationStatus.SUCCEEDED, ""));

  // expect usage stats to come from the scheduler report
  ApplicationResourceUsageReport report = 
      applicationAttempt.getApplicationResourceUsageReport();
  Assert.assertEquals(123456L, report.getMemorySeconds());
  Assert.assertEquals(55544L, report.getVcoreSeconds());

  // finish app attempt and remove it from scheduler 
  when(appResUsgRpt.getMemorySeconds()).thenReturn(223456L);
  when(appResUsgRpt.getVcoreSeconds()).thenReturn(75544L);
  sendAttemptUpdateSavedEvent(applicationAttempt);
  NodeId anyNodeId = NodeId.newInstance("host", 1234);
  applicationAttempt.handle(new RMAppAttemptContainerFinishedEvent(
      attemptId, 
      ContainerStatus.newInstance(
          amContainer.getId(), ContainerState.COMPLETE, "", 0), anyNodeId));

  when(scheduler.getSchedulerAppInfo(eq(attemptId))).thenReturn(null);

  report = applicationAttempt.getApplicationResourceUsageReport();
  Assert.assertEquals(223456, report.getMemorySeconds());
  Assert.assertEquals(75544, report.getVcoreSeconds());
}

Source File: TestClientRMService.java From big-c with Apache License 2.0

5 votes

@Test
public void testGetApplicationReport() throws Exception {
  YarnScheduler yarnScheduler = mock(YarnScheduler.class);
  RMContext rmContext = mock(RMContext.class);
  mockRMContext(yarnScheduler, rmContext);

  ApplicationId appId1 = getApplicationId(1);

  ApplicationACLsManager mockAclsManager = mock(ApplicationACLsManager.class);
  when(
      mockAclsManager.checkAccess(UserGroupInformation.getCurrentUser(),
          ApplicationAccessType.VIEW_APP, null, appId1)).thenReturn(true);

  ClientRMService rmService = new ClientRMService(rmContext, yarnScheduler,
      null, mockAclsManager, null, null);
  try {
    RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null);
    GetApplicationReportRequest request = recordFactory
        .newRecordInstance(GetApplicationReportRequest.class);
    request.setApplicationId(appId1);
    GetApplicationReportResponse response = 
        rmService.getApplicationReport(request);
    ApplicationReport report = response.getApplicationReport();
    ApplicationResourceUsageReport usageReport = 
        report.getApplicationResourceUsageReport();
    Assert.assertEquals(10, usageReport.getMemorySeconds());
    Assert.assertEquals(3, usageReport.getVcoreSeconds());
  } finally {
    rmService.close();
  }
}

Source File: TypeConverter.java From big-c with Apache License 2.0

5 votes

public static JobStatus fromYarn(ApplicationReport application,
    String jobFile) {
  String trackingUrl = application.getTrackingUrl();
  trackingUrl = trackingUrl == null ? "" : trackingUrl;
  JobStatus jobStatus =
    new JobStatus(
        TypeConverter.fromYarn(application.getApplicationId()),
        0.0f, 0.0f, 0.0f, 0.0f,
        TypeConverter.fromYarn(application.getYarnApplicationState(), application.getFinalApplicationStatus()),
        org.apache.hadoop.mapreduce.JobPriority.NORMAL,
        application.getUser(), application.getName(),
        application.getQueue(), jobFile, trackingUrl, false
    );
  jobStatus.setSchedulingInfo(trackingUrl); // Set AM tracking url
  jobStatus.setStartTime(application.getStartTime());
  jobStatus.setFinishTime(application.getFinishTime());
  jobStatus.setFailureInfo(application.getDiagnostics());
  ApplicationResourceUsageReport resourceUsageReport =
      application.getApplicationResourceUsageReport();
  if (resourceUsageReport != null) {
    jobStatus.setNeededMem(
        resourceUsageReport.getNeededResources().getMemory());
    jobStatus.setNumReservedSlots(
        resourceUsageReport.getNumReservedContainers());
    jobStatus.setNumUsedSlots(resourceUsageReport.getNumUsedContainers());
    jobStatus.setReservedMem(
        resourceUsageReport.getReservedResources().getMemory());
    jobStatus.setUsedMem(resourceUsageReport.getUsedResources().getMemory());
  }
  return jobStatus;
}

Source File: TestTypeConverter.java From big-c with Apache License 2.0

5 votes

@Test
public void testFromYarn() throws Exception {
  int appStartTime = 612354;
  int appFinishTime = 612355;
  YarnApplicationState state = YarnApplicationState.RUNNING;
  ApplicationId applicationId = ApplicationId.newInstance(0, 0);
  ApplicationReport applicationReport = Records
      .newRecord(ApplicationReport.class);
  applicationReport.setApplicationId(applicationId);
  applicationReport.setYarnApplicationState(state);
  applicationReport.setStartTime(appStartTime);
  applicationReport.setFinishTime(appFinishTime);
  applicationReport.setUser("TestTypeConverter-user");
  ApplicationResourceUsageReport appUsageRpt = Records
      .newRecord(ApplicationResourceUsageReport.class);
  Resource r = Records.newRecord(Resource.class);
  r.setMemory(2048);
  appUsageRpt.setNeededResources(r);
  appUsageRpt.setNumReservedContainers(1);
  appUsageRpt.setNumUsedContainers(3);
  appUsageRpt.setReservedResources(r);
  appUsageRpt.setUsedResources(r);
  applicationReport.setApplicationResourceUsageReport(appUsageRpt);
  JobStatus jobStatus = TypeConverter.fromYarn(applicationReport, "dummy-jobfile");
  Assert.assertEquals(appStartTime, jobStatus.getStartTime());
  Assert.assertEquals(appFinishTime, jobStatus.getFinishTime());    
  Assert.assertEquals(state.toString(), jobStatus.getState().toString());
}

Source File: GobblinYarnAppLauncher.java From incubator-gobblin with Apache License 2.0

5 votes

private void sendEmailOnShutdown(Optional<ApplicationReport> applicationReport) {
  String subject = String.format("Gobblin Yarn application %s completed", this.applicationName);

  StringBuilder messageBuilder = new StringBuilder("Gobblin Yarn ApplicationReport:");
  if (applicationReport.isPresent()) {
    messageBuilder.append("\n");
    messageBuilder.append("\tApplication ID: ").append(applicationReport.get().getApplicationId()).append("\n");
    messageBuilder.append("\tApplication attempt ID: ")
        .append(applicationReport.get().getCurrentApplicationAttemptId()).append("\n");
    messageBuilder.append("\tFinal application status: ").append(applicationReport.get().getFinalApplicationStatus())
        .append("\n");
    messageBuilder.append("\tStart time: ").append(applicationReport.get().getStartTime()).append("\n");
    messageBuilder.append("\tFinish time: ").append(applicationReport.get().getFinishTime()).append("\n");

    if (!Strings.isNullOrEmpty(applicationReport.get().getDiagnostics())) {
      messageBuilder.append("\tDiagnostics: ").append(applicationReport.get().getDiagnostics()).append("\n");
    }

    ApplicationResourceUsageReport resourceUsageReport = applicationReport.get().getApplicationResourceUsageReport();
    if (resourceUsageReport != null) {
      messageBuilder.append("\tUsed containers: ").append(resourceUsageReport.getNumUsedContainers()).append("\n");
      Resource usedResource = resourceUsageReport.getUsedResources();
      if (usedResource != null) {
        messageBuilder.append("\tUsed memory (MBs): ").append(usedResource.getMemory()).append("\n");
        messageBuilder.append("\tUsed vcores: ").append(usedResource.getVirtualCores()).append("\n");
      }
    }
  } else {
    messageBuilder.append(' ').append("Not available");
  }

  try {
    EmailUtils.sendEmail(ConfigUtils.configToState(this.config), subject, messageBuilder.toString());
  } catch (EmailException ee) {
    LOGGER.error("Failed to send email notification on shutdown", ee);
  }
}

org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport Java Examples