Java Code Examples for org.apache.hadoop.yarn.api.records.ApplicationReport#getApplicationResourceUsageReport()
The following examples show how to use
org.apache.hadoop.yarn.api.records.ApplicationReport#getApplicationResourceUsageReport() .
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
| Source File: Utils.java From AthenaX with Apache License 2.0 | 6 votes |
|
static InstanceInfo extractInstanceInfo(String clusterName, ApplicationReport report) {
InstanceMetadata md = getMetadata(report.getApplicationTags());
if (md == null) {
return null;
}
ApplicationResourceUsageReport usage = report.getApplicationResourceUsageReport();
InstanceStatus stat = new InstanceStatus()
.allocatedVCores((long) usage.getUsedResources().getVirtualCores())
.allocatedMB((long) usage.getUsedResources().getMemory())
.clusterId(clusterName)
.applicationId(report.getApplicationId().toString())
.startedTime(report.getStartTime())
.runningContainers((long) usage.getNumUsedContainers())
.trackingUrl(report.getTrackingUrl())
.state(InstanceStatus.StateEnum.fromValue(report.getYarnApplicationState().toString()));
return new InstanceInfo(clusterName, report.getApplicationId(), md, stat);
}
Example 2
| Source File: TestApplicationACLs.java From hadoop with Apache License 2.0 | 6 votes |
|
private void verifyEnemyAppReport(ApplicationReport appReport) {
Assert.assertEquals("Enemy should not see app host!",
UNAVAILABLE, appReport.getHost());
Assert.assertEquals("Enemy should not see app rpc port!",
-1, appReport.getRpcPort());
Assert.assertEquals("Enemy should not see app client token!",
null, appReport.getClientToAMToken());
Assert.assertEquals("Enemy should not see app diagnostics!",
UNAVAILABLE, appReport.getDiagnostics());
Assert.assertEquals("Enemy should not see app tracking url!",
UNAVAILABLE, appReport.getTrackingUrl());
Assert.assertEquals("Enemy should not see app original tracking url!",
UNAVAILABLE, appReport.getOriginalTrackingUrl());
ApplicationResourceUsageReport usageReport =
appReport.getApplicationResourceUsageReport();
Assert.assertEquals("Enemy should not see app used containers",
-1, usageReport.getNumUsedContainers());
Assert.assertEquals("Enemy should not see app reserved containers",
-1, usageReport.getNumReservedContainers());
Assert.assertEquals("Enemy should not see app used resources",
-1, usageReport.getUsedResources().getMemory());
Assert.assertEquals("Enemy should not see app reserved resources",
-1, usageReport.getReservedResources().getMemory());
Assert.assertEquals("Enemy should not see app needed resources",
-1, usageReport.getNeededResources().getMemory());
}
Example 3
| Source File: TestApplicationACLs.java From big-c with Apache License 2.0 | 6 votes |
|
private void verifyEnemyAppReport(ApplicationReport appReport) {
Assert.assertEquals("Enemy should not see app host!",
UNAVAILABLE, appReport.getHost());
Assert.assertEquals("Enemy should not see app rpc port!",
-1, appReport.getRpcPort());
Assert.assertEquals("Enemy should not see app client token!",
null, appReport.getClientToAMToken());
Assert.assertEquals("Enemy should not see app diagnostics!",
UNAVAILABLE, appReport.getDiagnostics());
Assert.assertEquals("Enemy should not see app tracking url!",
UNAVAILABLE, appReport.getTrackingUrl());
Assert.assertEquals("Enemy should not see app original tracking url!",
UNAVAILABLE, appReport.getOriginalTrackingUrl());
ApplicationResourceUsageReport usageReport =
appReport.getApplicationResourceUsageReport();
Assert.assertEquals("Enemy should not see app used containers",
-1, usageReport.getNumUsedContainers());
Assert.assertEquals("Enemy should not see app reserved containers",
-1, usageReport.getNumReservedContainers());
Assert.assertEquals("Enemy should not see app used resources",
-1, usageReport.getUsedResources().getMemory());
Assert.assertEquals("Enemy should not see app reserved resources",
-1, usageReport.getReservedResources().getMemory());
Assert.assertEquals("Enemy should not see app needed resources",
-1, usageReport.getNeededResources().getMemory());
}
Example 4
| Source File: ClusterAnalysisService.java From jumbune with GNU Lesser General Public License v3.0 | 6 votes |
|
/**
* It fetches used vcore and used containers for running applications
* from hadoop (resource manager) and persists into influxdb
* @param cluster
* @throws Exception
*/
private void persistRunningAppsDataToInfluxdb(Cluster cluster) throws Exception {
List<ApplicationReport> list = sessionUtils.getRM(cluster, getSession()).getRunningApplications();
InfluxDataWriter writer = new InfluxDataWriter(
AdminConfigurationUtil.getInfluxdbConfiguration(cluster.getClusterName()));
writer.setTableName(WebConstants.JOB_HISTORY_DETAILS_TABLE);
writer.setTimeUnit(TimeUnit.SECONDS);
ApplicationResourceUsageReport usage;
for (ApplicationReport report: list) {
usage = report.getApplicationResourceUsageReport();
writer.addColumn(WebConstants.USED_V_CORES, usage.getUsedResources().getVirtualCores());
writer.addColumn(WebConstants.USED_CONTAINERS, usage.getNumUsedContainers());
writer.addTag(WebConstants.JOB_ID, report.getApplicationId()
.toString().replace(WebConstants.APPLICATION, WebConstants.JOB));
writer.writeData();
}
}
Example 5
| Source File: AppReportConverter.java From cloudbreak with Apache License 2.0 | 6 votes |
|
@Override
public AppReportJson convert(ApplicationReport source) {
AppReportJson json = new AppReportJson();
json.setAppId(source.getApplicationId().toString());
json.setStart(source.getStartTime());
json.setFinish(source.getFinishTime());
json.setProgress(source.getProgress());
json.setQueue(source.getQueue());
json.setUrl(source.getTrackingUrl());
json.setUser(source.getUser());
json.setState(source.getYarnApplicationState().name());
ApplicationResourceUsageReport usageReport = source.getApplicationResourceUsageReport();
json.setReservedContainers(usageReport.getNumReservedContainers());
json.setUsedContainers(usageReport.getNumUsedContainers());
json.setUsedMemory(usageReport.getUsedResources().getMemory());
json.setUsedVCores(usageReport.getUsedResources().getVirtualCores());
return json;
}
Example 6
| Source File: TestClientRMService.java From hadoop with Apache License 2.0 | 5 votes |
|
@Test
public void testGetApplicationReport() throws Exception {
YarnScheduler yarnScheduler = mock(YarnScheduler.class);
RMContext rmContext = mock(RMContext.class);
mockRMContext(yarnScheduler, rmContext);
ApplicationId appId1 = getApplicationId(1);
ApplicationACLsManager mockAclsManager = mock(ApplicationACLsManager.class);
when(
mockAclsManager.checkAccess(UserGroupInformation.getCurrentUser(),
ApplicationAccessType.VIEW_APP, null, appId1)).thenReturn(true);
ClientRMService rmService = new ClientRMService(rmContext, yarnScheduler,
null, mockAclsManager, null, null);
try {
RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null);
GetApplicationReportRequest request = recordFactory
.newRecordInstance(GetApplicationReportRequest.class);
request.setApplicationId(appId1);
GetApplicationReportResponse response =
rmService.getApplicationReport(request);
ApplicationReport report = response.getApplicationReport();
ApplicationResourceUsageReport usageReport =
report.getApplicationResourceUsageReport();
Assert.assertEquals(10, usageReport.getMemorySeconds());
Assert.assertEquals(3, usageReport.getVcoreSeconds());
Assert.assertEquals(3, usageReport.getGcoreSeconds());
} finally {
rmService.close();
}
}
Example 7
| Source File: TypeConverter.java From hadoop with Apache License 2.0 | 5 votes |
|
public static JobStatus fromYarn(ApplicationReport application,
String jobFile) {
String trackingUrl = application.getTrackingUrl();
trackingUrl = trackingUrl == null ? "" : trackingUrl;
JobStatus jobStatus =
new JobStatus(
TypeConverter.fromYarn(application.getApplicationId()),
0.0f, 0.0f, 0.0f, 0.0f,
TypeConverter.fromYarn(application.getYarnApplicationState(), application.getFinalApplicationStatus()),
org.apache.hadoop.mapreduce.JobPriority.NORMAL,
application.getUser(), application.getName(),
application.getQueue(), jobFile, trackingUrl, false
);
jobStatus.setSchedulingInfo(trackingUrl); // Set AM tracking url
jobStatus.setStartTime(application.getStartTime());
jobStatus.setFinishTime(application.getFinishTime());
jobStatus.setFailureInfo(application.getDiagnostics());
ApplicationResourceUsageReport resourceUsageReport =
application.getApplicationResourceUsageReport();
if (resourceUsageReport != null) {
jobStatus.setNeededMem(
resourceUsageReport.getNeededResources().getMemory());
jobStatus.setNumReservedSlots(
resourceUsageReport.getNumReservedContainers());
jobStatus.setNumUsedSlots(resourceUsageReport.getNumUsedContainers());
jobStatus.setReservedMem(
resourceUsageReport.getReservedResources().getMemory());
jobStatus.setUsedMem(resourceUsageReport.getUsedResources().getMemory());
}
return jobStatus;
}
Example 8
| Source File: TestClientRMService.java From big-c with Apache License 2.0 | 5 votes |
|
@Test
public void testGetApplicationReport() throws Exception {
YarnScheduler yarnScheduler = mock(YarnScheduler.class);
RMContext rmContext = mock(RMContext.class);
mockRMContext(yarnScheduler, rmContext);
ApplicationId appId1 = getApplicationId(1);
ApplicationACLsManager mockAclsManager = mock(ApplicationACLsManager.class);
when(
mockAclsManager.checkAccess(UserGroupInformation.getCurrentUser(),
ApplicationAccessType.VIEW_APP, null, appId1)).thenReturn(true);
ClientRMService rmService = new ClientRMService(rmContext, yarnScheduler,
null, mockAclsManager, null, null);
try {
RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null);
GetApplicationReportRequest request = recordFactory
.newRecordInstance(GetApplicationReportRequest.class);
request.setApplicationId(appId1);
GetApplicationReportResponse response =
rmService.getApplicationReport(request);
ApplicationReport report = response.getApplicationReport();
ApplicationResourceUsageReport usageReport =
report.getApplicationResourceUsageReport();
Assert.assertEquals(10, usageReport.getMemorySeconds());
Assert.assertEquals(3, usageReport.getVcoreSeconds());
} finally {
rmService.close();
}
}
Example 9
| Source File: TypeConverter.java From big-c with Apache License 2.0 | 5 votes |
|
public static JobStatus fromYarn(ApplicationReport application,
String jobFile) {
String trackingUrl = application.getTrackingUrl();
trackingUrl = trackingUrl == null ? "" : trackingUrl;
JobStatus jobStatus =
new JobStatus(
TypeConverter.fromYarn(application.getApplicationId()),
0.0f, 0.0f, 0.0f, 0.0f,
TypeConverter.fromYarn(application.getYarnApplicationState(), application.getFinalApplicationStatus()),
org.apache.hadoop.mapreduce.JobPriority.NORMAL,
application.getUser(), application.getName(),
application.getQueue(), jobFile, trackingUrl, false
);
jobStatus.setSchedulingInfo(trackingUrl); // Set AM tracking url
jobStatus.setStartTime(application.getStartTime());
jobStatus.setFinishTime(application.getFinishTime());
jobStatus.setFailureInfo(application.getDiagnostics());
ApplicationResourceUsageReport resourceUsageReport =
application.getApplicationResourceUsageReport();
if (resourceUsageReport != null) {
jobStatus.setNeededMem(
resourceUsageReport.getNeededResources().getMemory());
jobStatus.setNumReservedSlots(
resourceUsageReport.getNumReservedContainers());
jobStatus.setNumUsedSlots(resourceUsageReport.getNumUsedContainers());
jobStatus.setReservedMem(
resourceUsageReport.getReservedResources().getMemory());
jobStatus.setUsedMem(resourceUsageReport.getUsedResources().getMemory());
}
return jobStatus;
}
Example 10
| Source File: TestApplicationHistoryManagerOnTimelineStore.java From hadoop with Apache License 2.0 | 4 votes |
|
@Test
public void testGetApplicationReport() throws Exception {
for (int i = 1; i <= 2; ++i) {
final ApplicationId appId = ApplicationId.newInstance(0, i);
ApplicationReport app;
if (callerUGI == null) {
app = historyManager.getApplication(appId);
} else {
app =
callerUGI.doAs(new PrivilegedExceptionAction<ApplicationReport> () {
@Override
public ApplicationReport run() throws Exception {
return historyManager.getApplication(appId);
}
});
}
Assert.assertNotNull(app);
Assert.assertEquals(appId, app.getApplicationId());
Assert.assertEquals("test app", app.getName());
Assert.assertEquals("test app type", app.getApplicationType());
Assert.assertEquals("user1", app.getUser());
Assert.assertEquals("test queue", app.getQueue());
Assert.assertEquals(Integer.MAX_VALUE + 2L, app.getStartTime());
Assert.assertEquals(Integer.MAX_VALUE + 3L, app.getFinishTime());
Assert.assertTrue(Math.abs(app.getProgress() - 1.0F) < 0.0001);
// App 2 doesn't have the ACLs, such that the default ACLs " " will be used.
// Nobody except admin and owner has access to the details of the app.
if ((i == 1 && callerUGI != null &&
callerUGI.getShortUserName().equals("user3")) ||
(i == 2 && callerUGI != null &&
(callerUGI.getShortUserName().equals("user2") ||
callerUGI.getShortUserName().equals("user3")))) {
Assert.assertEquals(ApplicationAttemptId.newInstance(appId, -1),
app.getCurrentApplicationAttemptId());
Assert.assertEquals(ApplicationHistoryManagerOnTimelineStore.UNAVAILABLE,
app.getHost());
Assert.assertEquals(-1, app.getRpcPort());
Assert.assertEquals(ApplicationHistoryManagerOnTimelineStore.UNAVAILABLE,
app.getTrackingUrl());
Assert.assertEquals(ApplicationHistoryManagerOnTimelineStore.UNAVAILABLE,
app.getOriginalTrackingUrl());
Assert.assertEquals("", app.getDiagnostics());
} else {
Assert.assertEquals(ApplicationAttemptId.newInstance(appId, 1),
app.getCurrentApplicationAttemptId());
Assert.assertEquals("test host", app.getHost());
Assert.assertEquals(100, app.getRpcPort());
Assert.assertEquals("test tracking url", app.getTrackingUrl());
Assert.assertEquals("test original tracking url",
app.getOriginalTrackingUrl());
Assert.assertEquals("test diagnostics info", app.getDiagnostics());
}
ApplicationResourceUsageReport applicationResourceUsageReport =
app.getApplicationResourceUsageReport();
Assert.assertEquals(123,
applicationResourceUsageReport.getMemorySeconds());
Assert
.assertEquals(345, applicationResourceUsageReport.getVcoreSeconds());
Assert
.assertEquals(345, applicationResourceUsageReport.getGcoreSeconds());
Assert.assertEquals(FinalApplicationStatus.UNDEFINED,
app.getFinalApplicationStatus());
Assert.assertEquals(YarnApplicationState.FINISHED,
app.getYarnApplicationState());
}
}
Example 11
| Source File: ApplicationCLI.java From hadoop with Apache License 2.0 | 4 votes |
|
/**
* Prints the application report for an application id.
*
* @param applicationId
* @return exitCode
* @throws YarnException
*/
private int printApplicationReport(String applicationId)
throws YarnException, IOException {
ApplicationReport appReport = null;
try {
appReport = client.getApplicationReport(ConverterUtils
.toApplicationId(applicationId));
} catch (ApplicationNotFoundException e) {
sysout.println("Application with id '" + applicationId
+ "' doesn't exist in RM or Timeline Server.");
return -1;
}
// Use PrintWriter.println, which uses correct platform line ending.
ByteArrayOutputStream baos = new ByteArrayOutputStream();
PrintWriter appReportStr = new PrintWriter(
new OutputStreamWriter(baos, Charset.forName("UTF-8")));
if (appReport != null) {
appReportStr.println("Application Report : ");
appReportStr.print("\tApplication-Id : ");
appReportStr.println(appReport.getApplicationId());
appReportStr.print("\tApplication-Name : ");
appReportStr.println(appReport.getName());
appReportStr.print("\tApplication-Type : ");
appReportStr.println(appReport.getApplicationType());
appReportStr.print("\tUser : ");
appReportStr.println(appReport.getUser());
appReportStr.print("\tQueue : ");
appReportStr.println(appReport.getQueue());
appReportStr.print("\tStart-Time : ");
appReportStr.println(appReport.getStartTime());
appReportStr.print("\tFinish-Time : ");
appReportStr.println(appReport.getFinishTime());
appReportStr.print("\tProgress : ");
DecimalFormat formatter = new DecimalFormat("###.##%");
String progress = formatter.format(appReport.getProgress());
appReportStr.println(progress);
appReportStr.print("\tState : ");
appReportStr.println(appReport.getYarnApplicationState());
appReportStr.print("\tFinal-State : ");
appReportStr.println(appReport.getFinalApplicationStatus());
appReportStr.print("\tTracking-URL : ");
appReportStr.println(appReport.getOriginalTrackingUrl());
appReportStr.print("\tRPC Port : ");
appReportStr.println(appReport.getRpcPort());
appReportStr.print("\tAM Host : ");
appReportStr.println(appReport.getHost());
appReportStr.print("\tAggregate Resource Allocation : ");
ApplicationResourceUsageReport usageReport =
appReport.getApplicationResourceUsageReport();
if (usageReport != null) {
//completed app report in the timeline server doesn't have usage report
appReportStr.print(usageReport.getMemorySeconds() + " MB-seconds, ");
appReportStr.println(usageReport.getVcoreSeconds() + " vcore-seconds");
appReportStr.println(usageReport.getGcoreSeconds() + " gcore-seconds");
} else {
appReportStr.println("N/A");
}
appReportStr.print("\tDiagnostics : ");
appReportStr.print(appReport.getDiagnostics());
} else {
appReportStr.print("Application with id '" + applicationId
+ "' doesn't exist in RM.");
appReportStr.close();
sysout.println(baos.toString("UTF-8"));
return -1;
}
appReportStr.close();
sysout.println(baos.toString("UTF-8"));
return 0;
}
Example 12
| Source File: TestApplicationHistoryManagerOnTimelineStore.java From big-c with Apache License 2.0 | 4 votes |
|
@Test
public void testGetApplicationReport() throws Exception {
for (int i = 1; i <= 2; ++i) {
final ApplicationId appId = ApplicationId.newInstance(0, i);
ApplicationReport app;
if (callerUGI == null) {
app = historyManager.getApplication(appId);
} else {
app =
callerUGI.doAs(new PrivilegedExceptionAction<ApplicationReport> () {
@Override
public ApplicationReport run() throws Exception {
return historyManager.getApplication(appId);
}
});
}
Assert.assertNotNull(app);
Assert.assertEquals(appId, app.getApplicationId());
Assert.assertEquals("test app", app.getName());
Assert.assertEquals("test app type", app.getApplicationType());
Assert.assertEquals("user1", app.getUser());
Assert.assertEquals("test queue", app.getQueue());
Assert.assertEquals(Integer.MAX_VALUE + 2L, app.getStartTime());
Assert.assertEquals(Integer.MAX_VALUE + 3L, app.getFinishTime());
Assert.assertTrue(Math.abs(app.getProgress() - 1.0F) < 0.0001);
// App 2 doesn't have the ACLs, such that the default ACLs " " will be used.
// Nobody except admin and owner has access to the details of the app.
if ((i == 1 && callerUGI != null &&
callerUGI.getShortUserName().equals("user3")) ||
(i == 2 && callerUGI != null &&
(callerUGI.getShortUserName().equals("user2") ||
callerUGI.getShortUserName().equals("user3")))) {
Assert.assertEquals(ApplicationAttemptId.newInstance(appId, -1),
app.getCurrentApplicationAttemptId());
Assert.assertEquals(ApplicationHistoryManagerOnTimelineStore.UNAVAILABLE,
app.getHost());
Assert.assertEquals(-1, app.getRpcPort());
Assert.assertEquals(ApplicationHistoryManagerOnTimelineStore.UNAVAILABLE,
app.getTrackingUrl());
Assert.assertEquals(ApplicationHistoryManagerOnTimelineStore.UNAVAILABLE,
app.getOriginalTrackingUrl());
Assert.assertEquals("", app.getDiagnostics());
} else {
Assert.assertEquals(ApplicationAttemptId.newInstance(appId, 1),
app.getCurrentApplicationAttemptId());
Assert.assertEquals("test host", app.getHost());
Assert.assertEquals(100, app.getRpcPort());
Assert.assertEquals("test tracking url", app.getTrackingUrl());
Assert.assertEquals("test original tracking url",
app.getOriginalTrackingUrl());
Assert.assertEquals("test diagnostics info", app.getDiagnostics());
}
ApplicationResourceUsageReport applicationResourceUsageReport =
app.getApplicationResourceUsageReport();
Assert.assertEquals(123,
applicationResourceUsageReport.getMemorySeconds());
Assert
.assertEquals(345, applicationResourceUsageReport.getVcoreSeconds());
Assert.assertEquals(FinalApplicationStatus.UNDEFINED,
app.getFinalApplicationStatus());
Assert.assertEquals(YarnApplicationState.FINISHED,
app.getYarnApplicationState());
}
}
Example 13
| Source File: ApplicationCLI.java From big-c with Apache License 2.0 | 4 votes |
|
/**
* Prints the application report for an application id.
*
* @param applicationId
* @return exitCode
* @throws YarnException
*/
private int printApplicationReport(String applicationId)
throws YarnException, IOException {
ApplicationReport appReport = null;
try {
appReport = client.getApplicationReport(ConverterUtils
.toApplicationId(applicationId));
} catch (ApplicationNotFoundException e) {
sysout.println("Application with id '" + applicationId
+ "' doesn't exist in RM or Timeline Server.");
return -1;
}
// Use PrintWriter.println, which uses correct platform line ending.
ByteArrayOutputStream baos = new ByteArrayOutputStream();
PrintWriter appReportStr = new PrintWriter(
new OutputStreamWriter(baos, Charset.forName("UTF-8")));
if (appReport != null) {
appReportStr.println("Application Report : ");
appReportStr.print("\tApplication-Id : ");
appReportStr.println(appReport.getApplicationId());
appReportStr.print("\tApplication-Name : ");
appReportStr.println(appReport.getName());
appReportStr.print("\tApplication-Type : ");
appReportStr.println(appReport.getApplicationType());
appReportStr.print("\tUser : ");
appReportStr.println(appReport.getUser());
appReportStr.print("\tQueue : ");
appReportStr.println(appReport.getQueue());
appReportStr.print("\tStart-Time : ");
appReportStr.println(appReport.getStartTime());
appReportStr.print("\tFinish-Time : ");
appReportStr.println(appReport.getFinishTime());
appReportStr.print("\tProgress : ");
DecimalFormat formatter = new DecimalFormat("###.##%");
String progress = formatter.format(appReport.getProgress());
appReportStr.println(progress);
appReportStr.print("\tState : ");
appReportStr.println(appReport.getYarnApplicationState());
appReportStr.print("\tFinal-State : ");
appReportStr.println(appReport.getFinalApplicationStatus());
appReportStr.print("\tTracking-URL : ");
appReportStr.println(appReport.getOriginalTrackingUrl());
appReportStr.print("\tRPC Port : ");
appReportStr.println(appReport.getRpcPort());
appReportStr.print("\tAM Host : ");
appReportStr.println(appReport.getHost());
appReportStr.print("\tAggregate Resource Allocation : ");
ApplicationResourceUsageReport usageReport =
appReport.getApplicationResourceUsageReport();
if (usageReport != null) {
//completed app report in the timeline server doesn't have usage report
appReportStr.print(usageReport.getMemorySeconds() + " MB-seconds, ");
appReportStr.println(usageReport.getVcoreSeconds() + " vcore-seconds");
} else {
appReportStr.println("N/A");
}
appReportStr.print("\tDiagnostics : ");
appReportStr.print(appReport.getDiagnostics());
} else {
appReportStr.print("Application with id '" + applicationId
+ "' doesn't exist in RM.");
appReportStr.close();
sysout.println(baos.toString("UTF-8"));
return -1;
}
appReportStr.close();
sysout.println(baos.toString("UTF-8"));
return 0;
}
