Java Code Examples for org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp#getRMAppMetrics()
The following examples show how to use
org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp#getRMAppMetrics() .
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: TestCapacityScheduler.java From hadoop with Apache License 2.0 | 6 votes |
private void waitForAppPreemptionInfo(RMApp app, Resource preempted, int numAMPreempted, int numTaskPreempted, Resource currentAttemptPreempted, boolean currentAttemptAMPreempted, int numLatestAttemptTaskPreempted) throws InterruptedException { while (true) { RMAppMetrics appPM = app.getRMAppMetrics(); RMAppAttemptMetrics attemptPM = app.getCurrentAppAttempt().getRMAppAttemptMetrics(); if (appPM.getResourcePreempted().equals(preempted) && appPM.getNumAMContainersPreempted() == numAMPreempted && appPM.getNumNonAMContainersPreempted() == numTaskPreempted && attemptPM.getResourcePreempted().equals(currentAttemptPreempted) && app.getCurrentAppAttempt().getRMAppAttemptMetrics() .getIsPreempted() == currentAttemptAMPreempted && attemptPM.getNumNonAMContainersPreempted() == numLatestAttemptTaskPreempted) { return; } Thread.sleep(500); } }
Example 2
Source File: TestCapacityScheduler.java From big-c with Apache License 2.0 | 6 votes |
private void waitForAppPreemptionInfo(RMApp app, Resource preempted, int numAMPreempted, int numTaskPreempted, Resource currentAttemptPreempted, boolean currentAttemptAMPreempted, int numLatestAttemptTaskPreempted) throws InterruptedException { while (true) { RMAppMetrics appPM = app.getRMAppMetrics(); RMAppAttemptMetrics attemptPM = app.getCurrentAppAttempt().getRMAppAttemptMetrics(); if (appPM.getResourcePreempted().equals(preempted) && appPM.getNumAMContainersPreempted() == numAMPreempted && appPM.getNumNonAMContainersPreempted() == numTaskPreempted && attemptPM.getResourcePreempted().equals(currentAttemptPreempted) && app.getCurrentAppAttempt().getRMAppAttemptMetrics() .getIsPreempted() == currentAttemptAMPreempted && attemptPM.getNumNonAMContainersPreempted() == numLatestAttemptTaskPreempted) { return; } Thread.sleep(500); } }
Example 3
Source File: RMAppManager.java From hadoop with Apache License 2.0 | 5 votes |
/** * create a summary of the application's runtime. * * @param app {@link RMApp} whose summary is to be created, cannot * be <code>null</code>. */ public static SummaryBuilder createAppSummary(RMApp app) { String trackingUrl = "N/A"; String host = "N/A"; RMAppAttempt attempt = app.getCurrentAppAttempt(); if (attempt != null) { trackingUrl = attempt.getTrackingUrl(); host = attempt.getHost(); } RMAppMetrics metrics = app.getRMAppMetrics(); SummaryBuilder summary = new SummaryBuilder() .add("appId", app.getApplicationId()) .add("name", app.getName()) .add("user", app.getUser()) .add("queue", app.getQueue()) .add("state", app.getState()) .add("trackingUrl", trackingUrl) .add("appMasterHost", host) .add("startTime", app.getStartTime()) .add("finishTime", app.getFinishTime()) .add("finalStatus", app.getFinalApplicationStatus()) .add("memorySeconds", metrics.getMemorySeconds()) .add("vcoreSeconds", metrics.getVcoreSeconds()) .add("gcoreSeconds", metrics.getGcoreSeconds()) .add("preemptedAMContainers", metrics.getNumAMContainersPreempted()) .add("preemptedNonAMContainers", metrics.getNumNonAMContainersPreempted()) .add("preemptedResources", metrics.getResourcePreempted()) .add("applicationType", app.getApplicationType()); return summary; }
Example 4
Source File: RMAppManager.java From big-c with Apache License 2.0 | 5 votes |
/** * create a summary of the application's runtime. * * @param app {@link RMApp} whose summary is to be created, cannot * be <code>null</code>. */ public static SummaryBuilder createAppSummary(RMApp app) { String trackingUrl = "N/A"; String host = "N/A"; RMAppAttempt attempt = app.getCurrentAppAttempt(); if (attempt != null) { trackingUrl = attempt.getTrackingUrl(); host = attempt.getHost(); } RMAppMetrics metrics = app.getRMAppMetrics(); SummaryBuilder summary = new SummaryBuilder() .add("appId", app.getApplicationId()) .add("name", app.getName()) .add("user", app.getUser()) .add("queue", app.getQueue()) .add("state", app.getState()) .add("trackingUrl", trackingUrl) .add("appMasterHost", host) .add("startTime", app.getStartTime()) .add("finishTime", app.getFinishTime()) .add("finalStatus", app.getFinalApplicationStatus()) .add("memorySeconds", metrics.getMemorySeconds()) .add("vcoreSeconds", metrics.getVcoreSeconds()) .add("preemptedAMContainers", metrics.getNumAMContainersPreempted()) .add("preemptedNonAMContainers", metrics.getNumNonAMContainersPreempted()) .add("preemptedResources", metrics.getResourcePreempted()) .add("applicationType", app.getApplicationType()); return summary; }
Example 5
Source File: RMContextImplEventRunnable.java From garmadon with Apache License 2.0 | 4 votes |
public void sendAppEvent(ApplicationId applicationId, RMApp rmApp) { if (cacheFinishedApp.getIfPresent(applicationId.toString()) == null) { Header.Builder headerBuilder = Header.newBuilder() .withId(applicationId.toString()) .withApplicationID(applicationId.toString()) .withUser(rmApp.getUser()) .withApplicationName(rmApp.getName()) .withFramework(rmApp.getApplicationType().toUpperCase()); ApplicationEvent.Builder eventBuilder = ApplicationEvent.newBuilder() .setState(rmApp.getState().name()) .setQueue(rmApp.getQueue()); rmApp.getApplicationTags().stream() .filter(tag -> YARN_TAGS_TO_EXTRACT.stream().noneMatch(tag::startsWith) && !tag.contains(":")) .forEach(eventBuilder::addYarnTags); rmApp.getApplicationTags().stream() .filter(tag -> tag.contains(":") && YARN_TAGS_TO_EXTRACT.stream().anyMatch(tag::startsWith)) .map(tag -> { int idx = tag.indexOf(':'); String key = tag.substring(0, idx); String value = tag.substring(idx + 1); return new String[] {key, value}; }) .forEach(splitTag -> BUILDERS.get(splitTag[0]).accept(splitTag[1], eventBuilder)); eventBuilder.setFinalStatus(rmApp.getFinalApplicationStatus().name()); eventBuilder.setStartTime(rmApp.getStartTime()); eventBuilder.setFinishTime(rmApp.getFinishTime()); RMAppMetrics rmAppMetrics = rmApp.getRMAppMetrics(); if (rmAppMetrics != null) { eventBuilder.setMemorySeconds(rmAppMetrics.getMemorySeconds()); eventBuilder.setVcoreSeconds(rmAppMetrics.getVcoreSeconds()); } RMAppAttempt rmAppAttempt = rmApp.getCurrentAppAttempt(); if (rmAppAttempt != null) { headerBuilder.withAttemptID(rmAppAttempt.getAppAttemptId().toString()); Container container = rmAppAttempt.getMasterContainer(); if (container != null) { eventBuilder.setAmContainerId(container.getId().toString()); } } if (rmApp.getTrackingUrl() != null) { eventBuilder.setTrackingUrl(normalizeTrackingUrl(rmApp.getTrackingUrl())); } if (rmApp.getOriginalTrackingUrl() != null && !"N/A".equals(rmApp.getOriginalTrackingUrl())) { eventBuilder.setOriginalTrackingUrl(normalizeTrackingUrl(rmApp.getOriginalTrackingUrl())); } eventHandler.accept(System.currentTimeMillis(), headerBuilder.build(), eventBuilder.build()); if (rmApp.getState() == RMAppState.FINISHED || rmApp.getState() == RMAppState.KILLED || rmApp.getState() == RMAppState.FAILED) { cacheFinishedApp.put(applicationId.toString(), rmApp.getState().name()); } } }
Example 6
Source File: AppInfo.java From hadoop with Apache License 2.0 | 4 votes |
@SuppressWarnings({ "rawtypes", "unchecked" }) public AppInfo(ResourceManager rm, RMApp app, Boolean hasAccess, String schemePrefix) { this.schemePrefix = schemePrefix; if (app != null) { String trackingUrl = app.getTrackingUrl(); this.state = app.createApplicationState(); this.trackingUrlIsNotReady = trackingUrl == null || trackingUrl.isEmpty() || YarnApplicationState.NEW == this.state || YarnApplicationState.NEW_SAVING == this.state || YarnApplicationState.SUBMITTED == this.state || YarnApplicationState.ACCEPTED == this.state; this.trackingUI = this.trackingUrlIsNotReady ? "UNASSIGNED" : (app .getFinishTime() == 0 ? "ApplicationMaster" : "History"); if (!trackingUrlIsNotReady) { this.trackingUrl = WebAppUtils.getURLWithScheme(schemePrefix, trackingUrl); this.trackingUrlPretty = this.trackingUrl; } else { this.trackingUrlPretty = "UNASSIGNED"; } this.applicationId = app.getApplicationId(); this.applicationType = app.getApplicationType(); this.appIdNum = String.valueOf(app.getApplicationId().getId()); this.id = app.getApplicationId().toString(); this.user = app.getUser().toString(); this.name = app.getName().toString(); this.queue = app.getQueue().toString(); this.progress = app.getProgress() * 100; this.diagnostics = app.getDiagnostics().toString(); if (diagnostics == null || diagnostics.isEmpty()) { this.diagnostics = ""; } if (app.getApplicationTags() != null && !app.getApplicationTags().isEmpty()) { this.applicationTags = Joiner.on(',').join(app.getApplicationTags()); } this.finalStatus = app.getFinalApplicationStatus(); this.clusterId = ResourceManager.getClusterTimeStamp(); if (hasAccess) { this.startedTime = app.getStartTime(); this.finishedTime = app.getFinishTime(); this.elapsedTime = Times.elapsed(app.getStartTime(), app.getFinishTime()); RMAppAttempt attempt = app.getCurrentAppAttempt(); if (attempt != null) { Container masterContainer = attempt.getMasterContainer(); if (masterContainer != null) { this.amContainerLogsExist = true; this.amContainerLogs = WebAppUtils.getRunningLogURL( schemePrefix + masterContainer.getNodeHttpAddress(), ConverterUtils.toString(masterContainer.getId()), app.getUser()); this.amHostHttpAddress = masterContainer.getNodeHttpAddress(); } ApplicationResourceUsageReport resourceReport = attempt .getApplicationResourceUsageReport(); if (resourceReport != null) { Resource usedResources = resourceReport.getUsedResources(); allocatedMB = usedResources.getMemory(); allocatedVCores = usedResources.getVirtualCores(); allocatedGCores = usedResources.getGpuCores(); runningContainers = resourceReport.getNumUsedContainers(); } resourceRequests = ((AbstractYarnScheduler) rm.getRMContext().getScheduler()) .getPendingResourceRequestsForAttempt(attempt.getAppAttemptId()); } } // copy preemption info fields RMAppMetrics appMetrics = app.getRMAppMetrics(); numAMContainerPreempted = appMetrics.getNumAMContainersPreempted(); preemptedResourceMB = appMetrics.getResourcePreempted().getMemory(); numNonAMContainerPreempted = appMetrics.getNumNonAMContainersPreempted(); preemptedResourceVCores = appMetrics.getResourcePreempted().getVirtualCores(); preemptedResourceGCores = appMetrics.getResourcePreempted().getGpuCores(); memorySeconds = appMetrics.getMemorySeconds(); vcoreSeconds = appMetrics.getVcoreSeconds(); gcoreSeconds = appMetrics.getGcoreSeconds(); } }
Example 7
Source File: RMAppBlock.java From hadoop with Apache License 2.0 | 4 votes |
@Override protected void createApplicationMetricsTable(Block html){ RMApp rmApp = this.rm.getRMContext().getRMApps().get(appID); RMAppMetrics appMetrics = rmApp == null ? null : rmApp.getRMAppMetrics(); // Get attempt metrics and fields, it is possible currentAttempt of RMApp is // null. In that case, we will assume resource preempted and number of Non // AM container preempted on that attempt is 0 RMAppAttemptMetrics attemptMetrics; if (rmApp == null || null == rmApp.getCurrentAppAttempt()) { attemptMetrics = null; } else { attemptMetrics = rmApp.getCurrentAppAttempt().getRMAppAttemptMetrics(); } Resource attemptResourcePreempted = attemptMetrics == null ? Resources.none() : attemptMetrics .getResourcePreempted(); int attemptNumNonAMContainerPreempted = attemptMetrics == null ? 0 : attemptMetrics .getNumNonAMContainersPreempted(); DIV<Hamlet> pdiv = html. _(InfoBlock.class). div(_INFO_WRAP); info("Application Overview").clear(); info("Application Metrics") ._("Total Resource Preempted:", appMetrics == null ? "N/A" : appMetrics.getResourcePreempted()) ._("Total Number of Non-AM Containers Preempted:", appMetrics == null ? "N/A" : appMetrics.getNumNonAMContainersPreempted()) ._("Total Number of AM Containers Preempted:", appMetrics == null ? "N/A" : appMetrics.getNumAMContainersPreempted()) ._("Resource Preempted from Current Attempt:", attemptResourcePreempted) ._("Number of Non-AM Containers Preempted from Current Attempt:", attemptNumNonAMContainerPreempted) ._("Aggregate Resource Allocation:", String.format("%d MB-seconds, %d vcore-seconds, %d gcore-seconds", appMetrics == null ? "N/A" : appMetrics.getMemorySeconds(), appMetrics == null ? "N/A" : appMetrics.getVcoreSeconds(), appMetrics == null ? "N/A" : appMetrics.getGcoreSeconds())); pdiv._(); }
Example 8
Source File: TestContainerResourceUsage.java From hadoop with Apache License 2.0 | 4 votes |
@Test (timeout = 120000) public void testUsageWithOneAttemptAndOneContainer() throws Exception { MockRM rm = new MockRM(conf); rm.start(); MockNM nm = new MockNM("127.0.0.1:1234", 15120, rm.getResourceTrackerService()); nm.registerNode(); RMApp app0 = rm.submitApp(200); RMAppMetrics rmAppMetrics = app0.getRMAppMetrics(); Assert.assertTrue( "Before app submittion, memory seconds should have been 0 but was " + rmAppMetrics.getMemorySeconds(), rmAppMetrics.getMemorySeconds() == 0); Assert.assertTrue( "Before app submission, vcore seconds should have been 0 but was " + rmAppMetrics.getVcoreSeconds(), rmAppMetrics.getVcoreSeconds() == 0); Assert.assertTrue( "Before app submission, gcore seconds should have been 0 but was " + rmAppMetrics.getGcoreSeconds(), rmAppMetrics.getGcoreSeconds() == 0); RMAppAttempt attempt0 = app0.getCurrentAppAttempt(); nm.nodeHeartbeat(true); MockAM am0 = rm.sendAMLaunched(attempt0.getAppAttemptId()); am0.registerAppAttempt(); RMContainer rmContainer = rm.getResourceScheduler() .getRMContainer(attempt0.getMasterContainer().getId()); // Allow metrics to accumulate. int sleepInterval = 1000; int cumulativeSleepTime = 0; while (rmAppMetrics.getMemorySeconds() <= 0 && cumulativeSleepTime < 5000) { Thread.sleep(sleepInterval); cumulativeSleepTime += sleepInterval; } rmAppMetrics = app0.getRMAppMetrics(); Assert.assertTrue( "While app is running, memory seconds should be >0 but is " + rmAppMetrics.getMemorySeconds(), rmAppMetrics.getMemorySeconds() > 0); Assert.assertTrue( "While app is running, vcore seconds should be >0 but is " + rmAppMetrics.getVcoreSeconds(), rmAppMetrics.getVcoreSeconds() > 0); MockRM.finishAMAndVerifyAppState(app0, rm, nm, am0); AggregateAppResourceUsage ru = calculateContainerResourceMetrics(rmContainer); rmAppMetrics = app0.getRMAppMetrics(); Assert.assertEquals("Unexcpected MemorySeconds value", ru.getMemorySeconds(), rmAppMetrics.getMemorySeconds()); Assert.assertEquals("Unexpected VcoreSeconds value", ru.getVcoreSeconds(), rmAppMetrics.getVcoreSeconds()); Assert.assertEquals("Unexpected GcoreSeconds value", ru.getGcoreSeconds(), rmAppMetrics.getGcoreSeconds()); rm.stop(); }
Example 9
Source File: AppInfo.java From big-c with Apache License 2.0 | 4 votes |
@SuppressWarnings({ "rawtypes", "unchecked" }) public AppInfo(ResourceManager rm, RMApp app, Boolean hasAccess, String schemePrefix) { this.schemePrefix = schemePrefix; if (app != null) { String trackingUrl = app.getTrackingUrl(); this.state = app.createApplicationState(); this.trackingUrlIsNotReady = trackingUrl == null || trackingUrl.isEmpty() || YarnApplicationState.NEW == this.state || YarnApplicationState.NEW_SAVING == this.state || YarnApplicationState.SUBMITTED == this.state || YarnApplicationState.ACCEPTED == this.state; this.trackingUI = this.trackingUrlIsNotReady ? "UNASSIGNED" : (app .getFinishTime() == 0 ? "ApplicationMaster" : "History"); if (!trackingUrlIsNotReady) { this.trackingUrl = WebAppUtils.getURLWithScheme(schemePrefix, trackingUrl); this.trackingUrlPretty = this.trackingUrl; } else { this.trackingUrlPretty = "UNASSIGNED"; } this.applicationId = app.getApplicationId(); this.applicationType = app.getApplicationType(); this.appIdNum = String.valueOf(app.getApplicationId().getId()); this.id = app.getApplicationId().toString(); this.user = app.getUser().toString(); this.name = app.getName().toString(); this.queue = app.getQueue().toString(); this.progress = app.getProgress() * 100; this.diagnostics = app.getDiagnostics().toString(); if (diagnostics == null || diagnostics.isEmpty()) { this.diagnostics = ""; } if (app.getApplicationTags() != null && !app.getApplicationTags().isEmpty()) { this.applicationTags = Joiner.on(',').join(app.getApplicationTags()); } this.finalStatus = app.getFinalApplicationStatus(); this.clusterId = ResourceManager.getClusterTimeStamp(); if (hasAccess) { this.startedTime = app.getStartTime(); this.finishedTime = app.getFinishTime(); this.elapsedTime = Times.elapsed(app.getStartTime(), app.getFinishTime()); RMAppAttempt attempt = app.getCurrentAppAttempt(); if (attempt != null) { Container masterContainer = attempt.getMasterContainer(); if (masterContainer != null) { this.amContainerLogsExist = true; this.amContainerLogs = WebAppUtils.getRunningLogURL( schemePrefix + masterContainer.getNodeHttpAddress(), ConverterUtils.toString(masterContainer.getId()), app.getUser()); this.amHostHttpAddress = masterContainer.getNodeHttpAddress(); } ApplicationResourceUsageReport resourceReport = attempt .getApplicationResourceUsageReport(); if (resourceReport != null) { Resource usedResources = resourceReport.getUsedResources(); allocatedMB = usedResources.getMemory(); allocatedVCores = usedResources.getVirtualCores(); runningContainers = resourceReport.getNumUsedContainers(); } resourceRequests = ((AbstractYarnScheduler) rm.getRMContext().getScheduler()) .getPendingResourceRequestsForAttempt(attempt.getAppAttemptId()); } } // copy preemption info fields RMAppMetrics appMetrics = app.getRMAppMetrics(); numAMContainerPreempted = appMetrics.getNumAMContainersPreempted(); preemptedResourceMB = appMetrics.getResourcePreempted().getMemory(); numNonAMContainerPreempted = appMetrics.getNumNonAMContainersPreempted(); preemptedResourceVCores = appMetrics.getResourcePreempted().getVirtualCores(); memorySeconds = appMetrics.getMemorySeconds(); vcoreSeconds = appMetrics.getVcoreSeconds(); } }
Example 10
Source File: RMAppBlock.java From big-c with Apache License 2.0 | 4 votes |
@Override protected void createApplicationMetricsTable(Block html){ RMApp rmApp = this.rm.getRMContext().getRMApps().get(appID); RMAppMetrics appMetrics = rmApp == null ? null : rmApp.getRMAppMetrics(); // Get attempt metrics and fields, it is possible currentAttempt of RMApp is // null. In that case, we will assume resource preempted and number of Non // AM container preempted on that attempt is 0 RMAppAttemptMetrics attemptMetrics; if (rmApp == null || null == rmApp.getCurrentAppAttempt()) { attemptMetrics = null; } else { attemptMetrics = rmApp.getCurrentAppAttempt().getRMAppAttemptMetrics(); } Resource attemptResourcePreempted = attemptMetrics == null ? Resources.none() : attemptMetrics .getResourcePreempted(); int attemptNumNonAMContainerPreempted = attemptMetrics == null ? 0 : attemptMetrics .getNumNonAMContainersPreempted(); DIV<Hamlet> pdiv = html. _(InfoBlock.class). div(_INFO_WRAP); info("Application Overview").clear(); info("Application Metrics") ._("Total Resource Preempted:", appMetrics == null ? "N/A" : appMetrics.getResourcePreempted()) ._("Total Number of Non-AM Containers Preempted:", appMetrics == null ? "N/A" : appMetrics.getNumNonAMContainersPreempted()) ._("Total Number of AM Containers Preempted:", appMetrics == null ? "N/A" : appMetrics.getNumAMContainersPreempted()) ._("Resource Preempted from Current Attempt:", attemptResourcePreempted) ._("Number of Non-AM Containers Preempted from Current Attempt:", attemptNumNonAMContainerPreempted) ._("Aggregate Resource Allocation:", String.format("%d MB-seconds, %d vcore-seconds", appMetrics == null ? "N/A" : appMetrics.getMemorySeconds(), appMetrics == null ? "N/A" : appMetrics.getVcoreSeconds())); pdiv._(); }
Example 11
Source File: TestContainerResourceUsage.java From big-c with Apache License 2.0 | 4 votes |
@Test (timeout = 120000) public void testUsageWithOneAttemptAndOneContainer() throws Exception { MockRM rm = new MockRM(conf); rm.start(); MockNM nm = new MockNM("127.0.0.1:1234", 15120, rm.getResourceTrackerService()); nm.registerNode(); RMApp app0 = rm.submitApp(200); RMAppMetrics rmAppMetrics = app0.getRMAppMetrics(); Assert.assertTrue( "Before app submittion, memory seconds should have been 0 but was " + rmAppMetrics.getMemorySeconds(), rmAppMetrics.getMemorySeconds() == 0); Assert.assertTrue( "Before app submission, vcore seconds should have been 0 but was " + rmAppMetrics.getVcoreSeconds(), rmAppMetrics.getVcoreSeconds() == 0); RMAppAttempt attempt0 = app0.getCurrentAppAttempt(); nm.nodeHeartbeat(true); MockAM am0 = rm.sendAMLaunched(attempt0.getAppAttemptId()); am0.registerAppAttempt(); RMContainer rmContainer = rm.getResourceScheduler() .getRMContainer(attempt0.getMasterContainer().getId()); // Allow metrics to accumulate. int sleepInterval = 1000; int cumulativeSleepTime = 0; while (rmAppMetrics.getMemorySeconds() <= 0 && cumulativeSleepTime < 5000) { Thread.sleep(sleepInterval); cumulativeSleepTime += sleepInterval; } rmAppMetrics = app0.getRMAppMetrics(); Assert.assertTrue( "While app is running, memory seconds should be >0 but is " + rmAppMetrics.getMemorySeconds(), rmAppMetrics.getMemorySeconds() > 0); Assert.assertTrue( "While app is running, vcore seconds should be >0 but is " + rmAppMetrics.getVcoreSeconds(), rmAppMetrics.getVcoreSeconds() > 0); MockRM.finishAMAndVerifyAppState(app0, rm, nm, am0); AggregateAppResourceUsage ru = calculateContainerResourceMetrics(rmContainer); rmAppMetrics = app0.getRMAppMetrics(); Assert.assertEquals("Unexcpected MemorySeconds value", ru.getMemorySeconds(), rmAppMetrics.getMemorySeconds()); Assert.assertEquals("Unexpected VcoreSeconds value", ru.getVcoreSeconds(), rmAppMetrics.getVcoreSeconds()); rm.stop(); }