org.apache.hadoop.yarn.server.resourcemanager.ApplicationMasterService Java Examples
The following examples show how to use
org.apache.hadoop.yarn.server.resourcemanager.ApplicationMasterService.
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: RMAppAttemptImpl.java From hadoop with Apache License 2.0 | 6 votes |
public RMAppAttemptImpl(ApplicationAttemptId appAttemptId, RMContext rmContext, YarnScheduler scheduler, ApplicationMasterService masterService, ApplicationSubmissionContext submissionContext, Configuration conf, boolean maybeLastAttempt, ResourceRequest amReq) { this.conf = conf; this.applicationAttemptId = appAttemptId; this.rmContext = rmContext; this.eventHandler = rmContext.getDispatcher().getEventHandler(); this.submissionContext = submissionContext; this.scheduler = scheduler; this.masterService = masterService; ReentrantReadWriteLock lock = new ReentrantReadWriteLock(); this.readLock = lock.readLock(); this.writeLock = lock.writeLock(); this.proxiedTrackingUrl = generateProxyUriWithScheme(); this.maybeLastAttempt = maybeLastAttempt; this.stateMachine = stateMachineFactory.make(this); this.attemptMetrics = new RMAppAttemptMetrics(applicationAttemptId, rmContext); this.amReq = amReq; }
Example #2
Source File: RMAppAttemptImpl.java From big-c with Apache License 2.0 | 6 votes |
public RMAppAttemptImpl(ApplicationAttemptId appAttemptId, RMContext rmContext, YarnScheduler scheduler, ApplicationMasterService masterService, ApplicationSubmissionContext submissionContext, Configuration conf, boolean maybeLastAttempt, ResourceRequest amReq) { this.conf = conf; this.applicationAttemptId = appAttemptId; this.rmContext = rmContext; this.eventHandler = rmContext.getDispatcher().getEventHandler(); this.submissionContext = submissionContext; this.scheduler = scheduler; this.masterService = masterService; ReentrantReadWriteLock lock = new ReentrantReadWriteLock(); this.readLock = lock.readLock(); this.writeLock = lock.writeLock(); this.proxiedTrackingUrl = generateProxyUriWithScheme(); this.maybeLastAttempt = maybeLastAttempt; this.stateMachine = stateMachineFactory.make(this); this.attemptMetrics = new RMAppAttemptMetrics(applicationAttemptId, rmContext); this.amReq = amReq; }
Example #3
Source File: TestRMAppTransitions.java From hadoop with Apache License 2.0 | 5 votes |
protected RMApp createNewTestApp(ApplicationSubmissionContext submissionContext) { ApplicationId applicationId = MockApps.newAppID(appId++); String user = MockApps.newUserName(); String name = MockApps.newAppName(); String queue = MockApps.newQueue(); // ensure max application attempts set to known value conf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, maxAppAttempts); scheduler = mock(YarnScheduler.class); ApplicationMasterService masterService = new ApplicationMasterService(rmContext, scheduler); if(submissionContext == null) { submissionContext = new ApplicationSubmissionContextPBImpl(); } // applicationId will not be used because RMStateStore is mocked, // but applicationId is still set for safety submissionContext.setApplicationId(applicationId); RMApp application = new RMAppImpl(applicationId, rmContext, conf, name, user, queue, submissionContext, scheduler, masterService, System.currentTimeMillis(), "YARN", null, null); testAppStartState(applicationId, user, name, queue, application); this.rmContext.getRMApps().putIfAbsent(application.getApplicationId(), application); return application; }
Example #4
Source File: ProtocolHATestBase.java From hadoop with Apache License 2.0 | 5 votes |
@Override protected ResourceManager createResourceManager() { return new ResourceManager() { @Override protected void doSecureLogin() throws IOException { // Don't try to login using keytab in the testcases. } @Override protected ClientRMService createClientRMService() { if (overrideClientRMService) { return new CustomedClientRMService(this.rmContext, this.scheduler, this.rmAppManager, this.applicationACLsManager, this.queueACLsManager, this.rmContext.getRMDelegationTokenSecretManager()); } return super.createClientRMService(); } @Override protected ResourceTrackerService createResourceTrackerService() { if (overrideRTS) { return new CustomedResourceTrackerService(this.rmContext, this.nodesListManager, this.nmLivelinessMonitor, this.rmContext.getContainerTokenSecretManager(), this.rmContext.getNMTokenSecretManager()); } return super.createResourceTrackerService(); } @Override protected ApplicationMasterService createApplicationMasterService() { if (overrideApplicationMasterService) { return new CustomedApplicationMasterService(this.rmContext, this.scheduler); } return super.createApplicationMasterService(); } }; }
Example #5
Source File: TestRMAppTransitions.java From big-c with Apache License 2.0 | 5 votes |
protected RMApp createNewTestApp(ApplicationSubmissionContext submissionContext) { ApplicationId applicationId = MockApps.newAppID(appId++); String user = MockApps.newUserName(); String name = MockApps.newAppName(); String queue = MockApps.newQueue(); // ensure max application attempts set to known value conf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, maxAppAttempts); scheduler = mock(YarnScheduler.class); ApplicationMasterService masterService = new ApplicationMasterService(rmContext, scheduler); if(submissionContext == null) { submissionContext = new ApplicationSubmissionContextPBImpl(); } // applicationId will not be used because RMStateStore is mocked, // but applicationId is still set for safety submissionContext.setApplicationId(applicationId); RMApp application = new RMAppImpl(applicationId, rmContext, conf, name, user, queue, submissionContext, scheduler, masterService, System.currentTimeMillis(), "YARN", null, null); testAppStartState(applicationId, user, name, queue, application); this.rmContext.getRMApps().putIfAbsent(application.getApplicationId(), application); return application; }
Example #6
Source File: ProtocolHATestBase.java From big-c with Apache License 2.0 | 5 votes |
@Override protected ResourceManager createResourceManager() { return new ResourceManager() { @Override protected void doSecureLogin() throws IOException { // Don't try to login using keytab in the testcases. } @Override protected ClientRMService createClientRMService() { if (overrideClientRMService) { return new CustomedClientRMService(this.rmContext, this.scheduler, this.rmAppManager, this.applicationACLsManager, this.queueACLsManager, this.rmContext.getRMDelegationTokenSecretManager()); } return super.createClientRMService(); } @Override protected ResourceTrackerService createResourceTrackerService() { if (overrideRTS) { return new CustomedResourceTrackerService(this.rmContext, this.nodesListManager, this.nmLivelinessMonitor, this.rmContext.getContainerTokenSecretManager(), this.rmContext.getNMTokenSecretManager()); } return super.createResourceTrackerService(); } @Override protected ApplicationMasterService createApplicationMasterService() { if (overrideApplicationMasterService) { return new CustomedApplicationMasterService(this.rmContext, this.scheduler); } return super.createApplicationMasterService(); } }; }
Example #7
Source File: RMAppImpl.java From hadoop with Apache License 2.0 | 4 votes |
public RMAppImpl(ApplicationId applicationId, RMContext rmContext, Configuration config, String name, String user, String queue, ApplicationSubmissionContext submissionContext, YarnScheduler scheduler, ApplicationMasterService masterService, long submitTime, String applicationType, Set<String> applicationTags, ResourceRequest amReq) { this.systemClock = new SystemClock(); this.applicationId = applicationId; this.name = name; this.rmContext = rmContext; this.dispatcher = rmContext.getDispatcher(); this.handler = dispatcher.getEventHandler(); this.conf = config; this.user = user; this.queue = queue; this.submissionContext = submissionContext; this.scheduler = scheduler; this.masterService = masterService; this.submitTime = submitTime; this.startTime = this.systemClock.getTime(); this.applicationType = applicationType; this.applicationTags = applicationTags; this.amReq = amReq; int globalMaxAppAttempts = conf.getInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS); int individualMaxAppAttempts = submissionContext.getMaxAppAttempts(); if (individualMaxAppAttempts <= 0 || individualMaxAppAttempts > globalMaxAppAttempts) { this.maxAppAttempts = globalMaxAppAttempts; LOG.warn("The specific max attempts: " + individualMaxAppAttempts + " for application: " + applicationId.getId() + " is invalid, because it is out of the range [1, " + globalMaxAppAttempts + "]. Use the global max attempts instead."); } else { this.maxAppAttempts = individualMaxAppAttempts; } this.attemptFailuresValidityInterval = submissionContext.getAttemptFailuresValidityInterval(); if (this.attemptFailuresValidityInterval > 0) { LOG.info("The attemptFailuresValidityInterval for the application: " + this.applicationId + " is " + this.attemptFailuresValidityInterval + "."); } ReentrantReadWriteLock lock = new ReentrantReadWriteLock(); this.readLock = lock.readLock(); this.writeLock = lock.writeLock(); this.stateMachine = stateMachineFactory.make(this); rmContext.getRMApplicationHistoryWriter().applicationStarted(this); rmContext.getSystemMetricsPublisher().appCreated(this, startTime); }
Example #8
Source File: TestRMAppAttemptTransitions.java From hadoop with Apache License 2.0 | 4 votes |
@SuppressWarnings("deprecation") @Before public void setUp() throws Exception { AuthenticationMethod authMethod = AuthenticationMethod.SIMPLE; if (isSecurityEnabled) { authMethod = AuthenticationMethod.KERBEROS; } SecurityUtil.setAuthenticationMethod(authMethod, conf); UserGroupInformation.setConfiguration(conf); InlineDispatcher rmDispatcher = new InlineDispatcher(); ContainerAllocationExpirer containerAllocationExpirer = mock(ContainerAllocationExpirer.class); amLivelinessMonitor = mock(AMLivelinessMonitor.class); amFinishingMonitor = mock(AMLivelinessMonitor.class); writer = mock(RMApplicationHistoryWriter.class); MasterKeyData masterKeyData = amRMTokenManager.createNewMasterKey(); when(amRMTokenManager.getMasterKey()).thenReturn(masterKeyData); rmContext = new RMContextImpl(rmDispatcher, containerAllocationExpirer, amLivelinessMonitor, amFinishingMonitor, null, amRMTokenManager, new RMContainerTokenSecretManager(conf), nmTokenManager, clientToAMTokenManager, writer); store = mock(RMStateStore.class); ((RMContextImpl) rmContext).setStateStore(store); publisher = mock(SystemMetricsPublisher.class); ((RMContextImpl) rmContext).setSystemMetricsPublisher(publisher); scheduler = mock(YarnScheduler.class); masterService = mock(ApplicationMasterService.class); applicationMasterLauncher = mock(ApplicationMasterLauncher.class); rmDispatcher.register(RMAppAttemptEventType.class, new TestApplicationAttemptEventDispatcher()); rmDispatcher.register(RMAppEventType.class, new TestApplicationEventDispatcher()); rmDispatcher.register(SchedulerEventType.class, new TestSchedulerEventDispatcher()); rmDispatcher.register(AMLauncherEventType.class, new TestAMLauncherEventDispatcher()); rmnodeEventHandler = mock(RMNodeImpl.class); rmDispatcher.register(RMNodeEventType.class, rmnodeEventHandler); rmDispatcher.init(conf); rmDispatcher.start(); ApplicationId applicationId = MockApps.newAppID(appId++); ApplicationAttemptId applicationAttemptId = ApplicationAttemptId.newInstance(applicationId, 0); resourceScheduler = mock(ResourceScheduler.class); ApplicationResourceUsageReport appResUsgRpt = mock(ApplicationResourceUsageReport.class); when(appResUsgRpt.getMemorySeconds()).thenReturn(0L); when(appResUsgRpt.getVcoreSeconds()).thenReturn(0L); when(appResUsgRpt.getGcoreSeconds()).thenReturn(0L); when(resourceScheduler .getAppResourceUsageReport((ApplicationAttemptId)Matchers.any())) .thenReturn(appResUsgRpt); spyRMContext = spy(rmContext); Mockito.doReturn(resourceScheduler).when(spyRMContext).getScheduler(); final String user = MockApps.newUserName(); final String queue = MockApps.newQueue(); submissionContext = mock(ApplicationSubmissionContext.class); when(submissionContext.getQueue()).thenReturn(queue); Resource resource = BuilderUtils.newResource(1536, 1, 1); ContainerLaunchContext amContainerSpec = BuilderUtils.newContainerLaunchContext(null, null, null, null, null, null); when(submissionContext.getAMContainerSpec()).thenReturn(amContainerSpec); when(submissionContext.getResource()).thenReturn(resource); unmanagedAM = false; application = mock(RMAppImpl.class); applicationAttempt = new RMAppAttemptImpl(applicationAttemptId, spyRMContext, scheduler, masterService, submissionContext, new Configuration(), false, BuilderUtils.newResourceRequest( RMAppAttemptImpl.AM_CONTAINER_PRIORITY, ResourceRequest.ANY, submissionContext.getResource(), 1)); when(application.getCurrentAppAttempt()).thenReturn(applicationAttempt); when(application.getApplicationId()).thenReturn(applicationId); spyRMContext.getRMApps().put(application.getApplicationId(), application); testAppAttemptNewState(); }
Example #9
Source File: TestAMRMClientOnRMRestart.java From hadoop with Apache License 2.0 | 4 votes |
@Override protected ApplicationMasterService createApplicationMasterService() { return new ApplicationMasterService(getRMContext(), scheduler); }
Example #10
Source File: RMAppImpl.java From big-c with Apache License 2.0 | 4 votes |
public RMAppImpl(ApplicationId applicationId, RMContext rmContext, Configuration config, String name, String user, String queue, ApplicationSubmissionContext submissionContext, YarnScheduler scheduler, ApplicationMasterService masterService, long submitTime, String applicationType, Set<String> applicationTags, ResourceRequest amReq) { this.systemClock = new SystemClock(); this.applicationId = applicationId; this.name = name; this.rmContext = rmContext; this.dispatcher = rmContext.getDispatcher(); this.handler = dispatcher.getEventHandler(); this.conf = config; this.user = user; this.queue = queue; this.submissionContext = submissionContext; this.scheduler = scheduler; this.masterService = masterService; this.submitTime = submitTime; this.startTime = this.systemClock.getTime(); this.applicationType = applicationType; this.applicationTags = applicationTags; this.amReq = amReq; int globalMaxAppAttempts = conf.getInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS); int individualMaxAppAttempts = submissionContext.getMaxAppAttempts(); if (individualMaxAppAttempts <= 0 || individualMaxAppAttempts > globalMaxAppAttempts) { this.maxAppAttempts = globalMaxAppAttempts; LOG.warn("The specific max attempts: " + individualMaxAppAttempts + " for application: " + applicationId.getId() + " is invalid, because it is out of the range [1, " + globalMaxAppAttempts + "]. Use the global max attempts instead."); } else { this.maxAppAttempts = individualMaxAppAttempts; } this.attemptFailuresValidityInterval = submissionContext.getAttemptFailuresValidityInterval(); if (this.attemptFailuresValidityInterval > 0) { LOG.info("The attemptFailuresValidityInterval for the application: " + this.applicationId + " is " + this.attemptFailuresValidityInterval + "."); } ReentrantReadWriteLock lock = new ReentrantReadWriteLock(); this.readLock = lock.readLock(); this.writeLock = lock.writeLock(); this.stateMachine = stateMachineFactory.make(this); rmContext.getRMApplicationHistoryWriter().applicationStarted(this); rmContext.getSystemMetricsPublisher().appCreated(this, startTime); }
Example #11
Source File: TestRMAppAttemptTransitions.java From big-c with Apache License 2.0 | 4 votes |
@SuppressWarnings("deprecation") @Before public void setUp() throws Exception { AuthenticationMethod authMethod = AuthenticationMethod.SIMPLE; if (isSecurityEnabled) { authMethod = AuthenticationMethod.KERBEROS; } SecurityUtil.setAuthenticationMethod(authMethod, conf); UserGroupInformation.setConfiguration(conf); InlineDispatcher rmDispatcher = new InlineDispatcher(); ContainerAllocationExpirer containerAllocationExpirer = mock(ContainerAllocationExpirer.class); amLivelinessMonitor = mock(AMLivelinessMonitor.class); amFinishingMonitor = mock(AMLivelinessMonitor.class); writer = mock(RMApplicationHistoryWriter.class); MasterKeyData masterKeyData = amRMTokenManager.createNewMasterKey(); when(amRMTokenManager.getMasterKey()).thenReturn(masterKeyData); rmContext = new RMContextImpl(rmDispatcher, containerAllocationExpirer, amLivelinessMonitor, amFinishingMonitor, null, amRMTokenManager, new RMContainerTokenSecretManager(conf), nmTokenManager, clientToAMTokenManager, writer); store = mock(RMStateStore.class); ((RMContextImpl) rmContext).setStateStore(store); publisher = mock(SystemMetricsPublisher.class); ((RMContextImpl) rmContext).setSystemMetricsPublisher(publisher); scheduler = mock(YarnScheduler.class); masterService = mock(ApplicationMasterService.class); applicationMasterLauncher = mock(ApplicationMasterLauncher.class); rmDispatcher.register(RMAppAttemptEventType.class, new TestApplicationAttemptEventDispatcher()); rmDispatcher.register(RMAppEventType.class, new TestApplicationEventDispatcher()); rmDispatcher.register(SchedulerEventType.class, new TestSchedulerEventDispatcher()); rmDispatcher.register(AMLauncherEventType.class, new TestAMLauncherEventDispatcher()); rmnodeEventHandler = mock(RMNodeImpl.class); rmDispatcher.register(RMNodeEventType.class, rmnodeEventHandler); rmDispatcher.init(conf); rmDispatcher.start(); ApplicationId applicationId = MockApps.newAppID(appId++); ApplicationAttemptId applicationAttemptId = ApplicationAttemptId.newInstance(applicationId, 0); resourceScheduler = mock(ResourceScheduler.class); ApplicationResourceUsageReport appResUsgRpt = mock(ApplicationResourceUsageReport.class); when(appResUsgRpt.getMemorySeconds()).thenReturn(0L); when(appResUsgRpt.getVcoreSeconds()).thenReturn(0L); when(resourceScheduler .getAppResourceUsageReport((ApplicationAttemptId)Matchers.any())) .thenReturn(appResUsgRpt); spyRMContext = spy(rmContext); Mockito.doReturn(resourceScheduler).when(spyRMContext).getScheduler(); final String user = MockApps.newUserName(); final String queue = MockApps.newQueue(); submissionContext = mock(ApplicationSubmissionContext.class); when(submissionContext.getQueue()).thenReturn(queue); Resource resource = BuilderUtils.newResource(1536, 1); ContainerLaunchContext amContainerSpec = BuilderUtils.newContainerLaunchContext(null, null, null, null, null, null); when(submissionContext.getAMContainerSpec()).thenReturn(amContainerSpec); when(submissionContext.getResource()).thenReturn(resource); unmanagedAM = false; application = mock(RMAppImpl.class); applicationAttempt = new RMAppAttemptImpl(applicationAttemptId, spyRMContext, scheduler, masterService, submissionContext, new Configuration(), false, BuilderUtils.newResourceRequest( RMAppAttemptImpl.AM_CONTAINER_PRIORITY, ResourceRequest.ANY, submissionContext.getResource(), 1)); when(application.getCurrentAppAttempt()).thenReturn(applicationAttempt); when(application.getApplicationId()).thenReturn(applicationId); spyRMContext.getRMApps().put(application.getApplicationId(), application); testAppAttemptNewState(); }
Example #12
Source File: TestAMRMClientOnRMRestart.java From big-c with Apache License 2.0 | 4 votes |
@Override protected ApplicationMasterService createApplicationMasterService() { return new ApplicationMasterService(getRMContext(), scheduler); }
Example #13
Source File: MockRMContext.java From incubator-myriad with Apache License 2.0 | 4 votes |
public void setApplicationMasterService(ApplicationMasterService applicationMasterService) { this.applicationMasterService = applicationMasterService; }
Example #14
Source File: MockRMContext.java From incubator-myriad with Apache License 2.0 | 4 votes |
@Override public ApplicationMasterService getApplicationMasterService() { return applicationMasterService; }