Java Code Examples for org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext#setApplicationId()
The following examples show how to use
org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext#setApplicationId() .
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: TestApplicationClientProtocolOnHA.java From hadoop with Apache License 2.0 | 6 votes |
@Test(timeout = 15000) public void testSubmitApplicationOnHA() throws Exception { ApplicationSubmissionContext appContext = Records.newRecord(ApplicationSubmissionContext.class); appContext.setApplicationId(cluster.createFakeAppId()); ContainerLaunchContext amContainer = Records.newRecord(ContainerLaunchContext.class); appContext.setAMContainerSpec(amContainer); Resource capability = Records.newRecord(Resource.class); capability.setMemory(10); capability.setVirtualCores(1); capability.setGpuCores(1); appContext.setResource(capability); ApplicationId appId = client.submitApplication(appContext); Assert.assertTrue(getActiveRM().getRMContext().getRMApps() .containsKey(appId)); }
Example 2
Source File: YarnRemoteInterpreterProcess.java From zeppelin with Apache License 2.0 | 6 votes |
private ApplicationSubmissionContext createApplicationSubmissionContext( ApplicationSubmissionContext appContext) throws Exception { setResources(appContext); setPriority(appContext); setQueue(appContext); appContext.setApplicationId(appId); setApplicationName(appContext); appContext.setApplicationType("ZEPPELIN INTERPRETER"); appContext.setMaxAppAttempts(1); ContainerLaunchContext amContainer = setUpAMLaunchContext(); appContext.setAMContainerSpec(amContainer); appContext.setCancelTokensWhenComplete(true); return appContext; }
Example 3
Source File: RMStateStoreTestBase.java From big-c with Apache License 2.0 | 6 votes |
protected RMApp storeApp(RMStateStore store, ApplicationId appId, long submitTime, long startTime) throws Exception { ApplicationSubmissionContext context = new ApplicationSubmissionContextPBImpl(); context.setApplicationId(appId); RMApp mockApp = mock(RMApp.class); when(mockApp.getApplicationId()).thenReturn(appId); when(mockApp.getSubmitTime()).thenReturn(submitTime); when(mockApp.getStartTime()).thenReturn(startTime); when(mockApp.getApplicationSubmissionContext()).thenReturn(context); when(mockApp.getUser()).thenReturn("test"); store.storeNewApplication(mockApp); return mockApp; }
Example 4
Source File: BuilderUtils.java From big-c with Apache License 2.0 | 6 votes |
public static ApplicationSubmissionContext newApplicationSubmissionContext( ApplicationId applicationId, String applicationName, String queue, Priority priority, ContainerLaunchContext amContainer, boolean isUnmanagedAM, boolean cancelTokensWhenComplete, int maxAppAttempts, Resource resource, String applicationType) { ApplicationSubmissionContext context = recordFactory.newRecordInstance(ApplicationSubmissionContext.class); context.setApplicationId(applicationId); context.setApplicationName(applicationName); context.setQueue(queue); context.setPriority(priority); context.setAMContainerSpec(amContainer); context.setUnmanagedAM(isUnmanagedAM); context.setCancelTokensWhenComplete(cancelTokensWhenComplete); context.setMaxAppAttempts(maxAppAttempts); context.setResource(resource); context.setApplicationType(applicationType); return context; }
Example 5
Source File: AthenaXYarnClusterDescriptor.java From AthenaX with Apache License 2.0 | 6 votes |
ClusterClient<ApplicationId> deploy() { ApplicationSubmissionContext context = Records.newRecord(ApplicationSubmissionContext.class); context.setApplicationId(job.yarnAppId()); ApplicationReport report; try { report = startAppMaster(context); Configuration conf = getFlinkConfiguration(); conf.setString(JobManagerOptions.ADDRESS.key(), report.getHost()); conf.setInteger(JobManagerOptions.PORT.key(), report.getRpcPort()); return createYarnClusterClient(this, (int) job.taskManagerCount(), (int) job.slotCountPerTaskManager(), report, conf, false); } catch (Exception e) { throw new RuntimeException(e); } }
Example 6
Source File: TestApplicationClientProtocolOnHA.java From big-c with Apache License 2.0 | 6 votes |
@Test(timeout = 15000) public void testSubmitApplicationOnHA() throws Exception { ApplicationSubmissionContext appContext = Records.newRecord(ApplicationSubmissionContext.class); appContext.setApplicationId(cluster.createFakeAppId()); ContainerLaunchContext amContainer = Records.newRecord(ContainerLaunchContext.class); appContext.setAMContainerSpec(amContainer); Resource capability = Records.newRecord(Resource.class); capability.setMemory(10); capability.setVirtualCores(1); appContext.setResource(capability); ApplicationId appId = client.submitApplication(appContext); Assert.assertTrue(getActiveRM().getRMContext().getRMApps() .containsKey(appId)); }
Example 7
Source File: RMStateStoreTestBase.java From hadoop with Apache License 2.0 | 6 votes |
protected RMApp storeApp(RMStateStore store, ApplicationId appId, long submitTime, long startTime) throws Exception { ApplicationSubmissionContext context = new ApplicationSubmissionContextPBImpl(); context.setApplicationId(appId); RMApp mockApp = mock(RMApp.class); when(mockApp.getApplicationId()).thenReturn(appId); when(mockApp.getSubmitTime()).thenReturn(submitTime); when(mockApp.getStartTime()).thenReturn(startTime); when(mockApp.getApplicationSubmissionContext()).thenReturn(context); when(mockApp.getUser()).thenReturn("test"); store.storeNewApplication(mockApp); return mockApp; }
Example 8
Source File: BuilderUtils.java From hadoop with Apache License 2.0 | 6 votes |
public static ApplicationSubmissionContext newApplicationSubmissionContext( ApplicationId applicationId, String applicationName, String queue, Priority priority, ContainerLaunchContext amContainer, boolean isUnmanagedAM, boolean cancelTokensWhenComplete, int maxAppAttempts, Resource resource, String applicationType) { ApplicationSubmissionContext context = recordFactory.newRecordInstance(ApplicationSubmissionContext.class); context.setApplicationId(applicationId); context.setApplicationName(applicationName); context.setQueue(queue); context.setPriority(priority); context.setAMContainerSpec(amContainer); context.setUnmanagedAM(isUnmanagedAM); context.setCancelTokensWhenComplete(cancelTokensWhenComplete); context.setMaxAppAttempts(maxAppAttempts); context.setResource(resource); context.setApplicationType(applicationType); return context; }
Example 9
Source File: LocalClient.java From tez with Apache License 2.0 | 5 votes |
@Override public YarnClientApplication createApplication() throws YarnException, IOException { ApplicationSubmissionContext context = Records.newRecord(ApplicationSubmissionContext.class); ApplicationId appId = ApplicationId.newInstance(clusterTimeStamp, appIdNumber++); context.setApplicationId(appId); GetNewApplicationResponse response = Records.newRecord(GetNewApplicationResponse.class); response.setApplicationId(appId); return new YarnClientApplication(response, context); }
Example 10
Source File: YarnClientImpl.java From big-c with Apache License 2.0 | 5 votes |
@Override public YarnClientApplication createApplication() throws YarnException, IOException { ApplicationSubmissionContext context = Records.newRecord (ApplicationSubmissionContext.class); GetNewApplicationResponse newApp = getNewApplication(); ApplicationId appId = newApp.getApplicationId(); context.setApplicationId(appId); return new YarnClientApplication(newApp, context); }
Example 11
Source File: TestRMAppTransitions.java From hadoop with Apache License 2.0 | 5 votes |
protected RMApp createNewTestApp(ApplicationSubmissionContext submissionContext) { ApplicationId applicationId = MockApps.newAppID(appId++); String user = MockApps.newUserName(); String name = MockApps.newAppName(); String queue = MockApps.newQueue(); // ensure max application attempts set to known value conf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, maxAppAttempts); scheduler = mock(YarnScheduler.class); ApplicationMasterService masterService = new ApplicationMasterService(rmContext, scheduler); if(submissionContext == null) { submissionContext = new ApplicationSubmissionContextPBImpl(); } // applicationId will not be used because RMStateStore is mocked, // but applicationId is still set for safety submissionContext.setApplicationId(applicationId); RMApp application = new RMAppImpl(applicationId, rmContext, conf, name, user, queue, submissionContext, scheduler, masterService, System.currentTimeMillis(), "YARN", null, null); testAppStartState(applicationId, user, name, queue, application); this.rmContext.getRMApps().putIfAbsent(application.getApplicationId(), application); return application; }
Example 12
Source File: TestYarnClient.java From hadoop with Apache License 2.0 | 5 votes |
private ApplicationId createApp(YarnClient rmClient, boolean unmanaged) throws Exception { YarnClientApplication newApp = rmClient.createApplication(); ApplicationId appId = newApp.getNewApplicationResponse().getApplicationId(); // Create launch context for app master ApplicationSubmissionContext appContext = Records.newRecord(ApplicationSubmissionContext.class); // set the application id appContext.setApplicationId(appId); // set the application name appContext.setApplicationName("test"); // Set the priority for the application master Priority pri = Records.newRecord(Priority.class); pri.setPriority(1); appContext.setPriority(pri); // Set the queue to which this application is to be submitted in the RM appContext.setQueue("default"); // Set up the container launch context for the application master ContainerLaunchContext amContainer = Records.newRecord(ContainerLaunchContext.class); appContext.setAMContainerSpec(amContainer); appContext.setResource(Resource.newInstance(1024, 1)); appContext.setUnmanagedAM(unmanaged); // Submit the application to the applications manager rmClient.submitApplication(appContext); return appId; }
Example 13
Source File: TestApplicationACLs.java From hadoop with Apache License 2.0 | 5 votes |
@SuppressWarnings("deprecation") private ApplicationId submitAppAndGetAppId(AccessControlList viewACL, AccessControlList modifyACL) throws Exception { SubmitApplicationRequest submitRequest = recordFactory .newRecordInstance(SubmitApplicationRequest.class); ApplicationSubmissionContext context = recordFactory .newRecordInstance(ApplicationSubmissionContext.class); ApplicationId applicationId = rmClient.getNewApplication( recordFactory.newRecordInstance(GetNewApplicationRequest.class)) .getApplicationId(); context.setApplicationId(applicationId); Map<ApplicationAccessType, String> acls = new HashMap<ApplicationAccessType, String>(); acls.put(ApplicationAccessType.VIEW_APP, viewACL.getAclString()); acls.put(ApplicationAccessType.MODIFY_APP, modifyACL.getAclString()); ContainerLaunchContext amContainer = recordFactory .newRecordInstance(ContainerLaunchContext.class); Resource resource = BuilderUtils.newResource(1024, 1); context.setResource(resource); amContainer.setApplicationACLs(acls); context.setAMContainerSpec(amContainer); submitRequest.setApplicationSubmissionContext(context); rmClient.submitApplication(submitRequest); resourceManager.waitForState(applicationId, RMAppState.ACCEPTED); return applicationId; }
Example 14
Source File: TestApplicationACLs.java From big-c with Apache License 2.0 | 5 votes |
@SuppressWarnings("deprecation") private ApplicationId submitAppAndGetAppId(AccessControlList viewACL, AccessControlList modifyACL) throws Exception { SubmitApplicationRequest submitRequest = recordFactory .newRecordInstance(SubmitApplicationRequest.class); ApplicationSubmissionContext context = recordFactory .newRecordInstance(ApplicationSubmissionContext.class); ApplicationId applicationId = rmClient.getNewApplication( recordFactory.newRecordInstance(GetNewApplicationRequest.class)) .getApplicationId(); context.setApplicationId(applicationId); Map<ApplicationAccessType, String> acls = new HashMap<ApplicationAccessType, String>(); acls.put(ApplicationAccessType.VIEW_APP, viewACL.getAclString()); acls.put(ApplicationAccessType.MODIFY_APP, modifyACL.getAclString()); ContainerLaunchContext amContainer = recordFactory .newRecordInstance(ContainerLaunchContext.class); Resource resource = BuilderUtils.newResource(1024, 1); context.setResource(resource); amContainer.setApplicationACLs(acls); context.setAMContainerSpec(amContainer); submitRequest.setApplicationSubmissionContext(context); rmClient.submitApplication(submitRequest); resourceManager.waitForState(applicationId, RMAppState.ACCEPTED); return applicationId; }
Example 15
Source File: TestYarnClient.java From big-c with Apache License 2.0 | 5 votes |
private ApplicationId createApp(YarnClient rmClient, boolean unmanaged) throws Exception { YarnClientApplication newApp = rmClient.createApplication(); ApplicationId appId = newApp.getNewApplicationResponse().getApplicationId(); // Create launch context for app master ApplicationSubmissionContext appContext = Records.newRecord(ApplicationSubmissionContext.class); // set the application id appContext.setApplicationId(appId); // set the application name appContext.setApplicationName("test"); // Set the priority for the application master Priority pri = Records.newRecord(Priority.class); pri.setPriority(1); appContext.setPriority(pri); // Set the queue to which this application is to be submitted in the RM appContext.setQueue("default"); // Set up the container launch context for the application master ContainerLaunchContext amContainer = Records.newRecord(ContainerLaunchContext.class); appContext.setAMContainerSpec(amContainer); appContext.setResource(Resource.newInstance(1024, 1)); appContext.setUnmanagedAM(unmanaged); // Submit the application to the applications manager rmClient.submitApplication(appContext); return appId; }
Example 16
Source File: QueueACLsTestBase.java From hadoop with Apache License 2.0 | 5 votes |
private ApplicationId submitAppAndGetAppId(String submitter, String queueName, boolean setupACLs) throws Exception { GetNewApplicationRequest newAppRequest = GetNewApplicationRequest.newInstance(); ApplicationClientProtocol submitterClient = getRMClientForUser(submitter); ApplicationId applicationId = submitterClient.getNewApplication(newAppRequest).getApplicationId(); Resource resource = BuilderUtils.newResource(1024, 1); Map<ApplicationAccessType, String> acls = createACLs(submitter, setupACLs); ContainerLaunchContext amContainerSpec = ContainerLaunchContext.newInstance(null, null, null, null, null, acls); ApplicationSubmissionContext appSubmissionContext = ApplicationSubmissionContext.newInstance(applicationId, "applicationName", queueName, null, amContainerSpec, false, true, 1, resource, "applicationType"); appSubmissionContext.setApplicationId(applicationId); appSubmissionContext.setQueue(queueName); SubmitApplicationRequest submitRequest = SubmitApplicationRequest.newInstance(appSubmissionContext); submitterClient.submitApplication(submitRequest); resourceManager.waitForState(applicationId, RMAppState.ACCEPTED); return applicationId; }
Example 17
Source File: AMSimulator.java From big-c with Apache License 2.0 | 4 votes |
private void submitApp() throws YarnException, InterruptedException, IOException { // ask for new application GetNewApplicationRequest newAppRequest = Records.newRecord(GetNewApplicationRequest.class); GetNewApplicationResponse newAppResponse = rm.getClientRMService().getNewApplication(newAppRequest); appId = newAppResponse.getApplicationId(); // submit the application final SubmitApplicationRequest subAppRequest = Records.newRecord(SubmitApplicationRequest.class); ApplicationSubmissionContext appSubContext = Records.newRecord(ApplicationSubmissionContext.class); appSubContext.setApplicationId(appId); appSubContext.setMaxAppAttempts(1); appSubContext.setQueue(queue); appSubContext.setPriority(Priority.newInstance(0)); ContainerLaunchContext conLauContext = Records.newRecord(ContainerLaunchContext.class); conLauContext.setApplicationACLs( new HashMap<ApplicationAccessType, String>()); conLauContext.setCommands(new ArrayList<String>()); conLauContext.setEnvironment(new HashMap<String, String>()); conLauContext.setLocalResources(new HashMap<String, LocalResource>()); conLauContext.setServiceData(new HashMap<String, ByteBuffer>()); appSubContext.setAMContainerSpec(conLauContext); appSubContext.setUnmanagedAM(true); subAppRequest.setApplicationSubmissionContext(appSubContext); UserGroupInformation ugi = UserGroupInformation.createRemoteUser(user); ugi.doAs(new PrivilegedExceptionAction<Object>() { @Override public Object run() throws YarnException { rm.getClientRMService().submitApplication(subAppRequest); return null; } }); LOG.info(MessageFormat.format("Submit a new application {0}", appId)); // waiting until application ACCEPTED RMApp app = rm.getRMContext().getRMApps().get(appId); while(app.getState() != RMAppState.ACCEPTED) { Thread.sleep(10); } // Waiting until application attempt reach LAUNCHED // "Unmanaged AM must register after AM attempt reaches LAUNCHED state" this.appAttemptId = rm.getRMContext().getRMApps().get(appId) .getCurrentAppAttempt().getAppAttemptId(); RMAppAttempt rmAppAttempt = rm.getRMContext().getRMApps().get(appId) .getCurrentAppAttempt(); while (rmAppAttempt.getAppAttemptState() != RMAppAttemptState.LAUNCHED) { Thread.sleep(10); } }
Example 18
Source File: TestYarnClient.java From hadoop with Apache License 2.0 | 4 votes |
@Test (timeout = 30000) public void testSubmitIncorrectQueue() throws IOException { MiniYARNCluster cluster = new MiniYARNCluster("testMRAMTokens", 1, 1, 1); YarnClient rmClient = null; try { cluster.init(new YarnConfiguration()); cluster.start(); final Configuration yarnConf = cluster.getConfig(); rmClient = YarnClient.createYarnClient(); rmClient.init(yarnConf); rmClient.start(); YarnClientApplication newApp = rmClient.createApplication(); ApplicationId appId = newApp.getNewApplicationResponse().getApplicationId(); // Create launch context for app master ApplicationSubmissionContext appContext = Records.newRecord(ApplicationSubmissionContext.class); // set the application id appContext.setApplicationId(appId); // set the application name appContext.setApplicationName("test"); // Set the queue to which this application is to be submitted in the RM appContext.setQueue("nonexist"); // Set up the container launch context for the application master ContainerLaunchContext amContainer = Records.newRecord(ContainerLaunchContext.class); appContext.setAMContainerSpec(amContainer); appContext.setResource(Resource.newInstance(1024, 1)); // appContext.setUnmanagedAM(unmanaged); // Submit the application to the applications manager rmClient.submitApplication(appContext); Assert.fail("Job submission should have thrown an exception"); } catch (YarnException e) { Assert.assertTrue(e.getMessage().contains("Failed to submit")); } finally { if (rmClient != null) { rmClient.stop(); } cluster.stop(); } }
Example 19
Source File: AMSimulator.java From hadoop with Apache License 2.0 | 4 votes |
private void submitApp() throws YarnException, InterruptedException, IOException { // ask for new application GetNewApplicationRequest newAppRequest = Records.newRecord(GetNewApplicationRequest.class); GetNewApplicationResponse newAppResponse = rm.getClientRMService().getNewApplication(newAppRequest); appId = newAppResponse.getApplicationId(); // submit the application final SubmitApplicationRequest subAppRequest = Records.newRecord(SubmitApplicationRequest.class); ApplicationSubmissionContext appSubContext = Records.newRecord(ApplicationSubmissionContext.class); appSubContext.setApplicationId(appId); appSubContext.setMaxAppAttempts(1); appSubContext.setQueue(queue); appSubContext.setPriority(Priority.newInstance(0)); ContainerLaunchContext conLauContext = Records.newRecord(ContainerLaunchContext.class); conLauContext.setApplicationACLs( new HashMap<ApplicationAccessType, String>()); conLauContext.setCommands(new ArrayList<String>()); conLauContext.setEnvironment(new HashMap<String, String>()); conLauContext.setLocalResources(new HashMap<String, LocalResource>()); conLauContext.setServiceData(new HashMap<String, ByteBuffer>()); appSubContext.setAMContainerSpec(conLauContext); appSubContext.setUnmanagedAM(true); subAppRequest.setApplicationSubmissionContext(appSubContext); UserGroupInformation ugi = UserGroupInformation.createRemoteUser(user); ugi.doAs(new PrivilegedExceptionAction<Object>() { @Override public Object run() throws YarnException { rm.getClientRMService().submitApplication(subAppRequest); return null; } }); LOG.info(MessageFormat.format("Submit a new application {0}", appId)); // waiting until application ACCEPTED RMApp app = rm.getRMContext().getRMApps().get(appId); while(app.getState() != RMAppState.ACCEPTED) { Thread.sleep(10); } // Waiting until application attempt reach LAUNCHED // "Unmanaged AM must register after AM attempt reaches LAUNCHED state" this.appAttemptId = rm.getRMContext().getRMApps().get(appId) .getCurrentAppAttempt().getAppAttemptId(); RMAppAttempt rmAppAttempt = rm.getRMContext().getRMApps().get(appId) .getCurrentAppAttempt(); while (rmAppAttempt.getAppAttemptState() != RMAppAttemptState.LAUNCHED) { Thread.sleep(10); } }
Example 20
Source File: InlineAM.java From attic-apex-core with Apache License 2.0 | 4 votes |
public boolean run() throws Exception { LOG.info("Starting Client"); // Connect to ResourceManager rmClient.start(); try { // Get a new application id YarnClientApplication newApp = rmClient.createApplication(); ApplicationId appId = newApp.getNewApplicationResponse().getApplicationId(); // Create launch context for app master LOG.info("Setting up application submission context for ASM"); ApplicationSubmissionContext appContext = Records .newRecord(ApplicationSubmissionContext.class); // set the application id appContext.setApplicationId(appId); // set the application name appContext.setApplicationName(appName); // Set the priority for the application master Priority pri = Records.newRecord(Priority.class); pri.setPriority(amPriority); appContext.setPriority(pri); // Set the queue to which this application is to be submitted in the RM appContext.setQueue(amQueue); // Set up the container launch context for the application master ContainerLaunchContext amContainer = Records .newRecord(ContainerLaunchContext.class); appContext.setAMContainerSpec(amContainer); // unmanaged AM appContext.setUnmanagedAM(true); LOG.info("Setting unmanaged AM"); // Submit the application to the applications manager LOG.info("Submitting application to ASM"); rmClient.submitApplication(appContext); // Monitor the application to wait for launch state ApplicationReport appReport = monitorApplication(appId, EnumSet.of(YarnApplicationState.ACCEPTED)); ApplicationAttemptId attemptId = appReport.getCurrentApplicationAttemptId(); LOG.info("Launching application with id: " + attemptId); // launch AM runAM(attemptId); // Monitor the application for end state appReport = monitorApplication(appId, EnumSet.of( YarnApplicationState.KILLED, YarnApplicationState.FAILED, YarnApplicationState.FINISHED)); YarnApplicationState appState = appReport.getYarnApplicationState(); FinalApplicationStatus appStatus = appReport.getFinalApplicationStatus(); LOG.info("App ended with state: " + appReport.getYarnApplicationState() + " and status: " + appStatus); boolean success; if (YarnApplicationState.FINISHED == appState && FinalApplicationStatus.SUCCEEDED == appStatus) { LOG.info("Application has completed successfully."); success = true; } else { LOG.info("Application did finished unsuccessfully." + " YarnState=" + appState.toString() + ", FinalStatus=" + appStatus.toString()); success = false; } return success; } finally { rmClient.stop(); } }