Java Code Examples for org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext#setApplicationTags()
The following examples show how to use
org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext#setApplicationTags() .
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: TestClientRMService.java From hadoop with Apache License 2.0 | 6 votes |
@SuppressWarnings("deprecation") private SubmitApplicationRequest mockSubmitAppRequest(ApplicationId appId, String name, String queue, Set<String> tags, boolean unmanaged) { ContainerLaunchContext amContainerSpec = mock(ContainerLaunchContext.class); Resource resource = Resources.createResource( YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB); ApplicationSubmissionContext submissionContext = recordFactory.newRecordInstance(ApplicationSubmissionContext.class); submissionContext.setAMContainerSpec(amContainerSpec); submissionContext.setApplicationName(name); submissionContext.setQueue(queue); submissionContext.setApplicationId(appId); submissionContext.setResource(resource); submissionContext.setApplicationType(appType); submissionContext.setApplicationTags(tags); submissionContext.setUnmanagedAM(unmanaged); SubmitApplicationRequest submitRequest = recordFactory.newRecordInstance(SubmitApplicationRequest.class); submitRequest.setApplicationSubmissionContext(submissionContext); return submitRequest; }
Example 2
Source File: TestClientRMService.java From big-c with Apache License 2.0 | 6 votes |
@SuppressWarnings("deprecation") private SubmitApplicationRequest mockSubmitAppRequest(ApplicationId appId, String name, String queue, Set<String> tags, boolean unmanaged) { ContainerLaunchContext amContainerSpec = mock(ContainerLaunchContext.class); Resource resource = Resources.createResource( YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB); ApplicationSubmissionContext submissionContext = recordFactory.newRecordInstance(ApplicationSubmissionContext.class); submissionContext.setAMContainerSpec(amContainerSpec); submissionContext.setApplicationName(name); submissionContext.setQueue(queue); submissionContext.setApplicationId(appId); submissionContext.setResource(resource); submissionContext.setApplicationType(appType); submissionContext.setApplicationTags(tags); submissionContext.setUnmanagedAM(unmanaged); SubmitApplicationRequest submitRequest = recordFactory.newRecordInstance(SubmitApplicationRequest.class); submitRequest.setApplicationSubmissionContext(submissionContext); return submitRequest; }
Example 3
Source File: SylphSparkYarnClient.java From sylph with Apache License 2.0 | 5 votes |
@Override public ApplicationSubmissionContext createApplicationSubmissionContext(YarnClientApplication newApp, ContainerLaunchContext containerContext) { final ApplicationSubmissionContext appContext = super.createApplicationSubmissionContext(newApp, containerContext); appContext.setApplicationType("Sylph_SPARK"); appContext.setApplicationTags(ImmutableSet.of("a1", "a2")); appContext.setQueue(yarnQueue); appContext.setMaxAppAttempts(2); return appContext; }
Example 4
Source File: LaunchCluster.java From TensorFlowOnYARN with Apache License 2.0 | 5 votes |
ApplicationId submitApplication( YarnClientApplication app, String appName, ContainerLaunchContext launchContext, Resource resource, String queue) throws Exception { ApplicationSubmissionContext appContext = app.getApplicationSubmissionContext(); appContext.setApplicationName(appName); appContext.setApplicationTags(new HashSet<>()); appContext.setAMContainerSpec(launchContext); appContext.setResource(resource); appContext.setQueue(queue); return yarnClient.submitApplication(appContext); }
Example 5
Source File: RMWebServices.java From hadoop with Apache License 2.0 | 5 votes |
/** * Create the actual ApplicationSubmissionContext to be submitted to the RM * from the information provided by the user. * * @param newApp * the information provided by the user * @return returns the constructed ApplicationSubmissionContext * @throws IOException */ protected ApplicationSubmissionContext createAppSubmissionContext( ApplicationSubmissionContextInfo newApp) throws IOException { // create local resources and app submission context ApplicationId appid; String error = "Could not parse application id " + newApp.getApplicationId(); try { appid = ConverterUtils.toApplicationId(recordFactory, newApp.getApplicationId()); } catch (Exception e) { throw new BadRequestException(error); } ApplicationSubmissionContext appContext = ApplicationSubmissionContext.newInstance(appid, newApp.getApplicationName(), newApp.getQueue(), Priority.newInstance(newApp.getPriority()), createContainerLaunchContext(newApp), newApp.getUnmanagedAM(), newApp.getCancelTokensWhenComplete(), newApp.getMaxAppAttempts(), createAppSubmissionContextResource(newApp), newApp.getApplicationType(), newApp.getKeepContainersAcrossApplicationAttempts(), newApp.getAppNodeLabelExpression(), newApp.getAMContainerNodeLabelExpression()); appContext.setApplicationTags(newApp.getApplicationTags()); return appContext; }
Example 6
Source File: RMWebServices.java From big-c with Apache License 2.0 | 5 votes |
/** * Create the actual ApplicationSubmissionContext to be submitted to the RM * from the information provided by the user. * * @param newApp * the information provided by the user * @return returns the constructed ApplicationSubmissionContext * @throws IOException */ protected ApplicationSubmissionContext createAppSubmissionContext( ApplicationSubmissionContextInfo newApp) throws IOException { // create local resources and app submission context ApplicationId appid; String error = "Could not parse application id " + newApp.getApplicationId(); try { appid = ConverterUtils.toApplicationId(recordFactory, newApp.getApplicationId()); } catch (Exception e) { throw new BadRequestException(error); } ApplicationSubmissionContext appContext = ApplicationSubmissionContext.newInstance(appid, newApp.getApplicationName(), newApp.getQueue(), Priority.newInstance(newApp.getPriority()), createContainerLaunchContext(newApp), newApp.getUnmanagedAM(), newApp.getCancelTokensWhenComplete(), newApp.getMaxAppAttempts(), createAppSubmissionContextResource(newApp), newApp.getApplicationType(), newApp.getKeepContainersAcrossApplicationAttempts(), newApp.getAppNodeLabelExpression(), newApp.getAMContainerNodeLabelExpression()); appContext.setApplicationTags(newApp.getApplicationTags()); return appContext; }
Example 7
Source File: TestTezClientUtils.java From tez with Apache License 2.0 | 5 votes |
@Test(timeout=1000) public void testSetApplicationTags() { TezConfiguration conf = new TezConfiguration(false); conf.set(TezConfiguration.TEZ_APPLICATION_TAGS, "foo,bar"); AMConfiguration amconfig = new AMConfiguration(conf, null, null); ApplicationSubmissionContext appContext = Records .newRecord(ApplicationSubmissionContext.class); Collection<String> tagsFromConf = amconfig.getTezConfiguration().getTrimmedStringCollection( TezConfiguration.TEZ_APPLICATION_TAGS); appContext.setApplicationTags(new HashSet<String>(tagsFromConf)); assertTrue(appContext.getApplicationTags().contains("foo")); assertTrue(appContext.getApplicationTags().contains("bar")); }
Example 8
Source File: AthenaXYarnClusterDescriptor.java From AthenaX with Apache License 2.0 | 4 votes |
private ApplicationReport startAppMaster(ApplicationSubmissionContext appContext) throws Exception { appContext.setMaxAppAttempts(MAX_ATTEMPT); Map<String, LocalResource> localResources = new HashMap<>(); Set<Path> shippedPaths = new HashSet<>(); collectLocalResources(localResources, shippedPaths); final ContainerLaunchContext amContainer = setupApplicationMasterContainer( this.getYarnSessionClusterEntrypoint(), false, true, false, (int) job.taskManagerMemoryMb()); amContainer.setLocalResources(localResources); final String classPath = localResources.keySet().stream().collect(Collectors.joining(File.pathSeparator)); final String shippedFiles = shippedPaths.stream().map(x -> x.getName() + "=" + x.toString()) .collect(Collectors.joining(",")); // Setup CLASSPATH and environment variables for ApplicationMaster ApplicationId appId = appContext.getApplicationId(); final Map<String, String> appMasterEnv = setUpAmEnvironment( appId, classPath, shippedFiles, getDynamicPropertiesEncoded() ); amContainer.setEnvironment(appMasterEnv); // Set up resource type requirements for ApplicationMaster Resource capability = Records.newRecord(Resource.class); capability.setMemory(getFlinkConfiguration() .getInteger(JobManagerOptions.JOB_MANAGER_HEAP_MEMORY)); capability.setVirtualCores(1); appContext.setApplicationName(job.name()); appContext.setApplicationType(ATHENAX_APPLICATION_TYPE); appContext.setAMContainerSpec(amContainer); appContext.setResource(capability); appContext.setApplicationTags(Collections.singleton(job.metadata().serialize())); if (job.queue() != null) { appContext.setQueue(job.queue()); } LOG.info("Submitting application master {}", appId); yarnClient.submitApplication(appContext); PollDeploymentStatus poll = new PollDeploymentStatus(appId); YARN_POLL_EXECUTOR.submit(poll); try { return poll.result.get(); } catch (ExecutionException e) { LOG.warn("Failed to deploy {}, cause: {}", appId.toString(), e.getCause()); yarnClient.killApplication(appId); throw (Exception) e.getCause(); } }