Java Code Examples for org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext#setMaxAppAttempts()

The following examples show how to use org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext#setMaxAppAttempts() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: BuilderUtils.java    From hadoop with Apache License 2.0 6 votes vote down vote up 
public static ApplicationSubmissionContext newApplicationSubmissionContext(
    ApplicationId applicationId, String applicationName, String queue,
    Priority priority, ContainerLaunchContext amContainer,
    boolean isUnmanagedAM, boolean cancelTokensWhenComplete,
    int maxAppAttempts, Resource resource, String applicationType) {
  ApplicationSubmissionContext context =
      recordFactory.newRecordInstance(ApplicationSubmissionContext.class);
  context.setApplicationId(applicationId);
  context.setApplicationName(applicationName);
  context.setQueue(queue);
  context.setPriority(priority);
  context.setAMContainerSpec(amContainer);
  context.setUnmanagedAM(isUnmanagedAM);
  context.setCancelTokensWhenComplete(cancelTokensWhenComplete);
  context.setMaxAppAttempts(maxAppAttempts);
  context.setResource(resource);
  context.setApplicationType(applicationType);
  return context;
}
Example 2
Source File: BuilderUtils.java    From big-c with Apache License 2.0 6 votes vote down vote up 
public static ApplicationSubmissionContext newApplicationSubmissionContext(
    ApplicationId applicationId, String applicationName, String queue,
    Priority priority, ContainerLaunchContext amContainer,
    boolean isUnmanagedAM, boolean cancelTokensWhenComplete,
    int maxAppAttempts, Resource resource, String applicationType) {
  ApplicationSubmissionContext context =
      recordFactory.newRecordInstance(ApplicationSubmissionContext.class);
  context.setApplicationId(applicationId);
  context.setApplicationName(applicationName);
  context.setQueue(queue);
  context.setPriority(priority);
  context.setAMContainerSpec(amContainer);
  context.setUnmanagedAM(isUnmanagedAM);
  context.setCancelTokensWhenComplete(cancelTokensWhenComplete);
  context.setMaxAppAttempts(maxAppAttempts);
  context.setResource(resource);
  context.setApplicationType(applicationType);
  return context;
}
Example 3
Source File: YarnRemoteInterpreterProcess.java    From zeppelin with Apache License 2.0 6 votes vote down vote up 
private ApplicationSubmissionContext createApplicationSubmissionContext(
        ApplicationSubmissionContext appContext) throws Exception {

  setResources(appContext);
  setPriority(appContext);
  setQueue(appContext);
  appContext.setApplicationId(appId);
  setApplicationName(appContext);
  appContext.setApplicationType("ZEPPELIN INTERPRETER");
  appContext.setMaxAppAttempts(1);

  ContainerLaunchContext amContainer = setUpAMLaunchContext();
  appContext.setAMContainerSpec(amContainer);
  appContext.setCancelTokensWhenComplete(true);
  return appContext;
}
Example 4
Source File: TezClient.java    From tez with Apache License 2.0 6 votes vote down vote up 
private ApplicationSubmissionContext setupApplicationContext() throws IOException, YarnException {
  TezClientUtils.processTezLocalCredentialsFile(sessionCredentials,
          amConfig.getTezConfiguration());

  Map<String, LocalResource> tezJarResources = getTezJarResources(sessionCredentials);
  // Add session token for shuffle
  TezClientUtils.createSessionToken(sessionAppId.toString(),
          jobTokenSecretManager, sessionCredentials);

  ApplicationSubmissionContext appContext =
          TezClientUtils.createApplicationSubmissionContext(
                  sessionAppId,
                  null, clientName, amConfig,
                  tezJarResources, sessionCredentials, usingTezArchiveDeploy, apiVersionInfo,
                  servicePluginsDescriptor, javaOptsChecker);

  // Set Tez Sessions to not retry on AM crashes if recovery is disabled
  if (!amConfig.getTezConfiguration().getBoolean(
          TezConfiguration.DAG_RECOVERY_ENABLED,
          TezConfiguration.DAG_RECOVERY_ENABLED_DEFAULT)) {
    appContext.setMaxAppAttempts(1);
  }
  return appContext;
}
Example 5
Source File: SylphSparkYarnClient.java    From sylph with Apache License 2.0 5 votes vote down vote up 
@Override
public ApplicationSubmissionContext createApplicationSubmissionContext(YarnClientApplication newApp, ContainerLaunchContext containerContext)
{
    final ApplicationSubmissionContext appContext = super.createApplicationSubmissionContext(newApp, containerContext);
    appContext.setApplicationType("Sylph_SPARK");
    appContext.setApplicationTags(ImmutableSet.of("a1", "a2"));
    appContext.setQueue(yarnQueue);
    appContext.setMaxAppAttempts(2);
    return appContext;
}
Example 6
Source File: DeployerApplicationYarnClient.java    From spring-cloud-deployer-yarn with Apache License 2.0 5 votes vote down vote up 
@Override
protected ApplicationSubmissionContext getSubmissionContext(ApplicationId applicationId) {
	ApplicationSubmissionContext submissionContext = super.getSubmissionContext(applicationId);
	// force app fail after first appmaster failure regardles
	// what's set on yarn level.
	// TODO: should make configurable in spring yarn if possible
	submissionContext.setMaxAppAttempts(1);
	return submissionContext;
}
Example 7
Source File: Hadoop21YarnAppClient.java    From twill with Apache License 2.0 5 votes vote down vote up 
/**
 * Updates the {@link ApplicationSubmissionContext} based on configuration.
 */
protected void configureAppSubmissionContext(ApplicationSubmissionContext context) {
  int maxAttempts = configuration.getInt(Configs.Keys.YARN_MAX_APP_ATTEMPTS, -1);
  if (maxAttempts > 0) {
    context.setMaxAppAttempts(maxAttempts);
  } else {
    // Preserve the old behavior
    context.setMaxAppAttempts(2);
  }
}
Example 8
Source File: AthenaXYarnClusterDescriptor.java    From AthenaX with Apache License 2.0 4 votes vote down vote up 
private ApplicationReport startAppMaster(ApplicationSubmissionContext appContext) throws Exception {
  appContext.setMaxAppAttempts(MAX_ATTEMPT);

  Map<String, LocalResource> localResources = new HashMap<>();
  Set<Path> shippedPaths = new HashSet<>();
  collectLocalResources(localResources, shippedPaths);
  final ContainerLaunchContext amContainer = setupApplicationMasterContainer(
      this.getYarnSessionClusterEntrypoint(),
      false,
      true,
      false, (int) job.taskManagerMemoryMb());

  amContainer.setLocalResources(localResources);

  final String classPath = localResources.keySet().stream().collect(Collectors.joining(File.pathSeparator));
  final String shippedFiles = shippedPaths.stream().map(x -> x.getName() + "=" + x.toString())
      .collect(Collectors.joining(","));

  // Setup CLASSPATH and environment variables for ApplicationMaster
  ApplicationId appId = appContext.getApplicationId();
  final Map<String, String> appMasterEnv = setUpAmEnvironment(
      appId,
      classPath,
      shippedFiles,
      getDynamicPropertiesEncoded()
  );

  amContainer.setEnvironment(appMasterEnv);

  // Set up resource type requirements for ApplicationMaster
  Resource capability = Records.newRecord(Resource.class);
  capability.setMemory(getFlinkConfiguration()
      .getInteger(JobManagerOptions.JOB_MANAGER_HEAP_MEMORY));
  capability.setVirtualCores(1);

  appContext.setApplicationName(job.name());
  appContext.setApplicationType(ATHENAX_APPLICATION_TYPE);
  appContext.setAMContainerSpec(amContainer);
  appContext.setResource(capability);
  appContext.setApplicationTags(Collections.singleton(job.metadata().serialize()));
  if (job.queue() != null) {
    appContext.setQueue(job.queue());
  }

  LOG.info("Submitting application master {}", appId);
  yarnClient.submitApplication(appContext);

  PollDeploymentStatus poll = new PollDeploymentStatus(appId);
  YARN_POLL_EXECUTOR.submit(poll);
  try {
    return poll.result.get();
  } catch (ExecutionException e) {
    LOG.warn("Failed to deploy {}, cause: {}", appId.toString(), e.getCause());
    yarnClient.killApplication(appId);
    throw (Exception) e.getCause();
  }
}
Example 9
Source File: AMSimulator.java    From hadoop with Apache License 2.0 4 votes vote down vote up 
private void submitApp()
        throws YarnException, InterruptedException, IOException {
  // ask for new application
  GetNewApplicationRequest newAppRequest =
      Records.newRecord(GetNewApplicationRequest.class);
  GetNewApplicationResponse newAppResponse = 
      rm.getClientRMService().getNewApplication(newAppRequest);
  appId = newAppResponse.getApplicationId();
  
  // submit the application
  final SubmitApplicationRequest subAppRequest =
      Records.newRecord(SubmitApplicationRequest.class);
  ApplicationSubmissionContext appSubContext = 
      Records.newRecord(ApplicationSubmissionContext.class);
  appSubContext.setApplicationId(appId);
  appSubContext.setMaxAppAttempts(1);
  appSubContext.setQueue(queue);
  appSubContext.setPriority(Priority.newInstance(0));
  ContainerLaunchContext conLauContext = 
      Records.newRecord(ContainerLaunchContext.class);
  conLauContext.setApplicationACLs(
      new HashMap<ApplicationAccessType, String>());
  conLauContext.setCommands(new ArrayList<String>());
  conLauContext.setEnvironment(new HashMap<String, String>());
  conLauContext.setLocalResources(new HashMap<String, LocalResource>());
  conLauContext.setServiceData(new HashMap<String, ByteBuffer>());
  appSubContext.setAMContainerSpec(conLauContext);
  appSubContext.setUnmanagedAM(true);
  subAppRequest.setApplicationSubmissionContext(appSubContext);
  UserGroupInformation ugi = UserGroupInformation.createRemoteUser(user);
  ugi.doAs(new PrivilegedExceptionAction<Object>() {
    @Override
    public Object run() throws YarnException {
      rm.getClientRMService().submitApplication(subAppRequest);
      return null;
    }
  });
  LOG.info(MessageFormat.format("Submit a new application {0}", appId));
  
  // waiting until application ACCEPTED
  RMApp app = rm.getRMContext().getRMApps().get(appId);
  while(app.getState() != RMAppState.ACCEPTED) {
    Thread.sleep(10);
  }

  // Waiting until application attempt reach LAUNCHED
  // "Unmanaged AM must register after AM attempt reaches LAUNCHED state"
  this.appAttemptId = rm.getRMContext().getRMApps().get(appId)
      .getCurrentAppAttempt().getAppAttemptId();
  RMAppAttempt rmAppAttempt = rm.getRMContext().getRMApps().get(appId)
      .getCurrentAppAttempt();
  while (rmAppAttempt.getAppAttemptState() != RMAppAttemptState.LAUNCHED) {
    Thread.sleep(10);
  }
}
Example 10
Source File: AMSimulator.java    From big-c with Apache License 2.0 4 votes vote down vote up 
private void submitApp()
        throws YarnException, InterruptedException, IOException {
  // ask for new application
  GetNewApplicationRequest newAppRequest =
      Records.newRecord(GetNewApplicationRequest.class);
  GetNewApplicationResponse newAppResponse = 
      rm.getClientRMService().getNewApplication(newAppRequest);
  appId = newAppResponse.getApplicationId();
  
  // submit the application
  final SubmitApplicationRequest subAppRequest =
      Records.newRecord(SubmitApplicationRequest.class);
  ApplicationSubmissionContext appSubContext = 
      Records.newRecord(ApplicationSubmissionContext.class);
  appSubContext.setApplicationId(appId);
  appSubContext.setMaxAppAttempts(1);
  appSubContext.setQueue(queue);
  appSubContext.setPriority(Priority.newInstance(0));
  ContainerLaunchContext conLauContext = 
      Records.newRecord(ContainerLaunchContext.class);
  conLauContext.setApplicationACLs(
      new HashMap<ApplicationAccessType, String>());
  conLauContext.setCommands(new ArrayList<String>());
  conLauContext.setEnvironment(new HashMap<String, String>());
  conLauContext.setLocalResources(new HashMap<String, LocalResource>());
  conLauContext.setServiceData(new HashMap<String, ByteBuffer>());
  appSubContext.setAMContainerSpec(conLauContext);
  appSubContext.setUnmanagedAM(true);
  subAppRequest.setApplicationSubmissionContext(appSubContext);
  UserGroupInformation ugi = UserGroupInformation.createRemoteUser(user);
  ugi.doAs(new PrivilegedExceptionAction<Object>() {
    @Override
    public Object run() throws YarnException {
      rm.getClientRMService().submitApplication(subAppRequest);
      return null;
    }
  });
  LOG.info(MessageFormat.format("Submit a new application {0}", appId));
  
  // waiting until application ACCEPTED
  RMApp app = rm.getRMContext().getRMApps().get(appId);
  while(app.getState() != RMAppState.ACCEPTED) {
    Thread.sleep(10);
  }

  // Waiting until application attempt reach LAUNCHED
  // "Unmanaged AM must register after AM attempt reaches LAUNCHED state"
  this.appAttemptId = rm.getRMContext().getRMApps().get(appId)
      .getCurrentAppAttempt().getAppAttemptId();
  RMAppAttempt rmAppAttempt = rm.getRMContext().getRMApps().get(appId)
      .getCurrentAppAttempt();
  while (rmAppAttempt.getAppAttemptState() != RMAppAttemptState.LAUNCHED) {
    Thread.sleep(10);
  }
}
Example 11
Source File: GobblinYarnAppLauncher.java    From incubator-gobblin with Apache License 2.0 4 votes vote down vote up 
/**
 * Setup and submit the Gobblin Yarn application.
 *
 * @throws IOException if there's anything wrong setting up and submitting the Yarn application
 * @throws YarnException if there's anything wrong setting up and submitting the Yarn application
 */
@VisibleForTesting
ApplicationId setupAndSubmitApplication() throws IOException, YarnException {
  YarnClientApplication gobblinYarnApp = this.yarnClient.createApplication();
  ApplicationSubmissionContext appSubmissionContext = gobblinYarnApp.getApplicationSubmissionContext();
  appSubmissionContext.setApplicationType(GOBBLIN_YARN_APPLICATION_TYPE);
  appSubmissionContext.setMaxAppAttempts(ConfigUtils.getInt(config, GobblinYarnConfigurationKeys.APP_MASTER_MAX_ATTEMPTS_KEY, GobblinYarnConfigurationKeys.DEFAULT_APP_MASTER_MAX_ATTEMPTS_KEY));
  ApplicationId applicationId = appSubmissionContext.getApplicationId();

  GetNewApplicationResponse newApplicationResponse = gobblinYarnApp.getNewApplicationResponse();
  // Set up resource type requirements for ApplicationMaster
  Resource resource = prepareContainerResource(newApplicationResponse);

  // Add lib jars, and jars and files that the ApplicationMaster need as LocalResources
  Map<String, LocalResource> appMasterLocalResources = addAppMasterLocalResources(applicationId);

  ContainerLaunchContext amContainerLaunchContext = Records.newRecord(ContainerLaunchContext.class);
  amContainerLaunchContext.setLocalResources(appMasterLocalResources);
  amContainerLaunchContext.setEnvironment(YarnHelixUtils.getEnvironmentVariables(this.yarnConfiguration));
  amContainerLaunchContext.setCommands(Lists.newArrayList(buildApplicationMasterCommand(applicationId.toString(), resource.getMemory())));

  Map<ApplicationAccessType, String> acls = new HashMap<>(1);
  acls.put(ApplicationAccessType.VIEW_APP, this.appViewAcl);
  amContainerLaunchContext.setApplicationACLs(acls);

  if (UserGroupInformation.isSecurityEnabled()) {
    setupSecurityTokens(amContainerLaunchContext);
  }

  // Setup the application submission context
  appSubmissionContext.setApplicationName(this.applicationName);
  appSubmissionContext.setResource(resource);
  appSubmissionContext.setQueue(this.appQueueName);
  appSubmissionContext.setPriority(Priority.newInstance(0));
  appSubmissionContext.setAMContainerSpec(amContainerLaunchContext);
  // Also setup container local resources by copying local jars and files the container need to HDFS
  addContainerLocalResources(applicationId);

  // Submit the application
  LOGGER.info("Submitting application " + sanitizeApplicationId(applicationId.toString()));
  this.yarnClient.submitApplication(appSubmissionContext);

  LOGGER.info("Application successfully submitted and accepted");
  ApplicationReport applicationReport = this.yarnClient.getApplicationReport(applicationId);
  LOGGER.info("Application Name: " + applicationReport.getName());
  LOGGER.info("Application Tracking URL: " + applicationReport.getTrackingUrl());
  LOGGER.info("Application User: " + applicationReport.getUser() + " Queue: " + applicationReport.getQueue());

  return applicationId;
}
Example 12
Source File: TezClient.java    From incubator-tez with Apache License 2.0 4 votes vote down vote up 
/**
 * Start the client. This establishes a connection to the YARN cluster.
 * In session mode, this start the App Master thats runs all the DAGs in the
 * session.
 * @throws TezException
 * @throws IOException
 */
public synchronized void start() throws TezException, IOException {
  amConfig.setYarnConfiguration(new YarnConfiguration(amConfig.getTezConfiguration()));

  yarnClient = createYarnClient();
  yarnClient.init(amConfig.getYarnConfiguration());
  yarnClient.start();    

  if (isSession) {
    LOG.info("Session mode. Starting session.");
    TezClientUtils.processTezLocalCredentialsFile(sessionCredentials,
        amConfig.getTezConfiguration());

    Map<String, LocalResource> tezJarResources = getTezJarResources(sessionCredentials);

    clientTimeout = amConfig.getTezConfiguration().getInt(
        TezConfiguration.TEZ_SESSION_CLIENT_TIMEOUT_SECS,
        TezConfiguration.TEZ_SESSION_CLIENT_TIMEOUT_SECS_DEFAULT);

    try {
      if (sessionAppId == null) {
        sessionAppId = createApplication();
      }

      // Add session token for shuffle
      TezClientUtils.createSessionToken(sessionAppId.toString(),
          jobTokenSecretManager, sessionCredentials);

      ApplicationSubmissionContext appContext =
          TezClientUtils.createApplicationSubmissionContext(
              amConfig.getTezConfiguration(), sessionAppId,
              null, clientName, amConfig,
              tezJarResources, sessionCredentials);

      // Set Tez Sessions to not retry on AM crashes if recovery is disabled
      if (!amConfig.getTezConfiguration().getBoolean(
          TezConfiguration.DAG_RECOVERY_ENABLED,
          TezConfiguration.DAG_RECOVERY_ENABLED_DEFAULT)) {
        appContext.setMaxAppAttempts(1);
      }  
      yarnClient.submitApplication(appContext);
      sessionStarted = true;
    } catch (YarnException e) {
      throw new TezException(e);
    }
  }
}