org.apache.hadoop.yarn.client.api.YarnClientApplication Java Examples

public ApplicationId start(Job job)
        throws Exception
{
    JobGraph jobGraph = job.getJobDAG();
    FlinkJobConfig jobConfig = job.getConfig();

    Iterable<Path> userProvidedJars = getUserAdditionalJars(job.getDepends());
    YarnClientApplication application = yarnClient.createApplication();
    final YarnJobDescriptor descriptor = new YarnJobDescriptor(
            application,
            flinkConf,
            yarnClient,
            yarnConfiguration,
            jobConfig,
            job.getName(),
            userProvidedJars);
    return start(descriptor, jobGraph);
}
 

YarnJobDescriptor(
        YarnClientApplication application,
        FlinkConfiguration flinkConf,
        YarnClient yarnClient,
        YarnConfiguration yarnConfiguration,
        FlinkJobConfig appConf,
        String jobId,
        Iterable<Path> userProvidedJars)
        throws IOException
{
    super(flinkConf.flinkConfiguration(), yarnConfiguration, flinkConf.getConfigurationDirectory(), yarnClient, false);
    this.application = application;
    this.jobName = jobId;
    this.flinkConf = flinkConf;
    this.yarnClient = yarnClient;
    this.appConf = appConf;
    this.userProvidedJars = userProvidedJars;

    FileSystem fileSystem = FileSystem.get(yarnClient.getConfig());
    this.uploadingDir = new Path(new Path(fileSystem.getHomeDirectory(), ".sylph"), application.getApplicationSubmissionContext().getApplicationId().toString());
}
 

public boolean run() throws Exception {
  YarnClientApplication app = createApplication();
  ApplicationId appId = app.getNewApplicationResponse().getApplicationId();

  // Copy the application jar to the filesystem
  FileSystem fs = FileSystem.get(conf);
  String appIdStr = appId.toString();
  Path dstJarPath = Utils.copyLocalFileToDfs(fs, appIdStr, new Path(tfJar), Constants.TF_JAR_NAME);
  Path dstLibPath = Utils.copyLocalFileToDfs(fs, appIdStr, new Path(tfLib),
      Constants.TF_LIB_NAME);
  Map<String, Path> files = new HashMap<>();
  files.put(Constants.TF_JAR_NAME, dstJarPath);
  Map<String, LocalResource> localResources = Utils.makeLocalResources(fs, files);
  Map<String, String> javaEnv = Utils.setJavaEnv(conf);
  String command = makeAppMasterCommand(dstLibPath.toString(), dstJarPath.toString());
  LOG.info("Make ApplicationMaster command: " + command);
  ContainerLaunchContext launchContext = ContainerLaunchContext.newInstance(
      localResources, javaEnv, Lists.newArrayList(command), null, null, null);
  Resource resource = Resource.newInstance(amMemory, amVCores);
  submitApplication(app, appName, launchContext, resource, amQueue);
  return awaitApplication(appId);
}
 

/**
 * Kills YARN application and stops YARN client.
 *
 * <p>Use this method to kill the App before it has been properly deployed
 */
private void failSessionDuringDeployment(YarnClient yarnClient, YarnClientApplication yarnApplication) {
	LOG.info("Killing YARN application");

	try {
		yarnClient.killApplication(yarnApplication.getNewApplicationResponse().getApplicationId());
	} catch (Exception e) {
		// we only log a debug message here because the "killApplication" call is a best-effort
		// call (we don't know if the application has been deployed when the error occured).
		LOG.debug("Error while killing YARN application", e);
	}
}
 

DeploymentFailureHook(YarnClientApplication yarnApplication, Path yarnFilesDir) {
	this.yarnApplication = Preconditions.checkNotNull(yarnApplication);
	this.yarnFilesDir = Preconditions.checkNotNull(yarnFilesDir);

	// A new yarn client need to be created in shutdown hook in order to avoid
	// the yarn client has been closed by YarnClusterDescriptor.
	this.yarnClient = YarnClient.createYarnClient();
	this.yarnClient.init(yarnConfiguration);
}
 

private ApplicationId createApp(YarnClient rmClient, boolean unmanaged) 
  throws Exception {
  YarnClientApplication newApp = rmClient.createApplication();

  ApplicationId appId = newApp.getNewApplicationResponse().getApplicationId();

  // Create launch context for app master
  ApplicationSubmissionContext appContext
    = Records.newRecord(ApplicationSubmissionContext.class);

  // set the application id
  appContext.setApplicationId(appId);

  // set the application name
  appContext.setApplicationName("test");

  // Set the priority for the application master
  Priority pri = Records.newRecord(Priority.class);
  pri.setPriority(1);
  appContext.setPriority(pri);

  // Set the queue to which this application is to be submitted in the RM
  appContext.setQueue("default");

  // Set up the container launch context for the application master
  ContainerLaunchContext amContainer
    = Records.newRecord(ContainerLaunchContext.class);
  appContext.setAMContainerSpec(amContainer);
  appContext.setResource(Resource.newInstance(1024, 1));
  appContext.setUnmanagedAM(unmanaged);

  // Submit the application to the applications manager
  rmClient.submitApplication(appContext);

  return appId;
}
 

@Override
public String getSeedQueryId() throws IOException {
  try {
    YarnClientApplication app = yarnClient.createApplication();
    return app.getApplicationSubmissionContext().getApplicationId().toString();
  } catch (Exception e) {
    LOG.error(e.getMessage(), e);

    throw new IOException(e.getMessage(), e);
  }
}
 

@Override
public YarnClientApplication createApplication()
    throws YarnException, IOException {
  ApplicationSubmissionContext context = Records.newRecord
      (ApplicationSubmissionContext.class);
  GetNewApplicationResponse newApp = getNewApplication();
  ApplicationId appId = newApp.getApplicationId();
  context.setApplicationId(appId);
  return new YarnClientApplication(newApp, context);
}
 

private ApplicationId createApp(YarnClient rmClient, boolean unmanaged) 
  throws Exception {
  YarnClientApplication newApp = rmClient.createApplication();

  ApplicationId appId = newApp.getNewApplicationResponse().getApplicationId();

  // Create launch context for app master
  ApplicationSubmissionContext appContext
    = Records.newRecord(ApplicationSubmissionContext.class);

  // set the application id
  appContext.setApplicationId(appId);

  // set the application name
  appContext.setApplicationName("test");

  // Set the priority for the application master
  Priority pri = Records.newRecord(Priority.class);
  pri.setPriority(1);
  appContext.setPriority(pri);

  // Set the queue to which this application is to be submitted in the RM
  appContext.setQueue("default");

  // Set up the container launch context for the application master
  ContainerLaunchContext amContainer
    = Records.newRecord(ContainerLaunchContext.class);
  appContext.setAMContainerSpec(amContainer);
  appContext.setResource(Resource.newInstance(1024, 1));
  appContext.setUnmanagedAM(unmanaged);

  // Submit the application to the applications manager
  rmClient.submitApplication(appContext);

  return appId;
}
 

@Override
public YarnClientApplication createApplication()
    throws YarnException, IOException {
  ApplicationSubmissionContext context = Records.newRecord
      (ApplicationSubmissionContext.class);
  GetNewApplicationResponse newApp = getNewApplication();
  ApplicationId appId = newApp.getApplicationId();
  context.setApplicationId(appId);
  return new YarnClientApplication(newApp, context);
}
 

ApplicationId submitApplication(
    YarnClientApplication app,
    String appName,
    ContainerLaunchContext launchContext,
    Resource resource,
    String queue) throws Exception {
  ApplicationSubmissionContext appContext = app.getApplicationSubmissionContext();
  appContext.setApplicationName(appName);
  appContext.setApplicationTags(new HashSet<>());
  appContext.setAMContainerSpec(launchContext);
  appContext.setResource(resource);
  appContext.setQueue(queue);

  return yarnClient.submitApplication(appContext);
}
 

public YarnSubmissionHelper(final YarnConfiguration yarnConfiguration,
                            final REEFFileNames fileNames,
                            final ClasspathProvider classpath,
                            final YarnProxyUser yarnProxyUser,
                            final SecurityTokenProvider tokenProvider,
                            final boolean isUnmanaged,
                            final List<String> commandPrefixList) throws IOException, YarnException {

  this.classpath = classpath;
  this.yarnProxyUser = yarnProxyUser;
  this.isUnmanaged = isUnmanaged;

  this.driverStdoutFilePath =
      ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/" + fileNames.getDriverStdoutFileName();

  this.driverStderrFilePath =
      ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/" + fileNames.getDriverStderrFileName();

  LOG.log(Level.FINE, "Initializing YARN Client");
  this.yarnClient = YarnClient.createYarnClient();
  this.yarnClient.init(yarnConfiguration);
  this.yarnClient.start();
  LOG.log(Level.FINE, "Initialized YARN Client");

  LOG.log(Level.FINE, "Requesting Application ID from YARN.");
  final YarnClientApplication yarnClientApplication = this.yarnClient.createApplication();
  this.applicationResponse = yarnClientApplication.getNewApplicationResponse();
  this.applicationSubmissionContext = yarnClientApplication.getApplicationSubmissionContext();
  this.applicationSubmissionContext.setUnmanagedAM(isUnmanaged);
  this.applicationId = this.applicationSubmissionContext.getApplicationId();
  this.tokenProvider = tokenProvider;
  this.commandPrefixList = commandPrefixList;
  this.configurationFilePaths = Collections.singletonList(fileNames.getDriverConfigurationPath());
  LOG.log(Level.INFO, "YARN Application ID: {0}", this.applicationId);
}
 

UnmanagedAmYarnSubmissionHelper(
    final YarnConfiguration yarnConfiguration,
    final YarnProxyUser yarnProxyUser,
    final SecurityTokenProvider tokenProvider) throws IOException, YarnException {

  this.tokenProvider = tokenProvider;
  this.yarnProxyUser = yarnProxyUser;

  LOG.log(Level.FINE, "Initializing YARN Client");
  this.yarnClient = YarnClient.createYarnClient();
  this.yarnClient.init(yarnConfiguration);
  this.yarnClient.start();
  LOG.log(Level.FINE, "Initialized YARN Client");

  LOG.log(Level.FINE, "Requesting UNMANAGED Application ID from YARN.");

  final ContainerLaunchContext launchContext = YarnTypes.getContainerLaunchContext(
      Collections.<String>emptyList(), Collections.<String, LocalResource>emptyMap(), tokenProvider.getTokens());

  final YarnClientApplication yarnClientApplication = this.yarnClient.createApplication();

  this.applicationSubmissionContext = yarnClientApplication.getApplicationSubmissionContext();
  this.applicationSubmissionContext.setAMContainerSpec(launchContext);
  this.applicationSubmissionContext.setUnmanagedAM(true);

  this.applicationId = this.applicationSubmissionContext.getApplicationId();

  LOG.log(Level.INFO, "YARN UNMANAGED Application ID: {0}", this.applicationId);
}
 

/**
 * Kills YARN application and stops YARN client.
 *
 * <p>Use this method to kill the App before it has been properly deployed
 */
private void failSessionDuringDeployment(YarnClient yarnClient, YarnClientApplication yarnApplication)
{
    LOG.info("Killing YARN application");

    try {
        yarnClient.killApplication(yarnApplication.getNewApplicationResponse().getApplicationId());
    }
    catch (Exception e) {
        // we only log a debug message here because the "killApplication" call is a best-effort
        // call (we don't know if the application has been deployed when the error occured).
        LOG.debug("Error while killing YARN application", e);
    }
    yarnClient.stop();
}
 

public void run(VMConfig vmConfig, Configuration conf) throws Exception {
  try {
    vmConfig.overrideHadoopConfiguration(conf);
    System.out.println("Create YarnClient") ;
    YarnClient yarnClient = YarnClient.createYarnClient();
    yarnClient.init(conf);
    yarnClient.start();

    System.out.println("Create YarnClientApplication via YarnClient") ;
    YarnClientApplication app = yarnClient.createApplication();
    String appId =  app.getApplicationSubmissionContext().getApplicationId().toString() ;
    System.out.println("Application Id = " + appId) ;
    System.out.println("Set up the container launch context for the application master") ;
    ContainerLaunchContext amContainer = Records.newRecord(ContainerLaunchContext.class);
    StringBuilder sb = new StringBuilder();
    List<String> commands = Collections.singletonList(
        sb.append(vmConfig.buildCommand()).
        append(" 1> ").append(ApplicationConstants.LOG_DIR_EXPANSION_VAR).append("/stdout").
        append(" 2> ").append(ApplicationConstants.LOG_DIR_EXPANSION_VAR).append("/stderr")
        .toString()
    );
    amContainer.setCommands(commands) ;

    System.out.println("Setup the app classpath and resources") ;
    if(vmConfig.getVmResources().size() > 0) {
      amContainer.setLocalResources(new VMResources(conf, vmConfig));
    }
    
    System.out.println("Setup the classpath for ApplicationMaster, environment = " + vmConfig.getEnvironment()) ;
    Map<String, String> appMasterEnv = new HashMap<String, String>();
    boolean jvmEnv = vmConfig.getEnvironment() != VMConfig.Environment.YARN;
    Util.setupAppMasterEnv(jvmEnv , conf, appMasterEnv);
    amContainer.setEnvironment(appMasterEnv);

    System.out.println("Set up resource type requirements for ApplicationMaster") ;
    Resource resource = Records.newRecord(Resource.class);
    resource.setMemory(256);
    resource.setVirtualCores(1);

    System.out.println("Finally, set-up ApplicationSubmissionContext for the application");
    ApplicationSubmissionContext appContext = app.getApplicationSubmissionContext();
    appContext.setApplicationName(vmConfig.getName()); // application name
    appContext.setAMContainerSpec(amContainer);
    appContext.setResource(resource);
    appContext.setQueue("default"); // queue 

    // Submit application
    ApplicationId applicationId = appContext.getApplicationId();
    System.out.println("Submitting application " + applicationId);
    yarnClient.submitApplication(appContext);
  } catch(Exception ex) {
    ex.printStackTrace(); 
    throw ex ;
  }
}
 

@Override
public ApplicationSubmissionContext createApplicationSubmissionContext(YarnClientApplication newApp, ContainerLaunchContext containerContext)
{
    final ApplicationSubmissionContext appContext = super.createApplicationSubmissionContext(newApp, containerContext);
    appContext.setApplicationType("Sylph_SPARK");
    appContext.setApplicationTags(ImmutableSet.of("a1", "a2"));
    appContext.setQueue(yarnQueue);
    appContext.setMaxAppAttempts(2);
    return appContext;
}
 

@Override
public YarnClientApplication createApplication() throws YarnException, IOException {
  ApplicationSubmissionContext context = Records.newRecord(ApplicationSubmissionContext.class);
  ApplicationId appId = ApplicationId.newInstance(clusterTimeStamp, appIdNumber++);
  context.setApplicationId(appId);
  GetNewApplicationResponse response = Records.newRecord(GetNewApplicationResponse.class);
  response.setApplicationId(appId);
  return new YarnClientApplication(response, context);
}
 

/**
 * Kills YARN application and stops YARN client.
 *
 * <p>Use this method to kill the App before it has been properly deployed
 */
private void failSessionDuringDeployment(YarnClient yarnClient, YarnClientApplication yarnApplication) {
	LOG.info("Killing YARN application");

	try {
		yarnClient.killApplication(yarnApplication.getNewApplicationResponse().getApplicationId());
	} catch (Exception e) {
		// we only log a debug message here because the "killApplication" call is a best-effort
		// call (we don't know if the application has been deployed when the error occured).
		LOG.debug("Error while killing YARN application", e);
	}
	yarnClient.stop();
}
 

/**
 * Kills YARN application and stops YARN client.
 *
 * <p>Use this method to kill the App before it has been properly deployed
 */
private void failSessionDuringDeployment(YarnClient yarnClient, YarnClientApplication yarnApplication) {
	LOG.info("Killing YARN application");

	try {
		yarnClient.killApplication(yarnApplication.getNewApplicationResponse().getApplicationId());
	} catch (Exception e) {
		// we only log a debug message here because the "killApplication" call is a best-effort
		// call (we don't know if the application has been deployed when the error occured).
		LOG.debug("Error while killing YARN application", e);
	}
	yarnClient.stop();
}
 

public ApplicationId run() throws IOException, YarnException {
  LOG.info("calling run.");
  yarnClient.start();

  // YarnClientApplication is used to populate:
  //   1. GetNewApplication Response
  //   2. ApplicationSubmissionContext
  YarnClientApplication app = yarnClient.createApplication();
  
  // GetNewApplicationResponse can be used to determined resources available.
  GetNewApplicationResponse appResponse = app.getNewApplicationResponse();
  
  ApplicationSubmissionContext appContext = app.getApplicationSubmissionContext();
  this.appId = appContext.getApplicationId();
  appContext.setApplicationName(this.appname);

  // Set up the container launch context for AM.
  ContainerLaunchContext amContainer = Records.newRecord(ContainerLaunchContext.class);

  LocalResource appMasterJar;
  FileSystem fs = FileSystem.get(this.conf);
  
  amContainer.setLocalResources(
      Collections.singletonMap("master.jar",
          Util.newYarnAppResource(fs, new Path(this.hdfsJar), LocalResourceType.FILE,
              LocalResourceVisibility.APPLICATION)));
  // Set up CLASSPATH for ApplicationMaster
  Map<String, String> appMasterEnv = new HashMap<String, String>();
  setupAppMasterEnv(appMasterEnv);
  amContainer.setEnvironment(appMasterEnv);

  // Set up resource requirements for ApplicationMaster
  Resource capability = Records.newRecord(Resource.class);
  capability.setMemory(this.applicationMasterMem);
  capability.setVirtualCores(1); //TODO: Can we really setVirtualCores ?
  amContainer.setCommands(Collections.singletonList(this.getAppMasterCommand()));

  // put everything together.
  appContext.setAMContainerSpec(amContainer);
  appContext.setResource(capability);
  appContext.setQueue("default"); // TODO: Need to investigate more on queuing an scheduling.

  // Submit application
  yarnClient.submitApplication(appContext);
  LOG.info("APPID: "+this.appId.toString());
  return this.appId;
  //return this.monitorApplication(appId);
}
 

/**
 * Main run function for the client
 * @return true if application completed successfully
 * @throws java.io.IOException
 * @throws org.apache.hadoop.yarn.exceptions.YarnException
 */
public boolean run() throws IOException, YarnException {

  LOG.info("Running Client");
  yarnClient.start();

  // Get a new application id
  YarnClientApplication app = yarnClient.createApplication();
  GetNewApplicationResponse appResponse = app.getNewApplicationResponse();

  int maxMem = appResponse.getMaximumResourceCapability().getMemory();
  LOG.info("Max mem capabililty of resources in this cluster " + maxMem);

  // A resource ask cannot exceed the max.
  if (amMemory > maxMem) {
    LOG.info("AM memory specified above max threshold of cluster. Using max value."
        + ", specified=" + amMemory
        + ", max=" + maxMem);
    amMemory = maxMem;
  }

  int maxVCores = appResponse.getMaximumResourceCapability().getVirtualCores();
  LOG.info("Max virtual cores capabililty of resources in this cluster " + maxVCores);

  if (amVCores > maxVCores) {
    LOG.info("AM virtual cores specified above max threshold of cluster. "
        + "Using max value." + ", specified=" + amVCores
        + ", max=" + maxVCores);
    amVCores = maxVCores;
  }

  // set the application name
  ApplicationSubmissionContext appContext = app.getApplicationSubmissionContext();
  ApplicationId appId = appContext.getApplicationId();

  appContext.setApplicationName(appName);

  // Set up resource type requirements
  // For now, both memory and vcores are supported, so we set memory and
  // vcores requirements
  Resource capability = Records.newRecord(Resource.class);
  capability.setMemory(amMemory);
  capability.setVirtualCores(amVCores);
  appContext.setResource(capability);

  // Set the priority for the application master
  Priority pri = Records.newRecord(Priority.class);
  pri.setPriority(amPriority);
  appContext.setPriority(pri);

  // Set the queue to which this application is to be submitted in the RM
  appContext.setQueue(amQueue);

  // Set the ContainerLaunchContext to describe the Container ith which the ApplicationMaster is launched.
  appContext.setAMContainerSpec(getAMContainerSpec(appId.getId()));

  // Submit the application to the applications manager
  // SubmitApplicationResponse submitResp = applicationsManager.submitApplication(appRequest);
  // Ignore the response as either a valid response object is returned on success
  // or an exception thrown to denote some form of a failure
  LOG.info("Submitting application to ASM");

  yarnClient.submitApplication(appContext);

  // Monitor the application
  return monitorApplication(appId);
}
 

/**
 * Setup and submit the Gobblin Yarn application.
 *
 * @throws IOException if there's anything wrong setting up and submitting the Yarn application
 * @throws YarnException if there's anything wrong setting up and submitting the Yarn application
 */
@VisibleForTesting
ApplicationId setupAndSubmitApplication() throws IOException, YarnException {
  YarnClientApplication gobblinYarnApp = this.yarnClient.createApplication();
  ApplicationSubmissionContext appSubmissionContext = gobblinYarnApp.getApplicationSubmissionContext();
  appSubmissionContext.setApplicationType(GOBBLIN_YARN_APPLICATION_TYPE);
  appSubmissionContext.setMaxAppAttempts(ConfigUtils.getInt(config, GobblinYarnConfigurationKeys.APP_MASTER_MAX_ATTEMPTS_KEY, GobblinYarnConfigurationKeys.DEFAULT_APP_MASTER_MAX_ATTEMPTS_KEY));
  ApplicationId applicationId = appSubmissionContext.getApplicationId();

  GetNewApplicationResponse newApplicationResponse = gobblinYarnApp.getNewApplicationResponse();
  // Set up resource type requirements for ApplicationMaster
  Resource resource = prepareContainerResource(newApplicationResponse);

  // Add lib jars, and jars and files that the ApplicationMaster need as LocalResources
  Map<String, LocalResource> appMasterLocalResources = addAppMasterLocalResources(applicationId);

  ContainerLaunchContext amContainerLaunchContext = Records.newRecord(ContainerLaunchContext.class);
  amContainerLaunchContext.setLocalResources(appMasterLocalResources);
  amContainerLaunchContext.setEnvironment(YarnHelixUtils.getEnvironmentVariables(this.yarnConfiguration));
  amContainerLaunchContext.setCommands(Lists.newArrayList(buildApplicationMasterCommand(applicationId.toString(), resource.getMemory())));

  Map<ApplicationAccessType, String> acls = new HashMap<>(1);
  acls.put(ApplicationAccessType.VIEW_APP, this.appViewAcl);
  amContainerLaunchContext.setApplicationACLs(acls);

  if (UserGroupInformation.isSecurityEnabled()) {
    setupSecurityTokens(amContainerLaunchContext);
  }

  // Setup the application submission context
  appSubmissionContext.setApplicationName(this.applicationName);
  appSubmissionContext.setResource(resource);
  appSubmissionContext.setQueue(this.appQueueName);
  appSubmissionContext.setPriority(Priority.newInstance(0));
  appSubmissionContext.setAMContainerSpec(amContainerLaunchContext);
  // Also setup container local resources by copying local jars and files the container need to HDFS
  addContainerLocalResources(applicationId);

  // Submit the application
  LOGGER.info("Submitting application " + sanitizeApplicationId(applicationId.toString()));
  this.yarnClient.submitApplication(appSubmissionContext);

  LOGGER.info("Application successfully submitted and accepted");
  ApplicationReport applicationReport = this.yarnClient.getApplicationReport(applicationId);
  LOGGER.info("Application Name: " + applicationReport.getName());
  LOGGER.info("Application Tracking URL: " + applicationReport.getTrackingUrl());
  LOGGER.info("Application User: " + applicationReport.getUser() + " Queue: " + applicationReport.getQueue());

  return applicationId;
}
 

@Override
public YarnClientApplication createApplication() throws YarnException, IOException {
  return yarnClient.createApplication();
}
 

public void run(String[] args) throws Exception {
    final String command = args[0];
    final int n = Integer.valueOf(args[1]);
    final Path jarPath = new Path(args[2]);
    final String resourceManagerAddress = args[3];
    final String resourceManagerHostname = args[4];
    final String resourceManagerSchedulerAddress = args[5];
    final String resourceManagerResourceTrackerAddress = args[6];

    // Create yarnClient
    YarnConfiguration conf = new YarnConfiguration();
    conf.set("yarn.resourcemanager.address", resourceManagerAddress);
    conf.set("yarn.resourcemanager.hostname", resourceManagerHostname);
    conf.set("yarn.resourcemanager.scheduler.address", resourceManagerSchedulerAddress);
    conf.set("yarn.resourcemanager.resource-tracker.address", resourceManagerResourceTrackerAddress);
    YarnClient yarnClient = YarnClient.createYarnClient();
    yarnClient.init(conf);
    yarnClient.start();

    // Create application via yarnClient
    YarnClientApplication app = yarnClient.createApplication();

    // Set up the container launch context for the application master
    ContainerLaunchContext amContainer =
            Records.newRecord(ContainerLaunchContext.class);
    amContainer.setCommands(
            Collections.singletonList(
                    "$JAVA_HOME/bin/java" +
                            " -Xmx256M" +
                            " com.hortonworks.simpleyarnapp.ApplicationMaster" +
                            " " + command +
                            " " + String.valueOf(n) +
                            " " + resourceManagerAddress +
                            " " + resourceManagerHostname +
                            " " + resourceManagerSchedulerAddress +
                            " " + resourceManagerResourceTrackerAddress +
                            " 1>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/stdout" +
                            " 2>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/stderr"
            )
    );

    // Setup jar for ApplicationMaster
    LocalResource appMasterJar = Records.newRecord(LocalResource.class);
    setupAppMasterJar(jarPath, appMasterJar);
    amContainer.setLocalResources(
            Collections.singletonMap("simple-yarn-app-1.1.0.jar", appMasterJar));

    // Setup CLASSPATH for ApplicationMaster
    Map<String, String> appMasterEnv = new HashMap<String, String>();
    setupAppMasterEnv(appMasterEnv);
    amContainer.setEnvironment(appMasterEnv);

    // Set up resource type requirements for ApplicationMaster
    Resource capability = Records.newRecord(Resource.class);
    capability.setMemory(256);
    capability.setVirtualCores(1);

    // Finally, set-up ApplicationSubmissionContext for the application
    ApplicationSubmissionContext appContext =
            app.getApplicationSubmissionContext();
    appContext.setApplicationName("simple-yarn-app"); // application name
    appContext.setAMContainerSpec(amContainer);
    appContext.setResource(capability);
    appContext.setQueue("default"); // queue

    // Submit application
    ApplicationId appId = appContext.getApplicationId();
    System.out.println("Submitting application " + appId);
    yarnClient.submitApplication(appContext);

    ApplicationReport appReport = yarnClient.getApplicationReport(appId);
    YarnApplicationState appState = appReport.getYarnApplicationState();
    while (appState != YarnApplicationState.FINISHED &&
            appState != YarnApplicationState.KILLED &&
            appState != YarnApplicationState.FAILED) {
        Thread.sleep(100);
        appReport = yarnClient.getApplicationReport(appId);
        appState = appReport.getYarnApplicationState();
    }

    System.out.println(
            "Application " + appId + " finished with" +
                    " state " + appState +
                    " at " + appReport.getFinishTime());

}
 

public boolean run() throws Exception
{
  LOG.info("Starting Client");

  // Connect to ResourceManager
  rmClient.start();
  try {
    // Get a new application id
    YarnClientApplication newApp = rmClient.createApplication();
    ApplicationId appId = newApp.getNewApplicationResponse().getApplicationId();

    // Create launch context for app master
    LOG.info("Setting up application submission context for ASM");
    ApplicationSubmissionContext appContext = Records
        .newRecord(ApplicationSubmissionContext.class);

    // set the application id
    appContext.setApplicationId(appId);
    // set the application name
    appContext.setApplicationName(appName);

    // Set the priority for the application master
    Priority pri = Records.newRecord(Priority.class);
    pri.setPriority(amPriority);
    appContext.setPriority(pri);

    // Set the queue to which this application is to be submitted in the RM
    appContext.setQueue(amQueue);

    // Set up the container launch context for the application master
    ContainerLaunchContext amContainer = Records
        .newRecord(ContainerLaunchContext.class);
    appContext.setAMContainerSpec(amContainer);

    // unmanaged AM
    appContext.setUnmanagedAM(true);
    LOG.info("Setting unmanaged AM");

    // Submit the application to the applications manager
    LOG.info("Submitting application to ASM");
    rmClient.submitApplication(appContext);

    // Monitor the application to wait for launch state
    ApplicationReport appReport = monitorApplication(appId,
        EnumSet.of(YarnApplicationState.ACCEPTED));
    ApplicationAttemptId attemptId = appReport.getCurrentApplicationAttemptId();
    LOG.info("Launching application with id: " + attemptId);

    // launch AM
    runAM(attemptId);

    // Monitor the application for end state
    appReport = monitorApplication(appId, EnumSet.of(
        YarnApplicationState.KILLED, YarnApplicationState.FAILED,
        YarnApplicationState.FINISHED));
    YarnApplicationState appState = appReport.getYarnApplicationState();
    FinalApplicationStatus appStatus = appReport.getFinalApplicationStatus();

    LOG.info("App ended with state: " + appReport.getYarnApplicationState()
        + " and status: " + appStatus);

    boolean success;
    if (YarnApplicationState.FINISHED == appState
        && FinalApplicationStatus.SUCCEEDED == appStatus) {
      LOG.info("Application has completed successfully.");
      success = true;
    } else {
      LOG.info("Application did finished unsuccessfully." + " YarnState="
          + appState.toString() + ", FinalStatus=" + appStatus.toString());
      success = false;
    }

    return success;
  } finally {
    rmClient.stop();
  }
}
 

@Override
public YarnClientApplication createApplication() throws
    YarnException, IOException {
  return client.createApplication();
}
 

@Test (timeout = 30000)
public void testSubmitIncorrectQueue() throws IOException {
  MiniYARNCluster cluster = new MiniYARNCluster("testMRAMTokens", 1, 1, 1);
  YarnClient rmClient = null;
  try {
    cluster.init(new YarnConfiguration());
    cluster.start();
    final Configuration yarnConf = cluster.getConfig();
    rmClient = YarnClient.createYarnClient();
    rmClient.init(yarnConf);
    rmClient.start();
    YarnClientApplication newApp = rmClient.createApplication();

    ApplicationId appId = newApp.getNewApplicationResponse().getApplicationId();

    // Create launch context for app master
    ApplicationSubmissionContext appContext
      = Records.newRecord(ApplicationSubmissionContext.class);

    // set the application id
    appContext.setApplicationId(appId);

    // set the application name
    appContext.setApplicationName("test");

    // Set the queue to which this application is to be submitted in the RM
    appContext.setQueue("nonexist");

    // Set up the container launch context for the application master
    ContainerLaunchContext amContainer
      = Records.newRecord(ContainerLaunchContext.class);
    appContext.setAMContainerSpec(amContainer);
    appContext.setResource(Resource.newInstance(1024, 1));
    // appContext.setUnmanagedAM(unmanaged);

    // Submit the application to the applications manager
    rmClient.submitApplication(appContext);
    Assert.fail("Job submission should have thrown an exception");
  } catch (YarnException e) {
    Assert.assertTrue(e.getMessage().contains("Failed to submit"));
  } finally {
    if (rmClient != null) {
      rmClient.stop();
    }
    cluster.stop();
  }
}
 

@Override
public YarnClientApplication createApplication() throws
    YarnException, IOException {
  return client.createApplication();
}
 

@Test (timeout = 30000)
public void testSubmitIncorrectQueue() throws IOException {
  MiniYARNCluster cluster = new MiniYARNCluster("testMRAMTokens", 1, 1, 1);
  YarnClient rmClient = null;
  try {
    cluster.init(new YarnConfiguration());
    cluster.start();
    final Configuration yarnConf = cluster.getConfig();
    rmClient = YarnClient.createYarnClient();
    rmClient.init(yarnConf);
    rmClient.start();
    YarnClientApplication newApp = rmClient.createApplication();

    ApplicationId appId = newApp.getNewApplicationResponse().getApplicationId();

    // Create launch context for app master
    ApplicationSubmissionContext appContext
      = Records.newRecord(ApplicationSubmissionContext.class);

    // set the application id
    appContext.setApplicationId(appId);

    // set the application name
    appContext.setApplicationName("test");

    // Set the queue to which this application is to be submitted in the RM
    appContext.setQueue("nonexist");

    // Set up the container launch context for the application master
    ContainerLaunchContext amContainer
      = Records.newRecord(ContainerLaunchContext.class);
    appContext.setAMContainerSpec(amContainer);
    appContext.setResource(Resource.newInstance(1024, 1));
    // appContext.setUnmanagedAM(unmanaged);

    // Submit the application to the applications manager
    rmClient.submitApplication(appContext);
    Assert.fail("Job submission should have thrown an exception");
  } catch (YarnException e) {
    Assert.assertTrue(e.getMessage().contains("Failed to submit"));
  } finally {
    if (rmClient != null) {
      rmClient.stop();
    }
    cluster.stop();
  }
}
 

YarnClientApplication createApplication() throws Exception {
  return yarnClient.createApplication();
}