Java Code Examples for org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext#setPriority()

The following examples show how to use org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext#setPriority() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: TezClientUtils.java    From tez with Apache License 2.0 6 votes vote down vote up 
@VisibleForTesting
// this method will set the app priority only if the priority config has been defined
public static void setApplicationPriority(ApplicationSubmissionContext context,
    AMConfiguration amConfig) {
  if (amConfig.getTezConfiguration().get(TezConfiguration.TEZ_AM_APPLICATION_PRIORITY) != null) {
    // since we already checked not null condition, we are guaranteed that default value
    // (0 in this case for getInt) will not be returned/used.
    // The idea is to not use any default priority from TEZ side, if none provided in config;
    // let YARN determine the priority of the submitted application
    int priority = amConfig.getTezConfiguration().getInt(TezConfiguration.TEZ_AM_APPLICATION_PRIORITY, 0);
    context.setPriority(Priority.newInstance(priority));
    if (LOG.isDebugEnabled()) {
      LOG.debug("Settting TEZ application priority, applicationId= " + context.getApplicationId() +
          ", priority= " + context.getPriority().getPriority());
    }
  }
}
Example 2
Source File: BuilderUtils.java    From big-c with Apache License 2.0 6 votes vote down vote up 
public static ApplicationSubmissionContext newApplicationSubmissionContext(
    ApplicationId applicationId, String applicationName, String queue,
    Priority priority, ContainerLaunchContext amContainer,
    boolean isUnmanagedAM, boolean cancelTokensWhenComplete,
    int maxAppAttempts, Resource resource, String applicationType) {
  ApplicationSubmissionContext context =
      recordFactory.newRecordInstance(ApplicationSubmissionContext.class);
  context.setApplicationId(applicationId);
  context.setApplicationName(applicationName);
  context.setQueue(queue);
  context.setPriority(priority);
  context.setAMContainerSpec(amContainer);
  context.setUnmanagedAM(isUnmanagedAM);
  context.setCancelTokensWhenComplete(cancelTokensWhenComplete);
  context.setMaxAppAttempts(maxAppAttempts);
  context.setResource(resource);
  context.setApplicationType(applicationType);
  return context;
}
Example 3
Source File: BuilderUtils.java    From hadoop with Apache License 2.0 6 votes vote down vote up 
public static ApplicationSubmissionContext newApplicationSubmissionContext(
    ApplicationId applicationId, String applicationName, String queue,
    Priority priority, ContainerLaunchContext amContainer,
    boolean isUnmanagedAM, boolean cancelTokensWhenComplete,
    int maxAppAttempts, Resource resource, String applicationType) {
  ApplicationSubmissionContext context =
      recordFactory.newRecordInstance(ApplicationSubmissionContext.class);
  context.setApplicationId(applicationId);
  context.setApplicationName(applicationName);
  context.setQueue(queue);
  context.setPriority(priority);
  context.setAMContainerSpec(amContainer);
  context.setUnmanagedAM(isUnmanagedAM);
  context.setCancelTokensWhenComplete(cancelTokensWhenComplete);
  context.setMaxAppAttempts(maxAppAttempts);
  context.setResource(resource);
  context.setApplicationType(applicationType);
  return context;
}
Example 4
Source File: TestYarnClient.java    From hadoop with Apache License 2.0 5 votes vote down vote up 
private ApplicationId createApp(YarnClient rmClient, boolean unmanaged) 
  throws Exception {
  YarnClientApplication newApp = rmClient.createApplication();

  ApplicationId appId = newApp.getNewApplicationResponse().getApplicationId();

  // Create launch context for app master
  ApplicationSubmissionContext appContext
    = Records.newRecord(ApplicationSubmissionContext.class);

  // set the application id
  appContext.setApplicationId(appId);

  // set the application name
  appContext.setApplicationName("test");

  // Set the priority for the application master
  Priority pri = Records.newRecord(Priority.class);
  pri.setPriority(1);
  appContext.setPriority(pri);

  // Set the queue to which this application is to be submitted in the RM
  appContext.setQueue("default");

  // Set up the container launch context for the application master
  ContainerLaunchContext amContainer
    = Records.newRecord(ContainerLaunchContext.class);
  appContext.setAMContainerSpec(amContainer);
  appContext.setResource(Resource.newInstance(1024, 1));
  appContext.setUnmanagedAM(unmanaged);

  // Submit the application to the applications manager
  rmClient.submitApplication(appContext);

  return appId;
}
Example 5
Source File: YarnServiceTestWithExpiration.java    From incubator-gobblin with Apache License 2.0 5 votes vote down vote up 
private void startApp() throws Exception {
  // submit a dummy app
  ApplicationSubmissionContext appSubmissionContext =
      yarnClient.createApplication().getApplicationSubmissionContext();
  this.applicationId = appSubmissionContext.getApplicationId();

  ContainerLaunchContext containerLaunchContext =
      BuilderUtils.newContainerLaunchContext(Collections.emptyMap(), Collections.emptyMap(),
          Arrays.asList("sleep", "100"), Collections.emptyMap(), null, Collections.emptyMap());

  // Setup the application submission context
  appSubmissionContext.setApplicationName("TestApp");
  appSubmissionContext.setResource(Resource.newInstance(128, 1));
  appSubmissionContext.setPriority(Priority.newInstance(0));
  appSubmissionContext.setAMContainerSpec(containerLaunchContext);

  this.yarnClient.submitApplication(appSubmissionContext);

  // wait for application to be accepted
  int i;
  RMAppAttempt attempt = null;
  for (i = 0; i < 120; i++) {
    ApplicationReport appReport = yarnClient.getApplicationReport(applicationId);

    if (appReport.getYarnApplicationState() == YarnApplicationState.ACCEPTED) {
      this.applicationAttemptId = appReport.getCurrentApplicationAttemptId();
      attempt = yarnCluster.getResourceManager().getRMContext().getRMApps()
          .get(appReport.getCurrentApplicationAttemptId().getApplicationId()).getCurrentAppAttempt();
      break;
    }
    Thread.sleep(1000);
  }

  Assert.assertTrue(i < 120, "timed out waiting for ACCEPTED state");

  // Set the AM-RM token in the UGI for access during testing
  UserGroupInformation.setLoginUser(UserGroupInformation.createRemoteUser(UserGroupInformation.getCurrentUser()
      .getUserName()));
  UserGroupInformation.getCurrentUser().addToken(attempt.getAMRMToken());
}
Example 6
Source File: YarnServiceTest.java    From incubator-gobblin with Apache License 2.0 5 votes vote down vote up 
private void startApp() throws Exception {
  // submit a dummy app
  ApplicationSubmissionContext appSubmissionContext =
      yarnClient.createApplication().getApplicationSubmissionContext();
  this.applicationId = appSubmissionContext.getApplicationId();

  ContainerLaunchContext containerLaunchContext =
      BuilderUtils.newContainerLaunchContext(Collections.emptyMap(), Collections.emptyMap(),
          Arrays.asList("sleep", "100"), Collections.emptyMap(), null, Collections.emptyMap());

  // Setup the application submission context
  appSubmissionContext.setApplicationName("TestApp");
  appSubmissionContext.setResource(Resource.newInstance(128, 1));
  appSubmissionContext.setPriority(Priority.newInstance(0));
  appSubmissionContext.setAMContainerSpec(containerLaunchContext);

  this.yarnClient.submitApplication(appSubmissionContext);

  // wait for application to be accepted
  int i;
  RMAppAttempt attempt = null;
  for (i = 0; i < 120; i++) {
    ApplicationReport appReport = yarnClient.getApplicationReport(applicationId);

    if (appReport.getYarnApplicationState() == YarnApplicationState.ACCEPTED) {
      this.applicationAttemptId = appReport.getCurrentApplicationAttemptId();
      attempt = yarnCluster.getResourceManager().getRMContext().getRMApps()
          .get(appReport.getCurrentApplicationAttemptId().getApplicationId()).getCurrentAppAttempt();
      break;
    }
    Thread.sleep(1000);
  }

  Assert.assertTrue(i < 120, "timed out waiting for ACCEPTED state");

  // Set the AM-RM token in the UGI for access during testing
  UserGroupInformation.setLoginUser(UserGroupInformation.createRemoteUser(UserGroupInformation.getCurrentUser()
      .getUserName()));
  UserGroupInformation.getCurrentUser().addToken(attempt.getAMRMToken());
}
Example 7
Source File: TestYarnClient.java    From big-c with Apache License 2.0 5 votes vote down vote up 
private ApplicationId createApp(YarnClient rmClient, boolean unmanaged) 
  throws Exception {
  YarnClientApplication newApp = rmClient.createApplication();

  ApplicationId appId = newApp.getNewApplicationResponse().getApplicationId();

  // Create launch context for app master
  ApplicationSubmissionContext appContext
    = Records.newRecord(ApplicationSubmissionContext.class);

  // set the application id
  appContext.setApplicationId(appId);

  // set the application name
  appContext.setApplicationName("test");

  // Set the priority for the application master
  Priority pri = Records.newRecord(Priority.class);
  pri.setPriority(1);
  appContext.setPriority(pri);

  // Set the queue to which this application is to be submitted in the RM
  appContext.setQueue("default");

  // Set up the container launch context for the application master
  ContainerLaunchContext amContainer
    = Records.newRecord(ContainerLaunchContext.class);
  appContext.setAMContainerSpec(amContainer);
  appContext.setResource(Resource.newInstance(1024, 1));
  appContext.setUnmanagedAM(unmanaged);

  // Submit the application to the applications manager
  rmClient.submitApplication(appContext);

  return appId;
}
Example 8
Source File: TestAMRMClient.java    From big-c with Apache License 2.0 4 votes vote down vote up 
@Before
public void startApp() throws Exception {
  // submit new app
  ApplicationSubmissionContext appContext = 
      yarnClient.createApplication().getApplicationSubmissionContext();
  ApplicationId appId = appContext.getApplicationId();
  // set the application name
  appContext.setApplicationName("Test");
  // Set the priority for the application master
  Priority pri = Records.newRecord(Priority.class);
  pri.setPriority(0);
  appContext.setPriority(pri);
  // Set the queue to which this application is to be submitted in the RM
  appContext.setQueue("default");
  // Set up the container launch context for the application master
  ContainerLaunchContext amContainer =
      BuilderUtils.newContainerLaunchContext(
        Collections.<String, LocalResource> emptyMap(),
        new HashMap<String, String>(), Arrays.asList("sleep", "100"),
        new HashMap<String, ByteBuffer>(), null,
        new HashMap<ApplicationAccessType, String>());
  appContext.setAMContainerSpec(amContainer);
  appContext.setResource(Resource.newInstance(1024, 1));
  // Create the request to send to the applications manager
  SubmitApplicationRequest appRequest = Records
      .newRecord(SubmitApplicationRequest.class);
  appRequest.setApplicationSubmissionContext(appContext);
  // Submit the application to the applications manager
  yarnClient.submitApplication(appContext);

  // wait for app to start
  RMAppAttempt appAttempt = null;
  while (true) {
    ApplicationReport appReport = yarnClient.getApplicationReport(appId);
    if (appReport.getYarnApplicationState() == YarnApplicationState.ACCEPTED) {
      attemptId = appReport.getCurrentApplicationAttemptId();
      appAttempt =
          yarnCluster.getResourceManager().getRMContext().getRMApps()
            .get(attemptId.getApplicationId()).getCurrentAppAttempt();
      while (true) {
        if (appAttempt.getAppAttemptState() == RMAppAttemptState.LAUNCHED) {
          break;
        }
      }
      break;
    }
  }
  // Just dig into the ResourceManager and get the AMRMToken just for the sake
  // of testing.
  UserGroupInformation.setLoginUser(UserGroupInformation
    .createRemoteUser(UserGroupInformation.getCurrentUser().getUserName()));

  // emulate RM setup of AMRM token in credentials by adding the token
  // *before* setting the token service
  UserGroupInformation.getCurrentUser().addToken(appAttempt.getAMRMToken());
  appAttempt.getAMRMToken().setService(ClientRMProxy.getAMRMTokenService(conf));
}
Example 9
Source File: GobblinYarnAppLauncher.java    From incubator-gobblin with Apache License 2.0 4 votes vote down vote up 
/**
 * Setup and submit the Gobblin Yarn application.
 *
 * @throws IOException if there's anything wrong setting up and submitting the Yarn application
 * @throws YarnException if there's anything wrong setting up and submitting the Yarn application
 */
@VisibleForTesting
ApplicationId setupAndSubmitApplication() throws IOException, YarnException {
  YarnClientApplication gobblinYarnApp = this.yarnClient.createApplication();
  ApplicationSubmissionContext appSubmissionContext = gobblinYarnApp.getApplicationSubmissionContext();
  appSubmissionContext.setApplicationType(GOBBLIN_YARN_APPLICATION_TYPE);
  appSubmissionContext.setMaxAppAttempts(ConfigUtils.getInt(config, GobblinYarnConfigurationKeys.APP_MASTER_MAX_ATTEMPTS_KEY, GobblinYarnConfigurationKeys.DEFAULT_APP_MASTER_MAX_ATTEMPTS_KEY));
  ApplicationId applicationId = appSubmissionContext.getApplicationId();

  GetNewApplicationResponse newApplicationResponse = gobblinYarnApp.getNewApplicationResponse();
  // Set up resource type requirements for ApplicationMaster
  Resource resource = prepareContainerResource(newApplicationResponse);

  // Add lib jars, and jars and files that the ApplicationMaster need as LocalResources
  Map<String, LocalResource> appMasterLocalResources = addAppMasterLocalResources(applicationId);

  ContainerLaunchContext amContainerLaunchContext = Records.newRecord(ContainerLaunchContext.class);
  amContainerLaunchContext.setLocalResources(appMasterLocalResources);
  amContainerLaunchContext.setEnvironment(YarnHelixUtils.getEnvironmentVariables(this.yarnConfiguration));
  amContainerLaunchContext.setCommands(Lists.newArrayList(buildApplicationMasterCommand(applicationId.toString(), resource.getMemory())));

  Map<ApplicationAccessType, String> acls = new HashMap<>(1);
  acls.put(ApplicationAccessType.VIEW_APP, this.appViewAcl);
  amContainerLaunchContext.setApplicationACLs(acls);

  if (UserGroupInformation.isSecurityEnabled()) {
    setupSecurityTokens(amContainerLaunchContext);
  }

  // Setup the application submission context
  appSubmissionContext.setApplicationName(this.applicationName);
  appSubmissionContext.setResource(resource);
  appSubmissionContext.setQueue(this.appQueueName);
  appSubmissionContext.setPriority(Priority.newInstance(0));
  appSubmissionContext.setAMContainerSpec(amContainerLaunchContext);
  // Also setup container local resources by copying local jars and files the container need to HDFS
  addContainerLocalResources(applicationId);

  // Submit the application
  LOGGER.info("Submitting application " + sanitizeApplicationId(applicationId.toString()));
  this.yarnClient.submitApplication(appSubmissionContext);

  LOGGER.info("Application successfully submitted and accepted");
  ApplicationReport applicationReport = this.yarnClient.getApplicationReport(applicationId);
  LOGGER.info("Application Name: " + applicationReport.getName());
  LOGGER.info("Application Tracking URL: " + applicationReport.getTrackingUrl());
  LOGGER.info("Application User: " + applicationReport.getUser() + " Queue: " + applicationReport.getQueue());

  return applicationId;
}
Example 10
Source File: YarnRemoteInterpreterProcess.java    From zeppelin with Apache License 2.0 4 votes vote down vote up 
private void setPriority(ApplicationSubmissionContext appContext) {
  Priority pri = Records.newRecord(Priority.class);
  pri.setPriority(1);
  appContext.setPriority(pri);
}
Example 11
Source File: InlineAM.java    From attic-apex-core with Apache License 2.0 4 votes vote down vote up 
public boolean run() throws Exception
{
  LOG.info("Starting Client");

  // Connect to ResourceManager
  rmClient.start();
  try {
    // Get a new application id
    YarnClientApplication newApp = rmClient.createApplication();
    ApplicationId appId = newApp.getNewApplicationResponse().getApplicationId();

    // Create launch context for app master
    LOG.info("Setting up application submission context for ASM");
    ApplicationSubmissionContext appContext = Records
        .newRecord(ApplicationSubmissionContext.class);

    // set the application id
    appContext.setApplicationId(appId);
    // set the application name
    appContext.setApplicationName(appName);

    // Set the priority for the application master
    Priority pri = Records.newRecord(Priority.class);
    pri.setPriority(amPriority);
    appContext.setPriority(pri);

    // Set the queue to which this application is to be submitted in the RM
    appContext.setQueue(amQueue);

    // Set up the container launch context for the application master
    ContainerLaunchContext amContainer = Records
        .newRecord(ContainerLaunchContext.class);
    appContext.setAMContainerSpec(amContainer);

    // unmanaged AM
    appContext.setUnmanagedAM(true);
    LOG.info("Setting unmanaged AM");

    // Submit the application to the applications manager
    LOG.info("Submitting application to ASM");
    rmClient.submitApplication(appContext);

    // Monitor the application to wait for launch state
    ApplicationReport appReport = monitorApplication(appId,
        EnumSet.of(YarnApplicationState.ACCEPTED));
    ApplicationAttemptId attemptId = appReport.getCurrentApplicationAttemptId();
    LOG.info("Launching application with id: " + attemptId);

    // launch AM
    runAM(attemptId);

    // Monitor the application for end state
    appReport = monitorApplication(appId, EnumSet.of(
        YarnApplicationState.KILLED, YarnApplicationState.FAILED,
        YarnApplicationState.FINISHED));
    YarnApplicationState appState = appReport.getYarnApplicationState();
    FinalApplicationStatus appStatus = appReport.getFinalApplicationStatus();

    LOG.info("App ended with state: " + appReport.getYarnApplicationState()
        + " and status: " + appStatus);

    boolean success;
    if (YarnApplicationState.FINISHED == appState
        && FinalApplicationStatus.SUCCEEDED == appStatus) {
      LOG.info("Application has completed successfully.");
      success = true;
    } else {
      LOG.info("Application did finished unsuccessfully." + " YarnState="
          + appState.toString() + ", FinalStatus=" + appStatus.toString());
      success = false;
    }

    return success;
  } finally {
    rmClient.stop();
  }
}
Example 12
Source File: AMSimulator.java    From big-c with Apache License 2.0 4 votes vote down vote up 
private void submitApp()
        throws YarnException, InterruptedException, IOException {
  // ask for new application
  GetNewApplicationRequest newAppRequest =
      Records.newRecord(GetNewApplicationRequest.class);
  GetNewApplicationResponse newAppResponse = 
      rm.getClientRMService().getNewApplication(newAppRequest);
  appId = newAppResponse.getApplicationId();
  
  // submit the application
  final SubmitApplicationRequest subAppRequest =
      Records.newRecord(SubmitApplicationRequest.class);
  ApplicationSubmissionContext appSubContext = 
      Records.newRecord(ApplicationSubmissionContext.class);
  appSubContext.setApplicationId(appId);
  appSubContext.setMaxAppAttempts(1);
  appSubContext.setQueue(queue);
  appSubContext.setPriority(Priority.newInstance(0));
  ContainerLaunchContext conLauContext = 
      Records.newRecord(ContainerLaunchContext.class);
  conLauContext.setApplicationACLs(
      new HashMap<ApplicationAccessType, String>());
  conLauContext.setCommands(new ArrayList<String>());
  conLauContext.setEnvironment(new HashMap<String, String>());
  conLauContext.setLocalResources(new HashMap<String, LocalResource>());
  conLauContext.setServiceData(new HashMap<String, ByteBuffer>());
  appSubContext.setAMContainerSpec(conLauContext);
  appSubContext.setUnmanagedAM(true);
  subAppRequest.setApplicationSubmissionContext(appSubContext);
  UserGroupInformation ugi = UserGroupInformation.createRemoteUser(user);
  ugi.doAs(new PrivilegedExceptionAction<Object>() {
    @Override
    public Object run() throws YarnException {
      rm.getClientRMService().submitApplication(subAppRequest);
      return null;
    }
  });
  LOG.info(MessageFormat.format("Submit a new application {0}", appId));
  
  // waiting until application ACCEPTED
  RMApp app = rm.getRMContext().getRMApps().get(appId);
  while(app.getState() != RMAppState.ACCEPTED) {
    Thread.sleep(10);
  }

  // Waiting until application attempt reach LAUNCHED
  // "Unmanaged AM must register after AM attempt reaches LAUNCHED state"
  this.appAttemptId = rm.getRMContext().getRMApps().get(appId)
      .getCurrentAppAttempt().getAppAttemptId();
  RMAppAttempt rmAppAttempt = rm.getRMContext().getRMApps().get(appId)
      .getCurrentAppAttempt();
  while (rmAppAttempt.getAppAttemptState() != RMAppAttemptState.LAUNCHED) {
    Thread.sleep(10);
  }
}
Example 13
Source File: UnmanagedAMLauncher.java    From big-c with Apache License 2.0 4 votes vote down vote up 
public boolean run() throws IOException, YarnException {
  LOG.info("Starting Client");
  
  // Connect to ResourceManager
  rmClient.start();
  try {  
    // Create launch context for app master
    LOG.info("Setting up application submission context for ASM");
    ApplicationSubmissionContext appContext = rmClient.createApplication()
        .getApplicationSubmissionContext();
    ApplicationId appId = appContext.getApplicationId();

    // set the application name
    appContext.setApplicationName(appName);

    // Set the priority for the application master
    Priority pri = Records.newRecord(Priority.class);
    pri.setPriority(amPriority);
    appContext.setPriority(pri);

    // Set the queue to which this application is to be submitted in the RM
    appContext.setQueue(amQueue);

    // Set up the container launch context for the application master
    ContainerLaunchContext amContainer = Records
        .newRecord(ContainerLaunchContext.class);
    appContext.setAMContainerSpec(amContainer);

    // unmanaged AM
    appContext.setUnmanagedAM(true);
    LOG.info("Setting unmanaged AM");

    // Submit the application to the applications manager
    LOG.info("Submitting application to ASM");
    rmClient.submitApplication(appContext);

    ApplicationReport appReport =
        monitorApplication(appId, EnumSet.of(YarnApplicationState.ACCEPTED,
          YarnApplicationState.KILLED, YarnApplicationState.FAILED,
          YarnApplicationState.FINISHED));

    if (appReport.getYarnApplicationState() == YarnApplicationState.ACCEPTED) {
      // Monitor the application attempt to wait for launch state
      ApplicationAttemptReport attemptReport =
          monitorCurrentAppAttempt(appId,
            YarnApplicationAttemptState.LAUNCHED);
      ApplicationAttemptId attemptId =
          attemptReport.getApplicationAttemptId();
      LOG.info("Launching AM with application attempt id " + attemptId);
      // launch AM
      launchAM(attemptId);
      // Monitor the application for end state
      appReport =
          monitorApplication(appId, EnumSet.of(YarnApplicationState.KILLED,
            YarnApplicationState.FAILED, YarnApplicationState.FINISHED));
    }

    YarnApplicationState appState = appReport.getYarnApplicationState();
    FinalApplicationStatus appStatus = appReport.getFinalApplicationStatus();

    LOG.info("App ended with state: " + appReport.getYarnApplicationState()
        + " and status: " + appStatus);
    
    boolean success;
    if (YarnApplicationState.FINISHED == appState
        && FinalApplicationStatus.SUCCEEDED == appStatus) {
      LOG.info("Application has completed successfully.");
      success = true;
    } else {
      LOG.info("Application did finished unsuccessfully." + " YarnState="
          + appState.toString() + ", FinalStatus=" + appStatus.toString());
      success = false;
    }
    
    return success;
  } finally {
    rmClient.stop();
  }
}
Example 14
Source File: TestNMClient.java    From big-c with Apache License 2.0 4 votes vote down vote up 
@Before
public void setup() throws YarnException, IOException {
  // start minicluster
  conf = new YarnConfiguration();
  yarnCluster =
      new MiniYARNCluster(TestAMRMClient.class.getName(), nodeCount, 1, 1);
  yarnCluster.init(conf);
  yarnCluster.start();
  assertNotNull(yarnCluster);
  assertEquals(STATE.STARTED, yarnCluster.getServiceState());

  // start rm client
  yarnClient = (YarnClientImpl) YarnClient.createYarnClient();
  yarnClient.init(conf);
  yarnClient.start();
  assertNotNull(yarnClient);
  assertEquals(STATE.STARTED, yarnClient.getServiceState());

  // get node info
  nodeReports = yarnClient.getNodeReports(NodeState.RUNNING);

  // submit new app
  ApplicationSubmissionContext appContext = 
      yarnClient.createApplication().getApplicationSubmissionContext();
  ApplicationId appId = appContext.getApplicationId();
  // set the application name
  appContext.setApplicationName("Test");
  // Set the priority for the application master
  Priority pri = Priority.newInstance(0);
  appContext.setPriority(pri);
  // Set the queue to which this application is to be submitted in the RM
  appContext.setQueue("default");
  // Set up the container launch context for the application master
  ContainerLaunchContext amContainer = Records
      .newRecord(ContainerLaunchContext.class);
  appContext.setAMContainerSpec(amContainer);
  // unmanaged AM
  appContext.setUnmanagedAM(true);
  // Create the request to send to the applications manager
  SubmitApplicationRequest appRequest = Records
      .newRecord(SubmitApplicationRequest.class);
  appRequest.setApplicationSubmissionContext(appContext);
  // Submit the application to the applications manager
  yarnClient.submitApplication(appContext);

  // wait for app to start
  int iterationsLeft = 30;
  RMAppAttempt appAttempt = null;
  while (iterationsLeft > 0) {
    ApplicationReport appReport = yarnClient.getApplicationReport(appId);
    if (appReport.getYarnApplicationState() ==
        YarnApplicationState.ACCEPTED) {
      attemptId = appReport.getCurrentApplicationAttemptId();
      appAttempt =
          yarnCluster.getResourceManager().getRMContext().getRMApps()
            .get(attemptId.getApplicationId()).getCurrentAppAttempt();
      while (true) {
        if (appAttempt.getAppAttemptState() == RMAppAttemptState.LAUNCHED) {
          break;
        }
      }
      break;
    }
    sleep(1000);
    --iterationsLeft;
  }
  if (iterationsLeft == 0) {
    fail("Application hasn't bee started");
  }

  // Just dig into the ResourceManager and get the AMRMToken just for the sake
  // of testing.
  UserGroupInformation.setLoginUser(UserGroupInformation
    .createRemoteUser(UserGroupInformation.getCurrentUser().getUserName()));
  UserGroupInformation.getCurrentUser().addToken(appAttempt.getAMRMToken());

  //creating an instance NMTokenCase
  nmTokenCache = new NMTokenCache();
  
  // start am rm client
  rmClient =
      (AMRMClientImpl<ContainerRequest>) AMRMClient
        .<ContainerRequest> createAMRMClient();

  //setting an instance NMTokenCase
  rmClient.setNMTokenCache(nmTokenCache);
  rmClient.init(conf);
  rmClient.start();
  assertNotNull(rmClient);
  assertEquals(STATE.STARTED, rmClient.getServiceState());

  // start am nm client
  nmClient = (NMClientImpl) NMClient.createNMClient();
  
  //propagating the AMRMClient NMTokenCache instance
  nmClient.setNMTokenCache(rmClient.getNMTokenCache());
  nmClient.init(conf);
  nmClient.start();
  assertNotNull(nmClient);
  assertEquals(STATE.STARTED, nmClient.getServiceState());
}
Example 15
Source File: AMSimulator.java    From hadoop with Apache License 2.0 4 votes vote down vote up 
private void submitApp()
        throws YarnException, InterruptedException, IOException {
  // ask for new application
  GetNewApplicationRequest newAppRequest =
      Records.newRecord(GetNewApplicationRequest.class);
  GetNewApplicationResponse newAppResponse = 
      rm.getClientRMService().getNewApplication(newAppRequest);
  appId = newAppResponse.getApplicationId();
  
  // submit the application
  final SubmitApplicationRequest subAppRequest =
      Records.newRecord(SubmitApplicationRequest.class);
  ApplicationSubmissionContext appSubContext = 
      Records.newRecord(ApplicationSubmissionContext.class);
  appSubContext.setApplicationId(appId);
  appSubContext.setMaxAppAttempts(1);
  appSubContext.setQueue(queue);
  appSubContext.setPriority(Priority.newInstance(0));
  ContainerLaunchContext conLauContext = 
      Records.newRecord(ContainerLaunchContext.class);
  conLauContext.setApplicationACLs(
      new HashMap<ApplicationAccessType, String>());
  conLauContext.setCommands(new ArrayList<String>());
  conLauContext.setEnvironment(new HashMap<String, String>());
  conLauContext.setLocalResources(new HashMap<String, LocalResource>());
  conLauContext.setServiceData(new HashMap<String, ByteBuffer>());
  appSubContext.setAMContainerSpec(conLauContext);
  appSubContext.setUnmanagedAM(true);
  subAppRequest.setApplicationSubmissionContext(appSubContext);
  UserGroupInformation ugi = UserGroupInformation.createRemoteUser(user);
  ugi.doAs(new PrivilegedExceptionAction<Object>() {
    @Override
    public Object run() throws YarnException {
      rm.getClientRMService().submitApplication(subAppRequest);
      return null;
    }
  });
  LOG.info(MessageFormat.format("Submit a new application {0}", appId));
  
  // waiting until application ACCEPTED
  RMApp app = rm.getRMContext().getRMApps().get(appId);
  while(app.getState() != RMAppState.ACCEPTED) {
    Thread.sleep(10);
  }

  // Waiting until application attempt reach LAUNCHED
  // "Unmanaged AM must register after AM attempt reaches LAUNCHED state"
  this.appAttemptId = rm.getRMContext().getRMApps().get(appId)
      .getCurrentAppAttempt().getAppAttemptId();
  RMAppAttempt rmAppAttempt = rm.getRMContext().getRMApps().get(appId)
      .getCurrentAppAttempt();
  while (rmAppAttempt.getAppAttemptState() != RMAppAttemptState.LAUNCHED) {
    Thread.sleep(10);
  }
}
Example 16
Source File: UnmanagedAMLauncher.java    From hadoop with Apache License 2.0 4 votes vote down vote up 
public boolean run() throws IOException, YarnException {
  LOG.info("Starting Client");
  
  // Connect to ResourceManager
  rmClient.start();
  try {  
    // Create launch context for app master
    LOG.info("Setting up application submission context for ASM");
    ApplicationSubmissionContext appContext = rmClient.createApplication()
        .getApplicationSubmissionContext();
    ApplicationId appId = appContext.getApplicationId();

    // set the application name
    appContext.setApplicationName(appName);

    // Set the priority for the application master
    Priority pri = Records.newRecord(Priority.class);
    pri.setPriority(amPriority);
    appContext.setPriority(pri);

    // Set the queue to which this application is to be submitted in the RM
    appContext.setQueue(amQueue);

    // Set up the container launch context for the application master
    ContainerLaunchContext amContainer = Records
        .newRecord(ContainerLaunchContext.class);
    appContext.setAMContainerSpec(amContainer);

    // unmanaged AM
    appContext.setUnmanagedAM(true);
    LOG.info("Setting unmanaged AM");

    // Submit the application to the applications manager
    LOG.info("Submitting application to ASM");
    rmClient.submitApplication(appContext);

    ApplicationReport appReport =
        monitorApplication(appId, EnumSet.of(YarnApplicationState.ACCEPTED,
          YarnApplicationState.KILLED, YarnApplicationState.FAILED,
          YarnApplicationState.FINISHED));

    if (appReport.getYarnApplicationState() == YarnApplicationState.ACCEPTED) {
      // Monitor the application attempt to wait for launch state
      ApplicationAttemptReport attemptReport =
          monitorCurrentAppAttempt(appId,
            YarnApplicationAttemptState.LAUNCHED);
      ApplicationAttemptId attemptId =
          attemptReport.getApplicationAttemptId();
      LOG.info("Launching AM with application attempt id " + attemptId);
      // launch AM
      launchAM(attemptId);
      // Monitor the application for end state
      appReport =
          monitorApplication(appId, EnumSet.of(YarnApplicationState.KILLED,
            YarnApplicationState.FAILED, YarnApplicationState.FINISHED));
    }

    YarnApplicationState appState = appReport.getYarnApplicationState();
    FinalApplicationStatus appStatus = appReport.getFinalApplicationStatus();

    LOG.info("App ended with state: " + appReport.getYarnApplicationState()
        + " and status: " + appStatus);
    
    boolean success;
    if (YarnApplicationState.FINISHED == appState
        && FinalApplicationStatus.SUCCEEDED == appStatus) {
      LOG.info("Application has completed successfully.");
      success = true;
    } else {
      LOG.info("Application did finished unsuccessfully." + " YarnState="
          + appState.toString() + ", FinalStatus=" + appStatus.toString());
      success = false;
    }
    
    return success;
  } finally {
    rmClient.stop();
  }
}
Example 17
Source File: TestAMRMClient.java    From hadoop with Apache License 2.0 4 votes vote down vote up 
@Before
public void startApp() throws Exception {
  // submit new app
  ApplicationSubmissionContext appContext = 
      yarnClient.createApplication().getApplicationSubmissionContext();
  ApplicationId appId = appContext.getApplicationId();
  // set the application name
  appContext.setApplicationName("Test");
  // Set the priority for the application master
  Priority pri = Records.newRecord(Priority.class);
  pri.setPriority(0);
  appContext.setPriority(pri);
  // Set the queue to which this application is to be submitted in the RM
  appContext.setQueue("default");
  // Set up the container launch context for the application master
  ContainerLaunchContext amContainer =
      BuilderUtils.newContainerLaunchContext(
        Collections.<String, LocalResource> emptyMap(),
        new HashMap<String, String>(), Arrays.asList("sleep", "100"),
        new HashMap<String, ByteBuffer>(), null,
        new HashMap<ApplicationAccessType, String>());
  appContext.setAMContainerSpec(amContainer);
  appContext.setResource(Resource.newInstance(1024, 1, 1));
  // Create the request to send to the applications manager
  SubmitApplicationRequest appRequest = Records
      .newRecord(SubmitApplicationRequest.class);
  appRequest.setApplicationSubmissionContext(appContext);
  // Submit the application to the applications manager
  yarnClient.submitApplication(appContext);

  // wait for app to start
  RMAppAttempt appAttempt = null;
  while (true) {
    ApplicationReport appReport = yarnClient.getApplicationReport(appId);
    if (appReport.getYarnApplicationState() == YarnApplicationState.ACCEPTED) {
      attemptId = appReport.getCurrentApplicationAttemptId();
      appAttempt =
          yarnCluster.getResourceManager().getRMContext().getRMApps()
            .get(attemptId.getApplicationId()).getCurrentAppAttempt();
      while (true) {
        if (appAttempt.getAppAttemptState() == RMAppAttemptState.LAUNCHED) {
          break;
        }
      }
      break;
    }
  }
  // Just dig into the ResourceManager and get the AMRMToken just for the sake
  // of testing.
  UserGroupInformation.setLoginUser(UserGroupInformation
    .createRemoteUser(UserGroupInformation.getCurrentUser().getUserName()));

  // emulate RM setup of AMRM token in credentials by adding the token
  // *before* setting the token service
  UserGroupInformation.getCurrentUser().addToken(appAttempt.getAMRMToken());
  appAttempt.getAMRMToken().setService(ClientRMProxy.getAMRMTokenService(conf));
}
Example 18
Source File: TestNMClient.java    From hadoop with Apache License 2.0 4 votes vote down vote up 
@Before
public void setup() throws YarnException, IOException {
  // start minicluster
  conf = new YarnConfiguration();
  yarnCluster =
      new MiniYARNCluster(TestAMRMClient.class.getName(), nodeCount, 1, 1);
  yarnCluster.init(conf);
  yarnCluster.start();
  assertNotNull(yarnCluster);
  assertEquals(STATE.STARTED, yarnCluster.getServiceState());

  // start rm client
  yarnClient = (YarnClientImpl) YarnClient.createYarnClient();
  yarnClient.init(conf);
  yarnClient.start();
  assertNotNull(yarnClient);
  assertEquals(STATE.STARTED, yarnClient.getServiceState());

  // get node info
  nodeReports = yarnClient.getNodeReports(NodeState.RUNNING);

  // submit new app
  ApplicationSubmissionContext appContext = 
      yarnClient.createApplication().getApplicationSubmissionContext();
  ApplicationId appId = appContext.getApplicationId();
  // set the application name
  appContext.setApplicationName("Test");
  // Set the priority for the application master
  Priority pri = Priority.newInstance(0);
  appContext.setPriority(pri);
  // Set the queue to which this application is to be submitted in the RM
  appContext.setQueue("default");
  // Set up the container launch context for the application master
  ContainerLaunchContext amContainer = Records
      .newRecord(ContainerLaunchContext.class);
  appContext.setAMContainerSpec(amContainer);
  // unmanaged AM
  appContext.setUnmanagedAM(true);
  // Create the request to send to the applications manager
  SubmitApplicationRequest appRequest = Records
      .newRecord(SubmitApplicationRequest.class);
  appRequest.setApplicationSubmissionContext(appContext);
  // Submit the application to the applications manager
  yarnClient.submitApplication(appContext);

  // wait for app to start
  int iterationsLeft = 30;
  RMAppAttempt appAttempt = null;
  while (iterationsLeft > 0) {
    ApplicationReport appReport = yarnClient.getApplicationReport(appId);
    if (appReport.getYarnApplicationState() ==
        YarnApplicationState.ACCEPTED) {
      attemptId = appReport.getCurrentApplicationAttemptId();
      appAttempt =
          yarnCluster.getResourceManager().getRMContext().getRMApps()
            .get(attemptId.getApplicationId()).getCurrentAppAttempt();
      while (true) {
        if (appAttempt.getAppAttemptState() == RMAppAttemptState.LAUNCHED) {
          break;
        }
      }
      break;
    }
    sleep(1000);
    --iterationsLeft;
  }
  if (iterationsLeft == 0) {
    fail("Application hasn't bee started");
  }

  // Just dig into the ResourceManager and get the AMRMToken just for the sake
  // of testing.
  UserGroupInformation.setLoginUser(UserGroupInformation
    .createRemoteUser(UserGroupInformation.getCurrentUser().getUserName()));
  UserGroupInformation.getCurrentUser().addToken(appAttempt.getAMRMToken());

  //creating an instance NMTokenCase
  nmTokenCache = new NMTokenCache();
  
  // start am rm client
  rmClient =
      (AMRMClientImpl<ContainerRequest>) AMRMClient
        .<ContainerRequest> createAMRMClient();

  //setting an instance NMTokenCase
  rmClient.setNMTokenCache(nmTokenCache);
  rmClient.init(conf);
  rmClient.start();
  assertNotNull(rmClient);
  assertEquals(STATE.STARTED, rmClient.getServiceState());

  // start am nm client
  nmClient = (NMClientImpl) NMClient.createNMClient();
  
  //propagating the AMRMClient NMTokenCache instance
  nmClient.setNMTokenCache(rmClient.getNMTokenCache());
  nmClient.init(conf);
  nmClient.start();
  assertNotNull(nmClient);
  assertEquals(STATE.STARTED, nmClient.getServiceState());
}