org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext Java Examples

private void setApplicationTags(final ApplicationSubmissionContext appContext) throws InvocationTargetException,
	IllegalAccessException {

	final ApplicationSubmissionContextReflector reflector = ApplicationSubmissionContextReflector.getInstance();
	final String tagsString = flinkConfiguration.getString(YarnConfigOptions.APPLICATION_TAGS);

	final Set<String> applicationTags = new HashSet<>();

	// Trim whitespace and cull empty tags
	for (final String tag : tagsString.split(",")) {
		final String trimmedTag = tag.trim();
		if (!trimmedTag.isEmpty()) {
			applicationTags.add(trimmedTag);
		}
	}

	reflector.setApplicationTags(appContext, applicationTags);
}
 

@SuppressWarnings("deprecation")
@Before
public void setUp() {
  long now = System.currentTimeMillis();

  rmContext = mockRMContext(1, now - 10);
  ResourceScheduler scheduler = mockResourceScheduler();
  Configuration conf = new Configuration();
  ApplicationMasterService masterService =
      new ApplicationMasterService(rmContext, scheduler);
  appMonitor = new TestRMAppManager(rmContext,
      new ClientToAMTokenSecretManagerInRM(), scheduler, masterService,
      new ApplicationACLsManager(conf), conf);

  appId = MockApps.newAppID(1);
  RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null);
  asContext =
      recordFactory.newRecordInstance(ApplicationSubmissionContext.class);
  asContext.setApplicationId(appId);
  asContext.setAMContainerSpec(mockContainerLaunchContext(recordFactory));
  asContext.setResource(mockResource());
  setupDispatcher(rmContext, conf);
}
 

public static ApplicationSubmissionContext newApplicationSubmissionContext(
    ApplicationId applicationId, String applicationName, String queue,
    Priority priority, ContainerLaunchContext amContainer,
    boolean isUnmanagedAM, boolean cancelTokensWhenComplete,
    int maxAppAttempts, Resource resource, String applicationType) {
  ApplicationSubmissionContext context =
      recordFactory.newRecordInstance(ApplicationSubmissionContext.class);
  context.setApplicationId(applicationId);
  context.setApplicationName(applicationName);
  context.setQueue(queue);
  context.setPriority(priority);
  context.setAMContainerSpec(amContainer);
  context.setUnmanagedAM(isUnmanagedAM);
  context.setCancelTokensWhenComplete(cancelTokensWhenComplete);
  context.setMaxAppAttempts(maxAppAttempts);
  context.setResource(resource);
  context.setApplicationType(applicationType);
  return context;
}
 

public SchedulerApplicationAttempt(ApplicationAttemptId applicationAttemptId, 
    String user, Queue queue, ActiveUsersManager activeUsersManager,
    RMContext rmContext) {
  Preconditions.checkNotNull(rmContext, "RMContext should not be null");
  this.rmContext = rmContext;
  this.appSchedulingInfo = 
      new AppSchedulingInfo(applicationAttemptId, user, queue,  
          activeUsersManager, rmContext.getEpoch());
  this.queue = queue;
  this.pendingRelease = new HashSet<ContainerId>();
  this.attemptId = applicationAttemptId;
  if (rmContext.getRMApps() != null &&
      rmContext.getRMApps()
          .containsKey(applicationAttemptId.getApplicationId())) {
    ApplicationSubmissionContext appSubmissionContext =
        rmContext.getRMApps().get(applicationAttemptId.getApplicationId())
            .getApplicationSubmissionContext();
    if (appSubmissionContext != null) {
      unmanagedAM = appSubmissionContext.getUnmanagedAM();
      this.logAggregationContext =
          appSubmissionContext.getLogAggregationContext();
    }
  }
}
 

private ContainerLaunchContext createAMContainerLaunchContext(
    ApplicationSubmissionContext applicationMasterContext,
    ContainerId containerID) throws IOException {

  // Construct the actual Container
  ContainerLaunchContext container = 
      applicationMasterContext.getAMContainerSpec();
  LOG.info("Command to launch container "
      + containerID
      + " : "
      + StringUtils.arrayToString(container.getCommands().toArray(
          new String[0])));
  
  // Finalize the container
  setupTokens(container, containerID);
  
  return container;
}
 

@VisibleForTesting
// this method will set the app priority only if the priority config has been defined
public static void setApplicationPriority(ApplicationSubmissionContext context,
    AMConfiguration amConfig) {
  if (amConfig.getTezConfiguration().get(TezConfiguration.TEZ_AM_APPLICATION_PRIORITY) != null) {
    // since we already checked not null condition, we are guaranteed that default value
    // (0 in this case for getInt) will not be returned/used.
    // The idea is to not use any default priority from TEZ side, if none provided in config;
    // let YARN determine the priority of the submitted application
    int priority = amConfig.getTezConfiguration().getInt(TezConfiguration.TEZ_AM_APPLICATION_PRIORITY, 0);
    context.setPriority(Priority.newInstance(priority));
    if (LOG.isDebugEnabled()) {
      LOG.debug("Settting TEZ application priority, applicationId= " + context.getApplicationId() +
          ", priority= " + context.getPriority().getPriority());
    }
  }
}
 

protected RMApp testCreateAppFinished(
    ApplicationSubmissionContext submissionContext,
    String diagnostics) throws IOException {
  // unmanaged AMs don't use the FINISHING state
  RMApp application = null;
  if (submissionContext != null && submissionContext.getUnmanagedAM()) {
    application = testCreateAppRunning(submissionContext);
  } else {
    application = testCreateAppFinishing(submissionContext);
  }
  // RUNNING/FINISHING => FINISHED event RMAppEventType.ATTEMPT_FINISHED
  RMAppEvent finishedEvent = new RMAppFinishedAttemptEvent(
      application.getApplicationId(), diagnostics);
  application.handle(finishedEvent);
  assertAppState(RMAppState.FINISHED, application);
  assertTimesAtFinish(application);
  // finished without a proper unregister implies failed
  assertFinalAppStatus(FinalApplicationStatus.FAILED, application);
  Assert.assertTrue("Finished app missing diagnostics",
      application.getDiagnostics().indexOf(diagnostics) != -1);
  return application;
}
 

@Override
protected void submitApplication(
    ApplicationSubmissionContext submissionContext, long submitTime,
    String user) throws YarnException {
  //Do nothing, just add the application to RMContext
  RMAppImpl application =
      new RMAppImpl(submissionContext.getApplicationId(), this.rmContext,
          this.conf, submissionContext.getApplicationName(), user,
          submissionContext.getQueue(), submissionContext,
          this.rmContext.getScheduler(),
          this.rmContext.getApplicationMasterService(),
          submitTime, submissionContext.getApplicationType(),
          submissionContext.getApplicationTags(), null);
  this.rmContext.getRMApps().put(submissionContext.getApplicationId(),
      application);
  //Do not send RMAppEventType.START event
  //so the state of Application will not reach to NEW_SAVING state.
}
 

public RMAppAttemptImpl(ApplicationAttemptId appAttemptId,
    RMContext rmContext, YarnScheduler scheduler,
    ApplicationMasterService masterService,
    ApplicationSubmissionContext submissionContext,
    Configuration conf, boolean maybeLastAttempt, ResourceRequest amReq) {
  this.conf = conf;
  this.applicationAttemptId = appAttemptId;
  this.rmContext = rmContext;
  this.eventHandler = rmContext.getDispatcher().getEventHandler();
  this.submissionContext = submissionContext;
  this.scheduler = scheduler;
  this.masterService = masterService;

  ReentrantReadWriteLock lock = new ReentrantReadWriteLock();
  this.readLock = lock.readLock();
  this.writeLock = lock.writeLock();

  this.proxiedTrackingUrl = generateProxyUriWithScheme();
  this.maybeLastAttempt = maybeLastAttempt;
  this.stateMachine = stateMachineFactory.make(this);

  this.attemptMetrics =
      new RMAppAttemptMetrics(applicationAttemptId, rmContext);
  
  this.amReq = amReq;
}
 

private void setApplicationTags(final ApplicationSubmissionContext appContext) throws InvocationTargetException,
	IllegalAccessException {

	final ApplicationSubmissionContextReflector reflector = ApplicationSubmissionContextReflector.getInstance();
	final String tagsString = flinkConfiguration.getString(YarnConfigOptions.APPLICATION_TAGS);

	final Set<String> applicationTags = new HashSet<>();

	// Trim whitespace and cull empty tags
	for (final String tag : tagsString.split(",")) {
		final String trimmedTag = tag.trim();
		if (!trimmedTag.isEmpty()) {
			applicationTags.add(trimmedTag);
		}
	}

	reflector.setApplicationTags(appContext, applicationTags);
}
 

private ApplicationSubmissionContext setupApplicationContext() throws IOException, YarnException {
  TezClientUtils.processTezLocalCredentialsFile(sessionCredentials,
          amConfig.getTezConfiguration());

  Map<String, LocalResource> tezJarResources = getTezJarResources(sessionCredentials);
  // Add session token for shuffle
  TezClientUtils.createSessionToken(sessionAppId.toString(),
          jobTokenSecretManager, sessionCredentials);

  ApplicationSubmissionContext appContext =
          TezClientUtils.createApplicationSubmissionContext(
                  sessionAppId,
                  null, clientName, amConfig,
                  tezJarResources, sessionCredentials, usingTezArchiveDeploy, apiVersionInfo,
                  servicePluginsDescriptor, javaOptsChecker);

  // Set Tez Sessions to not retry on AM crashes if recovery is disabled
  if (!amConfig.getTezConfiguration().getBoolean(
          TezConfiguration.DAG_RECOVERY_ENABLED,
          TezConfiguration.DAG_RECOVERY_ENABLED_DEFAULT)) {
    appContext.setMaxAppAttempts(1);
  }
  return appContext;
}
 

private void setApplicationTags(final ApplicationSubmissionContext appContext) throws InvocationTargetException,
		IllegalAccessException {

	final ApplicationSubmissionContextReflector reflector = ApplicationSubmissionContextReflector.getInstance();
	final String tagsString = flinkConfiguration.getString(YarnConfigOptions.APPLICATION_TAGS);

	final Set<String> applicationTags = new HashSet<>();

	// Trim whitespace and cull empty tags
	for (final String tag : tagsString.split(",")) {
		final String trimmedTag = tag.trim();
		if (!trimmedTag.isEmpty()) {
			applicationTags.add(trimmedTag);
		}
	}

	reflector.setApplicationTags(appContext, applicationTags);
}
 

@Test(timeout=20000)
public void testAMProfiler() throws Exception {
  JobConf jobConf = new JobConf();

  jobConf.setBoolean(MRJobConfig.MR_AM_PROFILE, true);

  YARNRunner yarnRunner = new YARNRunner(jobConf);

  ApplicationSubmissionContext submissionContext =
      buildSubmitContext(yarnRunner, jobConf);

  ContainerLaunchContext containerSpec = submissionContext.getAMContainerSpec();
  List<String> commands = containerSpec.getCommands();

  for(String command : commands) {
    if (command != null) {
      if (command.contains(PROFILE_PARAMS)) {
        return;
      }
    }
  }
  throw new IllegalStateException("Profiler opts not found!");
}
 

protected RMApp storeApp(RMStateStore store, ApplicationId appId,
    long submitTime,
    long startTime) throws Exception {
  ApplicationSubmissionContext context =
      new ApplicationSubmissionContextPBImpl();
  context.setApplicationId(appId);

  RMApp mockApp = mock(RMApp.class);
  when(mockApp.getApplicationId()).thenReturn(appId);
  when(mockApp.getSubmitTime()).thenReturn(submitTime);
  when(mockApp.getStartTime()).thenReturn(startTime);
  when(mockApp.getApplicationSubmissionContext()).thenReturn(context);
  when(mockApp.getUser()).thenReturn("test");
  store.storeNewApplication(mockApp);
  return mockApp;
}
 

@Test(timeout = 2000)
// this test checks if the priority field is set properly in the
// ApplicationSubmissionContext
public void testAppSubmissionContextForPriority() throws Exception {
  TezConfiguration tezConf = new TezConfiguration();
  tezConf.set(TezConfiguration.TEZ_AM_STAGING_DIR, STAGING_DIR.getAbsolutePath());
  int testpriority = 999;
  ApplicationId appId = ApplicationId.newInstance(1000, 1);
  Credentials credentials = new Credentials();
  TezClientUtils.createSessionToken(appId.toString(),
      new JobTokenSecretManager(), credentials);
  tezConf.setBoolean(TezConfiguration.TEZ_IGNORE_LIB_URIS, true);
  Map<String, LocalResource> m = new HashMap<String, LocalResource>();
  tezConf.setInt(TezConfiguration.TEZ_AM_APPLICATION_PRIORITY, testpriority);
  AMConfiguration amConf =
      new AMConfiguration(tezConf, new HashMap<String, LocalResource>(), credentials);
  ApplicationSubmissionContext appcontext;
  appcontext = TezClientUtils.createApplicationSubmissionContext(
      appId, null, "dagname",
      amConf, m,
      credentials, false,
      new TezApiVersionInfo(), null, null);
  assertEquals(testpriority, appcontext.getPriority().getPriority());
}
 

protected RMApp createNewTestApp(ApplicationSubmissionContext submissionContext) {
  ApplicationId applicationId = MockApps.newAppID(appId++);
  String user = MockApps.newUserName();
  String name = MockApps.newAppName();
  String queue = MockApps.newQueue();
  // ensure max application attempts set to known value
  conf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, maxAppAttempts);
  scheduler = mock(YarnScheduler.class);

  ApplicationMasterService masterService =
      new ApplicationMasterService(rmContext, scheduler);
  
  if(submissionContext == null) {
    submissionContext = new ApplicationSubmissionContextPBImpl();
  }
  // applicationId will not be used because RMStateStore is mocked,
  // but applicationId is still set for safety
  submissionContext.setApplicationId(applicationId);

  RMApp application =
      new RMAppImpl(applicationId, rmContext, conf, name, user, queue,
        submissionContext, scheduler, masterService,
        System.currentTimeMillis(), "YARN", null, null);

  testAppStartState(applicationId, user, name, queue, application);
  this.rmContext.getRMApps().putIfAbsent(application.getApplicationId(),
      application);
  return application;
}
 

protected RMApp testCreateAppAccepted(
    ApplicationSubmissionContext submissionContext) throws IOException {
  RMApp application = testCreateAppSubmittedNoRecovery(submissionContext);
// SUBMITTED => ACCEPTED event RMAppEventType.APP_ACCEPTED
  RMAppEvent event = 
      new RMAppEvent(application.getApplicationId(), 
          RMAppEventType.APP_ACCEPTED);
  application.handle(event);
  assertStartTimeSet(application);
  assertAppState(RMAppState.ACCEPTED, application);
  return application;
}
 

@Test(timeout=20000)
public void testAppSubmissionWithInvalidDelegationToken() throws Exception {
  Configuration conf = new Configuration();
  conf.set(
      CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION,
      "kerberos");
  UserGroupInformation.setConfiguration(conf);
  MockRM rm = new MockRM(conf) {
    @Override
    protected void doSecureLogin() throws IOException {
      // Skip the login.
    }
  };
  ByteBuffer tokens = ByteBuffer.wrap("BOGUS".getBytes()); 
  ContainerLaunchContext amContainer =
      ContainerLaunchContext.newInstance(
          new HashMap<String, LocalResource>(), new HashMap<String, String>(),
          new ArrayList<String>(), new HashMap<String, ByteBuffer>(), tokens,
          new HashMap<ApplicationAccessType, String>());
  ApplicationSubmissionContext appSubContext =
      ApplicationSubmissionContext.newInstance(
          ApplicationId.newInstance(1234121, 0),
          "BOGUS", "default", Priority.UNDEFINED, amContainer, false,
          true, 1, Resource.newInstance(1024, 1), "BOGUS");
  SubmitApplicationRequest request =
      SubmitApplicationRequest.newInstance(appSubContext);
  try {
    rm.getClientRMService().submitApplication(request);
    fail("Error was excepted.");
  } catch (YarnException e) {
    Assert.assertTrue(e.getMessage().contains(
        "Bad header found in token storage"));
  }
}
 

@Test
public void testNodeLabelExp() throws Exception {
  JobConf jobConf = new JobConf();

  jobConf.set(MRJobConfig.JOB_NODE_LABEL_EXP, "GPU");
  jobConf.set(MRJobConfig.AM_NODE_LABEL_EXP, "highMem");

  YARNRunner yarnRunner = new YARNRunner(jobConf);
  ApplicationSubmissionContext appSubCtx =
      buildSubmitContext(yarnRunner, jobConf);

  assertEquals(appSubCtx.getNodeLabelExpression(), "GPU");
  assertEquals(appSubCtx.getAMContainerResourceRequest()
      .getNodeLabelExpression(), "highMem");
}
 

protected void createApplicationWithAMResource(ApplicationAttemptId attId,
    String queue, String user, Resource amResource) {
  RMContext rmContext = resourceManager.getRMContext();
  RMApp rmApp = new RMAppImpl(attId.getApplicationId(), rmContext, conf,
      null, null, null, ApplicationSubmissionContext.newInstance(null, null,
      null, null, null, false, false, 0, amResource, null), null, null,
      0, null, null, null);
  rmContext.getRMApps().put(attId.getApplicationId(), rmApp);
  AppAddedSchedulerEvent appAddedEvent = new AppAddedSchedulerEvent(
      attId.getApplicationId(), queue, user);
  scheduler.handle(appAddedEvent);
  AppAttemptAddedSchedulerEvent attempAddedEvent =
      new AppAttemptAddedSchedulerEvent(attId, false);
  scheduler.handle(attempAddedEvent);
}
 

protected RMApp testCreateAppFinalSaving(
    ApplicationSubmissionContext submissionContext) throws IOException {
  RMApp application = testCreateAppRunning(submissionContext);
  RMAppEvent finishingEvent =
      new RMAppEvent(application.getApplicationId(),
        RMAppEventType.ATTEMPT_UNREGISTERED);
  application.handle(finishingEvent);
  assertAppState(RMAppState.FINAL_SAVING, application);
  assertAppFinalStateSaved(application);
  return application;
}
 

@Override
public YarnClientApplication createApplication()
    throws YarnException, IOException {
  ApplicationSubmissionContext context = Records.newRecord
      (ApplicationSubmissionContext.class);
  GetNewApplicationResponse newApp = getNewApplication();
  ApplicationId appId = newApp.getApplicationId();
  context.setApplicationId(appId);
  return new YarnClientApplication(newApp, context);
}
 

@Override
public void setApplicationSubmissionContext(ApplicationSubmissionContext applicationSubmissionContext) {
  maybeInitBuilder();
  if (applicationSubmissionContext == null) 
    builder.clearApplicationSubmissionContext();
  this.applicationSubmissionContext = applicationSubmissionContext;
}
 

@Override
public ApplicationSubmissionContext getApplicationSubmissionContext() {
  SubmitApplicationRequestProtoOrBuilder p = viaProto ? proto : builder;
  if (this.applicationSubmissionContext != null) {
    return this.applicationSubmissionContext;
  }
  if (!p.hasApplicationSubmissionContext()) {
    return null;
  }
  this.applicationSubmissionContext = convertFromProtoFormat(p.getApplicationSubmissionContext());
  return this.applicationSubmissionContext;
}
 

protected RMApp testCreateAppAccepted(
    ApplicationSubmissionContext submissionContext) throws IOException {
  RMApp application = testCreateAppSubmittedNoRecovery(submissionContext);
// SUBMITTED => ACCEPTED event RMAppEventType.APP_ACCEPTED
  RMAppEvent event = 
      new RMAppEvent(application.getApplicationId(), 
          RMAppEventType.APP_ACCEPTED);
  application.handle(event);
  assertStartTimeSet(application);
  assertAppState(RMAppState.ACCEPTED, application);
  return application;
}
 

private void activateHighAvailabilitySupport(ApplicationSubmissionContext appContext) throws
		InvocationTargetException, IllegalAccessException {

	ApplicationSubmissionContextReflector reflector = ApplicationSubmissionContextReflector.getInstance();

	reflector.setKeepContainersAcrossApplicationAttempts(appContext, true);

	reflector.setAttemptFailuresValidityInterval(
			appContext,
			flinkConfiguration.getLong(YarnConfigOptions.APPLICATION_ATTEMPT_FAILURE_VALIDITY_INTERVAL));
}
 

@Override
public ApplicationSubmissionContext getApplicationSubmissionContext() {
  ApplicationStateDataProtoOrBuilder p = viaProto ? proto : builder;
  if(applicationSubmissionContext != null) {
    return applicationSubmissionContext;
  }
  if (!p.hasApplicationSubmissionContext()) {
    return null;
  }
  applicationSubmissionContext = 
      new ApplicationSubmissionContextPBImpl(
          p.getApplicationSubmissionContext());
  return applicationSubmissionContext;
}
 

@Test(timeout=1000)
public void testSetApplicationTags() {
  TezConfiguration conf = new TezConfiguration(false);
  conf.set(TezConfiguration.TEZ_APPLICATION_TAGS, "foo,bar");
  AMConfiguration amconfig = new AMConfiguration(conf, null, null);
  ApplicationSubmissionContext appContext = Records
      .newRecord(ApplicationSubmissionContext.class);
  Collection<String> tagsFromConf =
      amconfig.getTezConfiguration().getTrimmedStringCollection(
      TezConfiguration.TEZ_APPLICATION_TAGS);
  appContext.setApplicationTags(new HashSet<String>(tagsFromConf));
  assertTrue(appContext.getApplicationTags().contains("foo"));
  assertTrue(appContext.getApplicationTags().contains("bar"));
}
 

/**
 * Start the client. This establishes a connection to the YARN cluster.
 * In session mode, this start the App Master thats runs all the DAGs in the
 * session.
 * @throws TezException
 * @throws IOException
 */
public synchronized void start() throws TezException, IOException {
  amConfig.setYarnConfiguration(new YarnConfiguration(amConfig.getTezConfiguration()));
  startFrameworkClient();
  setupJavaOptsChecker();

  if (isSession) {
    LOG.info("Session mode. Starting session.");
    TezClientUtils.processTezLocalCredentialsFile(sessionCredentials,
        amConfig.getTezConfiguration());

    clientTimeout = amConfig.getTezConfiguration().getInt(
        TezConfiguration.TEZ_SESSION_CLIENT_TIMEOUT_SECS,
        TezConfiguration.TEZ_SESSION_CLIENT_TIMEOUT_SECS_DEFAULT);

    try {
      if (sessionAppId == null) {
        sessionAppId = createApplication();
      }

      ApplicationSubmissionContext appContext = setupApplicationContext();
      frameworkClient.submitApplication(appContext);
      ApplicationReport appReport = frameworkClient.getApplicationReport(sessionAppId);
      LOG.info("The url to track the Tez Session: " + appReport.getTrackingUrl());
      sessionStarted.set(true);
    } catch (YarnException e) {
      throw new TezException(e);
    }

    startClientHeartbeat();
    this.stagingFs = FileSystem.get(amConfig.getTezConfiguration());
  }
}
 

@Test
public void testDuplicateRMAppDeletion() throws Exception {
  TestZKRMStateStoreTester zkTester = new TestZKRMStateStoreTester();
  long submitTime = System.currentTimeMillis();
  long startTime = System.currentTimeMillis() + 1234;
  RMStateStore store = zkTester.getRMStateStore();
  TestDispatcher dispatcher = new TestDispatcher();
  store.setRMDispatcher(dispatcher);

  ApplicationAttemptId attemptIdRemoved = ConverterUtils
      .toApplicationAttemptId("appattempt_1352994193343_0002_000001");
  ApplicationId appIdRemoved = attemptIdRemoved.getApplicationId();
  storeApp(store, appIdRemoved, submitTime, startTime);
  storeAttempt(store, attemptIdRemoved,
      "container_1352994193343_0002_01_000001", null, null, dispatcher);

  ApplicationSubmissionContext context =
      new ApplicationSubmissionContextPBImpl();
  context.setApplicationId(appIdRemoved);
  ApplicationStateData appStateRemoved =
      ApplicationStateData.newInstance(
          submitTime, startTime, context, "user1");
  appStateRemoved.attempts.put(attemptIdRemoved, null);
  store.removeApplicationStateInternal(appStateRemoved);
  try {
    store.removeApplicationStateInternal(appStateRemoved);
  } catch (KeeperException.NoNodeException nne) {
    Assert.fail("NoNodeException should not happen.");
  }
  store.close();
}