org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppEvent Java Examples

The following examples show how to use org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppEvent. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: DelegationTokenRenewer.java    From hadoop with Apache License 2.0 6 votes vote down vote up
@SuppressWarnings("unchecked")
private void handleDTRenewerAppSubmitEvent(
    DelegationTokenRenewerAppSubmitEvent event) {
  /*
   * For applications submitted with delegation tokens we are not submitting
   * the application to scheduler from RMAppManager. Instead we are doing
   * it from here. The primary goal is to make token renewal as a part of
   * application submission asynchronous so that client thread is not
   * blocked during app submission.
   */
  try {
    // Setup tokens for renewal
    DelegationTokenRenewer.this.handleAppSubmitEvent(event);
    rmContext.getDispatcher().getEventHandler()
        .handle(new RMAppEvent(event.getApplicationId(), RMAppEventType.START));
  } catch (Throwable t) {
    LOG.warn(
        "Unable to add the application to the delegation token renewer.",
        t);
    // Sending APP_REJECTED is fine, since we assume that the
    // RMApp is in NEW state and thus we havne't yet informed the
    // Scheduler about the existence of the application
    rmContext.getDispatcher().getEventHandler().handle(
        new RMAppRejectedEvent(event.getApplicationId(), t.getMessage()));
  }
}
 
Example #2
Source File: RMStateStore.java    From big-c with Apache License 2.0 6 votes vote down vote up
@Override
public void transition(RMStateStore store, RMStateStoreEvent event) {
  if (!(event instanceof RMStateUpdateAppEvent)) {
    // should never happen
    LOG.error("Illegal event type: " + event.getClass());
    return;
  }
  ApplicationStateData appState =
      ((RMStateUpdateAppEvent) event).getAppState();
  ApplicationId appId =
      appState.getApplicationSubmissionContext().getApplicationId();
  LOG.info("Updating info for app: " + appId);
  try {
    store.updateApplicationStateInternal(appId, appState);
    store.notifyApplication(new RMAppEvent(appId,
           RMAppEventType.APP_UPDATE_SAVED));
  } catch (Exception e) {
    LOG.error("Error updating app: " + appId, e);
    store.notifyStoreOperationFailed(e);
  }
}
 
Example #3
Source File: RMStateStore.java    From big-c with Apache License 2.0 6 votes vote down vote up
@Override
public void transition(RMStateStore store, RMStateStoreEvent event) {
  if (!(event instanceof RMStateStoreAppEvent)) {
    // should never happen
    LOG.error("Illegal event type: " + event.getClass());
    return;
  }
  ApplicationStateData appState =
      ((RMStateStoreAppEvent) event).getAppState();
  ApplicationId appId =
      appState.getApplicationSubmissionContext().getApplicationId();
  LOG.info("Storing info for app: " + appId);
  try {
    store.storeApplicationStateInternal(appId, appState);
    store.notifyApplication(new RMAppEvent(appId,
           RMAppEventType.APP_NEW_SAVED));
  } catch (Exception e) {
    LOG.error("Error storing app: " + appId, e);
    store.notifyStoreOperationFailed(e);
  }
}
 
Example #4
Source File: AbstractYarnScheduler.java    From big-c with Apache License 2.0 6 votes vote down vote up
@Override
public synchronized void killAllAppsInQueue(String queueName)
    throws YarnException {
  // check if queue is a valid
  List<ApplicationAttemptId> apps = getAppsInQueue(queueName);
  if (apps == null) {
    String errMsg = "The specified Queue: " + queueName + " doesn't exist";
    LOG.warn(errMsg);
    throw new YarnException(errMsg);
  }
  // generate kill events for each pending/running app
  for (ApplicationAttemptId app : apps) {
    this.rmContext
        .getDispatcher()
        .getEventHandler()
        .handle(new RMAppEvent(app.getApplicationId(), RMAppEventType.KILL));
  }
}
 
Example #5
Source File: FifoScheduler.java    From big-c with Apache License 2.0 6 votes vote down vote up
@VisibleForTesting
public synchronized void addApplication(ApplicationId applicationId,
    String queue, String user, boolean isAppRecovering) {
  SchedulerApplication<FiCaSchedulerApp> application =
      new SchedulerApplication<FiCaSchedulerApp>(DEFAULT_QUEUE, user);
  applications.put(applicationId, application);
  metrics.submitApp(user);
  LOG.info("Accepted application " + applicationId + " from user: " + user
      + ", currently num of applications: " + applications.size());
  if (isAppRecovering) {
    if (LOG.isDebugEnabled()) {
      LOG.debug(applicationId + " is recovering. Skip notifying APP_ACCEPTED");
    }
  } else {
    rmContext.getDispatcher().getEventHandler()
      .handle(new RMAppEvent(applicationId, RMAppEventType.APP_ACCEPTED));
  }
}
 
Example #6
Source File: DelegationTokenRenewer.java    From big-c with Apache License 2.0 6 votes vote down vote up
@SuppressWarnings("unchecked")
private void handleDTRenewerAppSubmitEvent(
    DelegationTokenRenewerAppSubmitEvent event) {
  /*
   * For applications submitted with delegation tokens we are not submitting
   * the application to scheduler from RMAppManager. Instead we are doing
   * it from here. The primary goal is to make token renewal as a part of
   * application submission asynchronous so that client thread is not
   * blocked during app submission.
   */
  try {
    // Setup tokens for renewal
    DelegationTokenRenewer.this.handleAppSubmitEvent(event);
    rmContext.getDispatcher().getEventHandler()
        .handle(new RMAppEvent(event.getApplicationId(), RMAppEventType.START));
  } catch (Throwable t) {
    LOG.warn(
        "Unable to add the application to the delegation token renewer.",
        t);
    // Sending APP_REJECTED is fine, since we assume that the
    // RMApp is in NEW state and thus we havne't yet informed the
    // Scheduler about the existence of the application
    rmContext.getDispatcher().getEventHandler().handle(
        new RMAppRejectedEvent(event.getApplicationId(), t.getMessage()));
  }
}
 
Example #7
Source File: RMStateStore.java    From hadoop with Apache License 2.0 6 votes vote down vote up
@Override
public void transition(RMStateStore store, RMStateStoreEvent event) {
  if (!(event instanceof RMStateUpdateAppEvent)) {
    // should never happen
    LOG.error("Illegal event type: " + event.getClass());
    return;
  }
  ApplicationStateData appState =
      ((RMStateUpdateAppEvent) event).getAppState();
  ApplicationId appId =
      appState.getApplicationSubmissionContext().getApplicationId();
  LOG.info("Updating info for app: " + appId);
  try {
    store.updateApplicationStateInternal(appId, appState);
    store.notifyApplication(new RMAppEvent(appId,
           RMAppEventType.APP_UPDATE_SAVED));
  } catch (Exception e) {
    LOG.error("Error updating app: " + appId, e);
    store.notifyStoreOperationFailed(e);
  }
}
 
Example #8
Source File: RMStateStore.java    From hadoop with Apache License 2.0 6 votes vote down vote up
@Override
public void transition(RMStateStore store, RMStateStoreEvent event) {
  if (!(event instanceof RMStateStoreAppEvent)) {
    // should never happen
    LOG.error("Illegal event type: " + event.getClass());
    return;
  }
  ApplicationStateData appState =
      ((RMStateStoreAppEvent) event).getAppState();
  ApplicationId appId =
      appState.getApplicationSubmissionContext().getApplicationId();
  LOG.info("Storing info for app: " + appId);
  try {
    store.storeApplicationStateInternal(appId, appState);
    store.notifyApplication(new RMAppEvent(appId,
           RMAppEventType.APP_NEW_SAVED));
  } catch (Exception e) {
    LOG.error("Error storing app: " + appId, e);
    store.notifyStoreOperationFailed(e);
  }
}
 
Example #9
Source File: FifoScheduler.java    From hadoop with Apache License 2.0 6 votes vote down vote up
@VisibleForTesting
public synchronized void addApplication(ApplicationId applicationId,
    String queue, String user, boolean isAppRecovering) {
  SchedulerApplication<FiCaSchedulerApp> application =
      new SchedulerApplication<FiCaSchedulerApp>(DEFAULT_QUEUE, user);
  applications.put(applicationId, application);
  metrics.submitApp(user);
  LOG.info("Accepted application " + applicationId + " from user: " + user
      + ", currently num of applications: " + applications.size());
  if (isAppRecovering) {
    if (LOG.isDebugEnabled()) {
      LOG.debug(applicationId + " is recovering. Skip notifying APP_ACCEPTED");
    }
  } else {
    rmContext.getDispatcher().getEventHandler()
      .handle(new RMAppEvent(applicationId, RMAppEventType.APP_ACCEPTED));
  }
}
 
Example #10
Source File: AbstractYarnScheduler.java    From hadoop with Apache License 2.0 6 votes vote down vote up
@Override
public synchronized void killAllAppsInQueue(String queueName)
    throws YarnException {
  // check if queue is a valid
  List<ApplicationAttemptId> apps = getAppsInQueue(queueName);
  if (apps == null) {
    String errMsg = "The specified Queue: " + queueName + " doesn't exist";
    LOG.warn(errMsg);
    throw new YarnException(errMsg);
  }
  // generate kill events for each pending/running app
  for (ApplicationAttemptId app : apps) {
    this.rmContext
        .getDispatcher()
        .getEventHandler()
        .handle(new RMAppEvent(app.getApplicationId(), RMAppEventType.KILL));
  }
}
 
Example #11
Source File: TestRMAppAttemptTransitions.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Override
public void handle(RMAppEvent event) {
  assertEquals(application.getApplicationId(), event.getApplicationId());
  if (event instanceof RMAppFailedAttemptEvent) {
    transferStateFromPreviousAttempt =
        ((RMAppFailedAttemptEvent) event)
          .getTransferStateFromPreviousAttempt();
  }
  try {
    application.handle(event);
  } catch (Throwable t) {
    LOG.error("Error in handling event type " + event.getType()
        + " for application " + application.getApplicationId(), t);
  }
}
 
Example #12
Source File: RMAppAttemptImpl.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Override
public void transition(RMAppAttemptImpl appAttempt,
    RMAppAttemptEvent event) {
  long delay = System.currentTimeMillis() - appAttempt.launchAMEndTime;
  ClusterMetrics.getMetrics().addAMRegisterDelay(delay);
  RMAppAttemptRegistrationEvent registrationEvent
      = (RMAppAttemptRegistrationEvent) event;
  appAttempt.host = registrationEvent.getHost();
  appAttempt.rpcPort = registrationEvent.getRpcport();
  appAttempt.originalTrackingUrl =
      sanitizeTrackingUrl(registrationEvent.getTrackingurl());

  // Let the app know
  appAttempt.eventHandler.handle(new RMAppEvent(appAttempt
      .getAppAttemptId().getApplicationId(),
      RMAppEventType.ATTEMPT_REGISTERED));

  // TODO:FIXME: Note for future. Unfortunately we only do a state-store
  // write at AM launch time, so we don't save the AM's tracking URL anywhere
  // as that would mean an extra state-store write. For now, we hope that in
  // work-preserving restart, AMs are forced to reregister.

  appAttempt.rmContext.getRMApplicationHistoryWriter()
      .applicationAttemptStarted(appAttempt);
  appAttempt.rmContext.getSystemMetricsPublisher()
      .appAttemptRegistered(appAttempt, System.currentTimeMillis());
}
 
Example #13
Source File: RMAppAttemptImpl.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Override
public RMAppAttemptState transition(RMAppAttemptImpl appAttempt,
    RMAppAttemptEvent event) {
  // Tell the app
  if (appAttempt.getSubmissionContext().getUnmanagedAM()) {
    // Unmanaged AMs have no container to wait for, so they skip
    // the FINISHING state and go straight to FINISHED.
    appAttempt.updateInfoOnAMUnregister(event);
    new FinalTransition(RMAppAttemptState.FINISHED).transition(
        appAttempt, event);
    return RMAppAttemptState.FINISHED;
  }
  // Saving the attempt final state
  appAttempt.rememberTargetTransitionsAndStoreState(event,
    new FinalStateSavedAfterAMUnregisterTransition(),
    RMAppAttemptState.FINISHING, RMAppAttemptState.FINISHED);
  ApplicationId applicationId =
      appAttempt.getAppAttemptId().getApplicationId();

  // Tell the app immediately that AM is unregistering so that app itself
  // can save its state as soon as possible. Whether we do it like this, or
  // we wait till AppAttempt is saved, it doesn't make any difference on the
  // app side w.r.t failure conditions. The only event going out of
  // AppAttempt to App after this point of time is AM/AppAttempt Finished.
  appAttempt.eventHandler.handle(new RMAppEvent(applicationId,
    RMAppEventType.ATTEMPT_UNREGISTERED));
  return RMAppAttemptState.FINAL_SAVING;
}
 
Example #14
Source File: ResourceManager.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Override
public void handle(RMAppEvent event) {
  ApplicationId appID = event.getApplicationId();
  RMApp rmApp = this.rmContext.getRMApps().get(appID);
  if (rmApp != null) {
    try {
      rmApp.handle(event);
    } catch (Throwable t) {
      LOG.error("Error in handling event type " + event.getType()
          + " for application " + appID, t);
    }
  }
}
 
Example #15
Source File: RMStateStore.java    From big-c with Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("unchecked")
/**
 * This method is called to notify the application that
 * new application is stored or updated in state store
 * @param event App event containing the app id and event type
 */
private void notifyApplication(RMAppEvent event) {
  rmDispatcher.getEventHandler().handle(event);
}
 
Example #16
Source File: ResourceManager.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Override
public void handle(RMAppEvent event) {
  ApplicationId appID = event.getApplicationId();
  RMApp rmApp = this.rmContext.getRMApps().get(appID);
  if (rmApp != null) {
    try {
      rmApp.handle(event);
    } catch (Throwable t) {
      LOG.error("Error in handling event type " + event.getType()
          + " for application " + appID, t);
    }
  }
}
 
Example #17
Source File: RMAppAttemptImpl.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Override
public RMAppAttemptState transition(RMAppAttemptImpl appAttempt,
    RMAppAttemptEvent event) {
  // Tell the app
  if (appAttempt.getSubmissionContext().getUnmanagedAM()) {
    // Unmanaged AMs have no container to wait for, so they skip
    // the FINISHING state and go straight to FINISHED.
    appAttempt.updateInfoOnAMUnregister(event);
    new FinalTransition(RMAppAttemptState.FINISHED).transition(
        appAttempt, event);
    return RMAppAttemptState.FINISHED;
  }
  // Saving the attempt final state
  appAttempt.rememberTargetTransitionsAndStoreState(event,
    new FinalStateSavedAfterAMUnregisterTransition(),
    RMAppAttemptState.FINISHING, RMAppAttemptState.FINISHED);
  ApplicationId applicationId =
      appAttempt.getAppAttemptId().getApplicationId();

  // Tell the app immediately that AM is unregistering so that app itself
  // can save its state as soon as possible. Whether we do it like this, or
  // we wait till AppAttempt is saved, it doesn't make any difference on the
  // app side w.r.t failure conditions. The only event going out of
  // AppAttempt to App after this point of time is AM/AppAttempt Finished.
  appAttempt.eventHandler.handle(new RMAppEvent(applicationId,
    RMAppEventType.ATTEMPT_UNREGISTERED));
  return RMAppAttemptState.FINAL_SAVING;
}
 
Example #18
Source File: RMAppAttemptImpl.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Override
public void transition(RMAppAttemptImpl appAttempt,
    RMAppAttemptEvent event) {
  long delay = System.currentTimeMillis() - appAttempt.launchAMEndTime;
  ClusterMetrics.getMetrics().addAMRegisterDelay(delay);
  RMAppAttemptRegistrationEvent registrationEvent
      = (RMAppAttemptRegistrationEvent) event;
  appAttempt.host = registrationEvent.getHost();
  appAttempt.rpcPort = registrationEvent.getRpcport();
  appAttempt.originalTrackingUrl =
      sanitizeTrackingUrl(registrationEvent.getTrackingurl());

  // Let the app know
  appAttempt.eventHandler.handle(new RMAppEvent(appAttempt
      .getAppAttemptId().getApplicationId(),
      RMAppEventType.ATTEMPT_REGISTERED));

  // TODO:FIXME: Note for future. Unfortunately we only do a state-store
  // write at AM launch time, so we don't save the AM's tracking URL anywhere
  // as that would mean an extra state-store write. For now, we hope that in
  // work-preserving restart, AMs are forced to reregister.

  appAttempt.rmContext.getRMApplicationHistoryWriter()
      .applicationAttemptStarted(appAttempt);
  appAttempt.rmContext.getSystemMetricsPublisher()
      .appAttemptRegistered(appAttempt, System.currentTimeMillis());
}
 
Example #19
Source File: TestRMAppAttemptTransitions.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Override
public void handle(RMAppEvent event) {
  assertEquals(application.getApplicationId(), event.getApplicationId());
  if (event instanceof RMAppFailedAttemptEvent) {
    transferStateFromPreviousAttempt =
        ((RMAppFailedAttemptEvent) event)
          .getTransferStateFromPreviousAttempt();
  }
  try {
    application.handle(event);
  } catch (Throwable t) {
    LOG.error("Error in handling event type " + event.getType()
        + " for application " + application.getApplicationId(), t);
  }
}
 
Example #20
Source File: TestDelegationTokenRenewer.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Test(timeout=60000)
public void testAppRejectionWithCancelledDelegationToken() throws Exception {
  MyFS dfs = (MyFS)FileSystem.get(conf);
  LOG.info("dfs="+(Object)dfs.hashCode() + ";conf="+conf.hashCode());

  MyToken token = dfs.getDelegationToken("user1");
  token.cancelToken();

  Credentials ts = new Credentials();
  ts.addToken(token.getKind(), token);
  
  // register the tokens for renewal
  ApplicationId appId =  BuilderUtils.newApplicationId(0, 0);
  delegationTokenRenewer.addApplicationAsync(appId, ts, true, "user");
  int waitCnt = 20;
  while (waitCnt-- >0) {
    if (!eventQueue.isEmpty()) {
      Event evt = eventQueue.take();
      if (evt.getType() == RMAppEventType.APP_REJECTED) {
        Assert.assertTrue(
            ((RMAppEvent) evt).getApplicationId().equals(appId));
        return;
      }
    } else {
      Thread.sleep(500);
    }
  }
  fail("App submission with a cancelled token should have failed");
}
 
Example #21
Source File: TestDelegationTokenRenewer.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Test(timeout=60000)
public void testAppRejectionWithCancelledDelegationToken() throws Exception {
  MyFS dfs = (MyFS)FileSystem.get(conf);
  LOG.info("dfs="+(Object)dfs.hashCode() + ";conf="+conf.hashCode());

  MyToken token = dfs.getDelegationToken("user1");
  token.cancelToken();

  Credentials ts = new Credentials();
  ts.addToken(token.getKind(), token);
  
  // register the tokens for renewal
  ApplicationId appId =  BuilderUtils.newApplicationId(0, 0);
  delegationTokenRenewer.addApplicationAsync(appId, ts, true, "user");
  int waitCnt = 20;
  while (waitCnt-- >0) {
    if (!eventQueue.isEmpty()) {
      Event evt = eventQueue.take();
      if (evt.getType() == RMAppEventType.APP_REJECTED) {
        Assert.assertTrue(
            ((RMAppEvent) evt).getApplicationId().equals(appId));
        return;
      }
    } else {
      Thread.sleep(500);
    }
  }
  fail("App submission with a cancelled token should have failed");
}
 
Example #22
Source File: RMStateStore.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("unchecked")
/**
 * This method is called to notify the application that
 * new application is stored or updated in state store
 * @param event App event containing the app id and event type
 */
private void notifyApplication(RMAppEvent event) {
  rmDispatcher.getEventHandler().handle(event);
}
 
Example #23
Source File: TestAppManager.java    From big-c with Apache License 2.0 4 votes vote down vote up
@Override
public void handle(RMAppEvent event) {
  //RMApp rmApp = this.rmContext.getRMApps().get(appID);
  setAppEventType(event.getType());
  System.out.println("in handle routine " + getAppEventType().toString());
}
 
Example #24
Source File: TestDelegationTokenRenewer.java    From big-c with Apache License 2.0 4 votes vote down vote up
/**
 * Basic idea of the test:
 * 0. Setup token KEEP_ALIVE
 * 1. create tokens.
 * 2. register them for renewal - to be cancelled on app complete
 * 3. Complete app.
 * 4. Verify token is alive within the KEEP_ALIVE time
 * 5. Verify token has been cancelled after the KEEP_ALIVE_TIME
 * @throws IOException
 * @throws URISyntaxException
 */
@Test(timeout=60000)
public void testDTKeepAlive1 () throws Exception {
  Configuration lconf = new Configuration(conf);
  lconf.setBoolean(YarnConfiguration.LOG_AGGREGATION_ENABLED, true);
  //Keep tokens alive for 6 seconds.
  lconf.setLong(YarnConfiguration.RM_NM_EXPIRY_INTERVAL_MS, 6000l);
  //Try removing tokens every second.
  lconf.setLong(
      YarnConfiguration.RM_DELAYED_DELEGATION_TOKEN_REMOVAL_INTERVAL_MS,
      1000l);
  DelegationTokenRenewer localDtr =
      createNewDelegationTokenRenewer(lconf, counter);
  RMContext mockContext = mock(RMContext.class);
  when(mockContext.getSystemCredentialsForApps()).thenReturn(
    new ConcurrentHashMap<ApplicationId, ByteBuffer>());
  ClientRMService mockClientRMService = mock(ClientRMService.class);
  when(mockContext.getClientRMService()).thenReturn(mockClientRMService);
  when(mockContext.getDelegationTokenRenewer()).thenReturn(
      localDtr);
  when(mockContext.getDispatcher()).thenReturn(dispatcher);
  InetSocketAddress sockAddr =
      InetSocketAddress.createUnresolved("localhost", 1234);
  when(mockClientRMService.getBindAddress()).thenReturn(sockAddr);
  localDtr.setRMContext(mockContext);
  localDtr.init(lconf);
  localDtr.start();
  
  MyFS dfs = (MyFS)FileSystem.get(lconf);
  LOG.info("dfs="+(Object)dfs.hashCode() + ";conf="+lconf.hashCode());
  
  Credentials ts = new Credentials();
  // get the delegation tokens
  MyToken token1 = dfs.getDelegationToken("user1");

  String nn1 = DelegationTokenRenewer.SCHEME + "://host1:0";
  ts.addToken(new Text(nn1), token1);

  // register the tokens for renewal
  ApplicationId applicationId_0 =  BuilderUtils.newApplicationId(0, 0);
  localDtr.addApplicationAsync(applicationId_0, ts, true, "user");
  waitForEventsToGetProcessed(localDtr);
  if (!eventQueue.isEmpty()){
    Event evt = eventQueue.take();
    if (evt instanceof RMAppEvent) {
      Assert.assertEquals(((RMAppEvent)evt).getType(), RMAppEventType.START);
    } else {
      fail("RMAppEvent.START was expected!!");
    }
  }
  
  localDtr.applicationFinished(applicationId_0);
  waitForEventsToGetProcessed(localDtr);

  //Token should still be around. Renewal should not fail.
  token1.renew(lconf);

  //Allow the keepalive time to run out
  Thread.sleep(10000l);

  //The token should have been cancelled at this point. Renewal will fail.
  try {
    token1.renew(lconf);
    fail("Renewal of cancelled token should have failed");
  } catch (InvalidToken ite) {}
}
 
Example #25
Source File: TestRM.java    From big-c with Apache License 2.0 4 votes vote down vote up
/**
 * Validate killing an application when it is at accepted state.
 * @throws Exception exception
 */
@Test (timeout = 60000)
public void testApplicationKillAtAcceptedState() throws Exception {

  final Dispatcher dispatcher = new AsyncDispatcher() {
    @Override
    public EventHandler getEventHandler() {

      class EventArgMatcher extends ArgumentMatcher<AbstractEvent> {
        @Override
        public boolean matches(Object argument) {
          if (argument instanceof RMAppAttemptEvent) {
            if (((RMAppAttemptEvent) argument).getType().equals(
              RMAppAttemptEventType.KILL)) {
              return true;
            }
          }
          return false;
        }
      }

      EventHandler handler = spy(super.getEventHandler());
      doNothing().when(handler).handle(argThat(new EventArgMatcher()));
      return handler;
    }
  };

  MockRM rm = new MockRM(conf) {
    @Override
    protected Dispatcher createDispatcher() {
      return dispatcher;
    }
  };

  // test metrics
  QueueMetrics metrics = rm.getResourceScheduler().getRootQueueMetrics();
  int appsKilled = metrics.getAppsKilled();
  int appsSubmitted = metrics.getAppsSubmitted();

  rm.start();
  
  MockNM nm1 =
      new MockNM("127.0.0.1:1234", 15120, rm.getResourceTrackerService());
  nm1.registerNode();

  // a failed app
  RMApp application = rm.submitApp(200);
  MockAM am = MockRM.launchAM(application, rm, nm1);
  am.waitForState(RMAppAttemptState.LAUNCHED);
  nm1.nodeHeartbeat(am.getApplicationAttemptId(), 1, ContainerState.RUNNING);
  rm.waitForState(application.getApplicationId(), RMAppState.ACCEPTED);

  // Now kill the application before new attempt is launched, the app report
  // returns the invalid AM host and port.
  KillApplicationRequest request =
      KillApplicationRequest.newInstance(application.getApplicationId());
  rm.getClientRMService().forceKillApplication(request);

  // Specific test for YARN-1689 follows
  // Now let's say a race causes AM to register now. This should not crash RM.
  am.registerAppAttempt(false);

  // We explicitly intercepted the kill-event to RMAppAttempt, so app should
  // still be in KILLING state.
  rm.waitForState(application.getApplicationId(), RMAppState.KILLING);
  // AM should now be in running
  rm.waitForState(am.getApplicationAttemptId(), RMAppAttemptState.RUNNING);

  // Simulate that appAttempt is killed.
  rm.getRMContext().getDispatcher().getEventHandler().handle(
      new RMAppEvent(application.getApplicationId(),
        RMAppEventType.ATTEMPT_KILLED));
  rm.waitForState(application.getApplicationId(), RMAppState.KILLED);

  // test metrics
  metrics = rm.getResourceScheduler().getRootQueueMetrics();
  Assert.assertEquals(appsKilled + 1, metrics.getAppsKilled());
  Assert.assertEquals(appsSubmitted + 1, metrics.getAppsSubmitted());
}
 
Example #26
Source File: MockAsm.java    From big-c with Apache License 2.0 4 votes vote down vote up
@Override
public void handle(RMAppEvent event) {
  throw new UnsupportedOperationException("Not supported yet.");
}
 
Example #27
Source File: RMAppAttemptImpl.java    From hadoop with Apache License 2.0 4 votes vote down vote up
@Override
public void transition(RMAppAttemptImpl appAttempt,
    RMAppAttemptEvent event) {
  ApplicationAttemptId appAttemptId = appAttempt.getAppAttemptId();

  // Tell the AMS. Unregister from the ApplicationMasterService
  appAttempt.masterService.unregisterAttempt(appAttemptId);

  // Tell the application and the scheduler
  ApplicationId applicationId = appAttemptId.getApplicationId();
  RMAppEvent appEvent = null;
  boolean keepContainersAcrossAppAttempts = false;
  switch (finalAttemptState) {
    case FINISHED:
    {
      appEvent = new RMAppFinishedAttemptEvent(applicationId,
          appAttempt.getDiagnostics());
    }
    break;
    case KILLED:
    {
      appAttempt.invalidateAMHostAndPort();
      appEvent =
          new RMAppFailedAttemptEvent(applicationId,
              RMAppEventType.ATTEMPT_KILLED,
              "Application killed by user.", false);
    }
    break;
    case FAILED:
    {
      appAttempt.invalidateAMHostAndPort();

      if (appAttempt.submissionContext
        .getKeepContainersAcrossApplicationAttempts()
          && !appAttempt.submissionContext.getUnmanagedAM()) {
        // See if we should retain containers for non-unmanaged applications
        if (!appAttempt.shouldCountTowardsMaxAttemptRetry()) {
          // Premption, hardware failures, NM resync doesn't count towards
          // app-failures and so we should retain containers.
          keepContainersAcrossAppAttempts = true;
        } else if (!appAttempt.maybeLastAttempt) {
          // Not preemption, hardware failures or NM resync.
          // Not last-attempt too - keep containers.
          keepContainersAcrossAppAttempts = true;
        }
      }
      appEvent =
          new RMAppFailedAttemptEvent(applicationId,
            RMAppEventType.ATTEMPT_FAILED, appAttempt.getDiagnostics(),
            keepContainersAcrossAppAttempts);

    }
    break;
    default:
    {
      LOG.error("Cannot get this state!! Error!!");
    }
    break;
  }

  appAttempt.eventHandler.handle(appEvent);
  appAttempt.eventHandler.handle(new AppAttemptRemovedSchedulerEvent(
    appAttemptId, finalAttemptState, keepContainersAcrossAppAttempts));
  appAttempt.removeCredentials(appAttempt);

  appAttempt.rmContext.getRMApplicationHistoryWriter()
      .applicationAttemptFinished(appAttempt, finalAttemptState);
  appAttempt.rmContext.getSystemMetricsPublisher()
      .appAttemptFinished(appAttempt, finalAttemptState,
          appAttempt.rmContext.getRMApps().get(
              appAttempt.applicationAttemptId.getApplicationId()),
          System.currentTimeMillis());
}
 
Example #28
Source File: ClientRMService.java    From big-c with Apache License 2.0 4 votes vote down vote up
@SuppressWarnings("unchecked")
@Override
public KillApplicationResponse forceKillApplication(
    KillApplicationRequest request) throws YarnException {

  ApplicationId applicationId = request.getApplicationId();

  UserGroupInformation callerUGI;
  try {
    callerUGI = UserGroupInformation.getCurrentUser();
  } catch (IOException ie) {
    LOG.info("Error getting UGI ", ie);
    RMAuditLogger.logFailure("UNKNOWN", AuditConstants.KILL_APP_REQUEST,
        "UNKNOWN", "ClientRMService" , "Error getting UGI",
        applicationId);
    throw RPCUtil.getRemoteException(ie);
  }

  RMApp application = this.rmContext.getRMApps().get(applicationId);
  if (application == null) {
    RMAuditLogger.logFailure(callerUGI.getUserName(),
        AuditConstants.KILL_APP_REQUEST, "UNKNOWN", "ClientRMService",
        "Trying to kill an absent application", applicationId);
    throw new ApplicationNotFoundException("Trying to kill an absent"
        + " application " + applicationId);
  }

  if (!checkAccess(callerUGI, application.getUser(),
      ApplicationAccessType.MODIFY_APP, application)) {
    RMAuditLogger.logFailure(callerUGI.getShortUserName(),
        AuditConstants.KILL_APP_REQUEST,
        "User doesn't have permissions to "
            + ApplicationAccessType.MODIFY_APP.toString(), "ClientRMService",
        AuditConstants.UNAUTHORIZED_USER, applicationId);
    throw RPCUtil.getRemoteException(new AccessControlException("User "
        + callerUGI.getShortUserName() + " cannot perform operation "
        + ApplicationAccessType.MODIFY_APP.name() + " on " + applicationId));
  }

  if (application.isAppFinalStateStored()) {
    RMAuditLogger.logSuccess(callerUGI.getShortUserName(),
        AuditConstants.KILL_APP_REQUEST, "ClientRMService", applicationId);
    return KillApplicationResponse.newInstance(true);
  }

  this.rmContext.getDispatcher().getEventHandler()
      .handle(new RMAppEvent(applicationId, RMAppEventType.KILL));

  // For UnmanagedAMs, return true so they don't retry
  return KillApplicationResponse.newInstance(
      application.getApplicationSubmissionContext().getUnmanagedAM());
}
 
Example #29
Source File: RMAppAttemptImpl.java    From big-c with Apache License 2.0 4 votes vote down vote up
@Override
public void transition(RMAppAttemptImpl appAttempt,
    RMAppAttemptEvent event) {
  ApplicationAttemptId appAttemptId = appAttempt.getAppAttemptId();

  // Tell the AMS. Unregister from the ApplicationMasterService
  appAttempt.masterService.unregisterAttempt(appAttemptId);

  // Tell the application and the scheduler
  ApplicationId applicationId = appAttemptId.getApplicationId();
  RMAppEvent appEvent = null;
  boolean keepContainersAcrossAppAttempts = false;
  switch (finalAttemptState) {
    case FINISHED:
    {
      appEvent = new RMAppFinishedAttemptEvent(applicationId,
          appAttempt.getDiagnostics());
    }
    break;
    case KILLED:
    {
      appAttempt.invalidateAMHostAndPort();
      appEvent =
          new RMAppFailedAttemptEvent(applicationId,
              RMAppEventType.ATTEMPT_KILLED,
              "Application killed by user.", false);
    }
    break;
    case FAILED:
    {
      appAttempt.invalidateAMHostAndPort();

      if (appAttempt.submissionContext
        .getKeepContainersAcrossApplicationAttempts()
          && !appAttempt.submissionContext.getUnmanagedAM()) {
        // See if we should retain containers for non-unmanaged applications
        if (!appAttempt.shouldCountTowardsMaxAttemptRetry()) {
          // Premption, hardware failures, NM resync doesn't count towards
          // app-failures and so we should retain containers.
          keepContainersAcrossAppAttempts = true;
        } else if (!appAttempt.maybeLastAttempt) {
          // Not preemption, hardware failures or NM resync.
          // Not last-attempt too - keep containers.
          keepContainersAcrossAppAttempts = true;
        }
      }
      appEvent =
          new RMAppFailedAttemptEvent(applicationId,
            RMAppEventType.ATTEMPT_FAILED, appAttempt.getDiagnostics(),
            keepContainersAcrossAppAttempts);

    }
    break;
    default:
    {
      LOG.error("Cannot get this state!! Error!!");
    }
    break;
  }

  appAttempt.eventHandler.handle(appEvent);
  appAttempt.eventHandler.handle(new AppAttemptRemovedSchedulerEvent(
    appAttemptId, finalAttemptState, keepContainersAcrossAppAttempts));
  appAttempt.removeCredentials(appAttempt);

  appAttempt.rmContext.getRMApplicationHistoryWriter()
      .applicationAttemptFinished(appAttempt, finalAttemptState);
  appAttempt.rmContext.getSystemMetricsPublisher()
      .appAttemptFinished(appAttempt, finalAttemptState,
          appAttempt.rmContext.getRMApps().get(
              appAttempt.applicationAttemptId.getApplicationId()),
          System.currentTimeMillis());
}
 
Example #30
Source File: MockAsm.java    From hadoop with Apache License 2.0 4 votes vote down vote up
@Override
public void handle(RMAppEvent event) {
  throw new UnsupportedOperationException("Not supported yet.");
}