Java Code Examples for org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp#getState()

The following examples show how to use org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp#getState() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: TestMoveApplication.java    From hadoop with Apache License 2.0 6 votes vote down vote up 
@Test
public void testMoveRejectedByScheduler() throws Exception {
  failMove = true;
  
  // Submit application
  Application application = new Application("user1", resourceManager);
  application.submit();

  // Wait for app to be accepted
  RMApp app = resourceManager.rmContext.getRMApps()
          .get(application.getApplicationId());
  while (app.getState() != RMAppState.ACCEPTED) {
    Thread.sleep(100);
  }

  ClientRMService clientRMService = resourceManager.getClientRMService();
  try {
    // FIFO scheduler does not support moves
    clientRMService.moveApplicationAcrossQueues(
        MoveApplicationAcrossQueuesRequest.newInstance(
            application.getApplicationId(), "newqueue"));
    fail("Should have hit exception");
  } catch (YarnException ex) {
    assertEquals("Move not supported", ex.getCause().getMessage());
  }
}
Example 2
Source File: TestMoveApplication.java    From big-c with Apache License 2.0 6 votes vote down vote up 
@Test
public void testMoveRejectedByScheduler() throws Exception {
  failMove = true;
  
  // Submit application
  Application application = new Application("user1", resourceManager);
  application.submit();

  // Wait for app to be accepted
  RMApp app = resourceManager.rmContext.getRMApps()
          .get(application.getApplicationId());
  while (app.getState() != RMAppState.ACCEPTED) {
    Thread.sleep(100);
  }

  ClientRMService clientRMService = resourceManager.getClientRMService();
  try {
    // FIFO scheduler does not support moves
    clientRMService.moveApplicationAcrossQueues(
        MoveApplicationAcrossQueuesRequest.newInstance(
            application.getApplicationId(), "newqueue"));
    fail("Should have hit exception");
  } catch (YarnException ex) {
    assertEquals("Move not supported", ex.getCause().getMessage());
  }
}
Example 3
Source File: RMAppManager.java    From hadoop with Apache License 2.0 5 votes vote down vote up 
protected void writeAuditLog(ApplicationId appId) {
  RMApp app = rmContext.getRMApps().get(appId);
  String operation = "UNKONWN";
  boolean success = false;
  switch (app.getState()) {
    case FAILED: 
      operation = AuditConstants.FINISH_FAILED_APP;
      break;
    case FINISHED:
      operation = AuditConstants.FINISH_SUCCESS_APP;
      success = true;
      break;
    case KILLED: 
      operation = AuditConstants.FINISH_KILLED_APP;
      success = true;
      break;
    default:
      break;
  }
  
  if (success) {
    RMAuditLogger.logSuccess(app.getUser(), operation,
        "RMAppManager", app.getApplicationId());
  } else {
    StringBuilder diag = app.getDiagnostics(); 
    String msg = diag == null ? null : diag.toString();
    RMAuditLogger.logFailure(app.getUser(), operation, msg, "RMAppManager",
        "App failed with state: " + app.getState(), appId);
  }
}
Example 4
Source File: TestAppManager.java    From hadoop with Apache License 2.0 5 votes vote down vote up 
protected void addToCompletedApps(TestRMAppManager appMonitor, RMContext rmContext) {
  for (RMApp app : rmContext.getRMApps().values()) {
    if (app.getState() == RMAppState.FINISHED
        || app.getState() == RMAppState.KILLED 
        || app.getState() == RMAppState.FAILED) {
      appMonitor.finishApplication(app.getApplicationId());
    }
  }
}
Example 5
Source File: TestRMRestart.java    From hadoop with Apache License 2.0 5 votes vote down vote up 
private void waitForTokensToBeRenewed(MockRM rm2) throws Exception {
  int waitCnt = 20;
  boolean atleastOneAppInNEWState = true;
  while (waitCnt-- > 0 && atleastOneAppInNEWState) {
    atleastOneAppInNEWState = false;
    for (RMApp rmApp : rm2.getRMContext().getRMApps().values()) {
      if (rmApp.getState() == RMAppState.NEW) {
        Thread.sleep(1000);
        atleastOneAppInNEWState = true;
        break;
      }
    }
  }
}
Example 6
Source File: TestMoveApplication.java    From hadoop with Apache License 2.0 5 votes vote down vote up 
@Test (timeout = 10000)
public void testMoveTooLate() throws Exception {
  // Submit application
  Application application = new Application("user1", resourceManager);
  ApplicationId appId = application.getApplicationId();
  application.submit();
  
  ClientRMService clientRMService = resourceManager.getClientRMService();
  // Kill the application
  clientRMService.forceKillApplication(
      KillApplicationRequest.newInstance(appId));
  RMApp rmApp = resourceManager.getRMContext().getRMApps().get(appId);
  // wait until it's dead
  while (rmApp.getState() != RMAppState.KILLED) {
    Thread.sleep(100);
  }
  
  try {
    clientRMService.moveApplicationAcrossQueues(
        MoveApplicationAcrossQueuesRequest.newInstance(appId, "newqueue"));
    fail("Should have hit exception");
  } catch (YarnException ex) {
    assertEquals(YarnException.class,
        ex.getClass());
    assertEquals("App in KILLED state cannot be moved.", ex.getMessage());
  }
}
Example 7
Source File: RMAppManager.java    From big-c with Apache License 2.0 5 votes vote down vote up 
protected void writeAuditLog(ApplicationId appId) {
  RMApp app = rmContext.getRMApps().get(appId);
  String operation = "UNKONWN";
  boolean success = false;
  switch (app.getState()) {
    case FAILED: 
      operation = AuditConstants.FINISH_FAILED_APP;
      break;
    case FINISHED:
      operation = AuditConstants.FINISH_SUCCESS_APP;
      success = true;
      break;
    case KILLED: 
      operation = AuditConstants.FINISH_KILLED_APP;
      success = true;
      break;
    default:
      break;
  }
  
  if (success) {
    RMAuditLogger.logSuccess(app.getUser(), operation,
        "RMAppManager", app.getApplicationId());
  } else {
    StringBuilder diag = app.getDiagnostics(); 
    String msg = diag == null ? null : diag.toString();
    RMAuditLogger.logFailure(app.getUser(), operation, msg, "RMAppManager",
        "App failed with state: " + app.getState(), appId);
  }
}
Example 8
Source File: TestAppManager.java    From big-c with Apache License 2.0 5 votes vote down vote up 
protected void addToCompletedApps(TestRMAppManager appMonitor, RMContext rmContext) {
  for (RMApp app : rmContext.getRMApps().values()) {
    if (app.getState() == RMAppState.FINISHED
        || app.getState() == RMAppState.KILLED 
        || app.getState() == RMAppState.FAILED) {
      appMonitor.finishApplication(app.getApplicationId());
    }
  }
}
Example 9
Source File: TestRMRestart.java    From big-c with Apache License 2.0 5 votes vote down vote up 
private void waitForTokensToBeRenewed(MockRM rm2) throws Exception {
  int waitCnt = 20;
  boolean atleastOneAppInNEWState = true;
  while (waitCnt-- > 0 && atleastOneAppInNEWState) {
    atleastOneAppInNEWState = false;
    for (RMApp rmApp : rm2.getRMContext().getRMApps().values()) {
      if (rmApp.getState() == RMAppState.NEW) {
        Thread.sleep(1000);
        atleastOneAppInNEWState = true;
        break;
      }
    }
  }
}
Example 10
Source File: TestMoveApplication.java    From big-c with Apache License 2.0 5 votes vote down vote up 
@Test (timeout = 10000)
public void testMoveTooLate() throws Exception {
  // Submit application
  Application application = new Application("user1", resourceManager);
  ApplicationId appId = application.getApplicationId();
  application.submit();
  
  ClientRMService clientRMService = resourceManager.getClientRMService();
  // Kill the application
  clientRMService.forceKillApplication(
      KillApplicationRequest.newInstance(appId));
  RMApp rmApp = resourceManager.getRMContext().getRMApps().get(appId);
  // wait until it's dead
  while (rmApp.getState() != RMAppState.KILLED) {
    Thread.sleep(100);
  }
  
  try {
    clientRMService.moveApplicationAcrossQueues(
        MoveApplicationAcrossQueuesRequest.newInstance(appId, "newqueue"));
    fail("Should have hit exception");
  } catch (YarnException ex) {
    assertEquals(YarnException.class,
        ex.getClass());
    assertEquals("App in KILLED state cannot be moved.", ex.getMessage());
  }
}
Example 11
Source File: RMContextImplEventRunnable.java    From garmadon with Apache License 2.0 4 votes vote down vote up 
public void sendAppEvent(ApplicationId applicationId, RMApp rmApp) {
    if (cacheFinishedApp.getIfPresent(applicationId.toString()) == null) {
        Header.Builder headerBuilder = Header.newBuilder()
            .withId(applicationId.toString())
            .withApplicationID(applicationId.toString())
            .withUser(rmApp.getUser())
            .withApplicationName(rmApp.getName())
            .withFramework(rmApp.getApplicationType().toUpperCase());

        ApplicationEvent.Builder eventBuilder = ApplicationEvent.newBuilder()
            .setState(rmApp.getState().name())
            .setQueue(rmApp.getQueue());

        rmApp.getApplicationTags().stream()
            .filter(tag -> YARN_TAGS_TO_EXTRACT.stream().noneMatch(tag::startsWith) && !tag.contains(":"))
            .forEach(eventBuilder::addYarnTags);

        rmApp.getApplicationTags().stream()
            .filter(tag -> tag.contains(":") && YARN_TAGS_TO_EXTRACT.stream().anyMatch(tag::startsWith))
            .map(tag -> {
                int idx = tag.indexOf(':');
                String key = tag.substring(0, idx);
                String value = tag.substring(idx + 1);
                return new String[] {key, value};
            })
            .forEach(splitTag -> BUILDERS.get(splitTag[0]).accept(splitTag[1], eventBuilder));

        eventBuilder.setFinalStatus(rmApp.getFinalApplicationStatus().name());

        eventBuilder.setStartTime(rmApp.getStartTime());
        eventBuilder.setFinishTime(rmApp.getFinishTime());

        RMAppMetrics rmAppMetrics = rmApp.getRMAppMetrics();
        if (rmAppMetrics != null) {
            eventBuilder.setMemorySeconds(rmAppMetrics.getMemorySeconds());
            eventBuilder.setVcoreSeconds(rmAppMetrics.getVcoreSeconds());
        }

        RMAppAttempt rmAppAttempt = rmApp.getCurrentAppAttempt();
        if (rmAppAttempt != null) {
            headerBuilder.withAttemptID(rmAppAttempt.getAppAttemptId().toString());

            Container container = rmAppAttempt.getMasterContainer();
            if (container != null) {
                eventBuilder.setAmContainerId(container.getId().toString());
            }
        }

        if (rmApp.getTrackingUrl() != null) {
            eventBuilder.setTrackingUrl(normalizeTrackingUrl(rmApp.getTrackingUrl()));
        }

        if (rmApp.getOriginalTrackingUrl() != null && !"N/A".equals(rmApp.getOriginalTrackingUrl())) {
            eventBuilder.setOriginalTrackingUrl(normalizeTrackingUrl(rmApp.getOriginalTrackingUrl()));
        }

        eventHandler.accept(System.currentTimeMillis(), headerBuilder.build(), eventBuilder.build());

        if (rmApp.getState() == RMAppState.FINISHED || rmApp.getState() == RMAppState.KILLED || rmApp.getState() == RMAppState.FAILED) {
            cacheFinishedApp.put(applicationId.toString(), rmApp.getState().name());
        }
    }
}
Example 12
Source File: ClientRMService.java    From hadoop with Apache License 2.0 4 votes vote down vote up 
@SuppressWarnings("unchecked")
@Override
public MoveApplicationAcrossQueuesResponse moveApplicationAcrossQueues(
    MoveApplicationAcrossQueuesRequest request) throws YarnException {
  ApplicationId applicationId = request.getApplicationId();

  UserGroupInformation callerUGI;
  try {
    callerUGI = UserGroupInformation.getCurrentUser();
  } catch (IOException ie) {
    LOG.info("Error getting UGI ", ie);
    RMAuditLogger.logFailure("UNKNOWN", AuditConstants.MOVE_APP_REQUEST,
        "UNKNOWN", "ClientRMService" , "Error getting UGI",
        applicationId);
    throw RPCUtil.getRemoteException(ie);
  }

  RMApp application = this.rmContext.getRMApps().get(applicationId);
  if (application == null) {
    RMAuditLogger.logFailure(callerUGI.getUserName(),
        AuditConstants.MOVE_APP_REQUEST, "UNKNOWN", "ClientRMService",
        "Trying to move an absent application", applicationId);
    throw new ApplicationNotFoundException("Trying to move an absent"
        + " application " + applicationId);
  }

  if (!checkAccess(callerUGI, application.getUser(),
      ApplicationAccessType.MODIFY_APP, application)) {
    RMAuditLogger.logFailure(callerUGI.getShortUserName(),
        AuditConstants.MOVE_APP_REQUEST,
        "User doesn't have permissions to "
            + ApplicationAccessType.MODIFY_APP.toString(), "ClientRMService",
        AuditConstants.UNAUTHORIZED_USER, applicationId);
    throw RPCUtil.getRemoteException(new AccessControlException("User "
        + callerUGI.getShortUserName() + " cannot perform operation "
        + ApplicationAccessType.MODIFY_APP.name() + " on " + applicationId));
  }
  
  // Moves only allowed when app is in a state that means it is tracked by
  // the scheduler
  if (EnumSet.of(RMAppState.NEW, RMAppState.NEW_SAVING, RMAppState.FAILED,
      RMAppState.FINAL_SAVING, RMAppState.FINISHING, RMAppState.FINISHED,
      RMAppState.KILLED, RMAppState.KILLING, RMAppState.FAILED)
      .contains(application.getState())) {
    String msg = "App in " + application.getState() + " state cannot be moved.";
    RMAuditLogger.logFailure(callerUGI.getShortUserName(),
        AuditConstants.MOVE_APP_REQUEST, "UNKNOWN", "ClientRMService", msg);
    throw new YarnException(msg);
  }

  SettableFuture<Object> future = SettableFuture.create();
  this.rmContext.getDispatcher().getEventHandler().handle(
      new RMAppMoveEvent(applicationId, request.getTargetQueue(), future));
  
  try {
    Futures.get(future, YarnException.class);
  } catch (YarnException ex) {
    RMAuditLogger.logFailure(callerUGI.getShortUserName(),
        AuditConstants.MOVE_APP_REQUEST, "UNKNOWN", "ClientRMService",
        ex.getMessage());
    throw ex;
  }

  RMAuditLogger.logSuccess(callerUGI.getShortUserName(), 
      AuditConstants.MOVE_APP_REQUEST, "ClientRMService" , applicationId);
  MoveApplicationAcrossQueuesResponse response = recordFactory
      .newRecordInstance(MoveApplicationAcrossQueuesResponse.class);
  return response;
}
Example 13
Source File: AMSimulator.java    From hadoop with Apache License 2.0 4 votes vote down vote up 
private void submitApp()
        throws YarnException, InterruptedException, IOException {
  // ask for new application
  GetNewApplicationRequest newAppRequest =
      Records.newRecord(GetNewApplicationRequest.class);
  GetNewApplicationResponse newAppResponse = 
      rm.getClientRMService().getNewApplication(newAppRequest);
  appId = newAppResponse.getApplicationId();
  
  // submit the application
  final SubmitApplicationRequest subAppRequest =
      Records.newRecord(SubmitApplicationRequest.class);
  ApplicationSubmissionContext appSubContext = 
      Records.newRecord(ApplicationSubmissionContext.class);
  appSubContext.setApplicationId(appId);
  appSubContext.setMaxAppAttempts(1);
  appSubContext.setQueue(queue);
  appSubContext.setPriority(Priority.newInstance(0));
  ContainerLaunchContext conLauContext = 
      Records.newRecord(ContainerLaunchContext.class);
  conLauContext.setApplicationACLs(
      new HashMap<ApplicationAccessType, String>());
  conLauContext.setCommands(new ArrayList<String>());
  conLauContext.setEnvironment(new HashMap<String, String>());
  conLauContext.setLocalResources(new HashMap<String, LocalResource>());
  conLauContext.setServiceData(new HashMap<String, ByteBuffer>());
  appSubContext.setAMContainerSpec(conLauContext);
  appSubContext.setUnmanagedAM(true);
  subAppRequest.setApplicationSubmissionContext(appSubContext);
  UserGroupInformation ugi = UserGroupInformation.createRemoteUser(user);
  ugi.doAs(new PrivilegedExceptionAction<Object>() {
    @Override
    public Object run() throws YarnException {
      rm.getClientRMService().submitApplication(subAppRequest);
      return null;
    }
  });
  LOG.info(MessageFormat.format("Submit a new application {0}", appId));
  
  // waiting until application ACCEPTED
  RMApp app = rm.getRMContext().getRMApps().get(appId);
  while(app.getState() != RMAppState.ACCEPTED) {
    Thread.sleep(10);
  }

  // Waiting until application attempt reach LAUNCHED
  // "Unmanaged AM must register after AM attempt reaches LAUNCHED state"
  this.appAttemptId = rm.getRMContext().getRMApps().get(appId)
      .getCurrentAppAttempt().getAppAttemptId();
  RMAppAttempt rmAppAttempt = rm.getRMContext().getRMApps().get(appId)
      .getCurrentAppAttempt();
  while (rmAppAttempt.getAppAttemptState() != RMAppAttemptState.LAUNCHED) {
    Thread.sleep(10);
  }
}
Example 14
Source File: ClientRMService.java    From big-c with Apache License 2.0 4 votes vote down vote up 
@SuppressWarnings("unchecked")
@Override
public MoveApplicationAcrossQueuesResponse moveApplicationAcrossQueues(
    MoveApplicationAcrossQueuesRequest request) throws YarnException {
  ApplicationId applicationId = request.getApplicationId();

  UserGroupInformation callerUGI;
  try {
    callerUGI = UserGroupInformation.getCurrentUser();
  } catch (IOException ie) {
    LOG.info("Error getting UGI ", ie);
    RMAuditLogger.logFailure("UNKNOWN", AuditConstants.MOVE_APP_REQUEST,
        "UNKNOWN", "ClientRMService" , "Error getting UGI",
        applicationId);
    throw RPCUtil.getRemoteException(ie);
  }

  RMApp application = this.rmContext.getRMApps().get(applicationId);
  if (application == null) {
    RMAuditLogger.logFailure(callerUGI.getUserName(),
        AuditConstants.MOVE_APP_REQUEST, "UNKNOWN", "ClientRMService",
        "Trying to move an absent application", applicationId);
    throw new ApplicationNotFoundException("Trying to move an absent"
        + " application " + applicationId);
  }

  if (!checkAccess(callerUGI, application.getUser(),
      ApplicationAccessType.MODIFY_APP, application)) {
    RMAuditLogger.logFailure(callerUGI.getShortUserName(),
        AuditConstants.MOVE_APP_REQUEST,
        "User doesn't have permissions to "
            + ApplicationAccessType.MODIFY_APP.toString(), "ClientRMService",
        AuditConstants.UNAUTHORIZED_USER, applicationId);
    throw RPCUtil.getRemoteException(new AccessControlException("User "
        + callerUGI.getShortUserName() + " cannot perform operation "
        + ApplicationAccessType.MODIFY_APP.name() + " on " + applicationId));
  }
  
  // Moves only allowed when app is in a state that means it is tracked by
  // the scheduler
  if (EnumSet.of(RMAppState.NEW, RMAppState.NEW_SAVING, RMAppState.FAILED,
      RMAppState.FINAL_SAVING, RMAppState.FINISHING, RMAppState.FINISHED,
      RMAppState.KILLED, RMAppState.KILLING, RMAppState.FAILED)
      .contains(application.getState())) {
    String msg = "App in " + application.getState() + " state cannot be moved.";
    RMAuditLogger.logFailure(callerUGI.getShortUserName(),
        AuditConstants.MOVE_APP_REQUEST, "UNKNOWN", "ClientRMService", msg);
    throw new YarnException(msg);
  }

  SettableFuture<Object> future = SettableFuture.create();
  this.rmContext.getDispatcher().getEventHandler().handle(
      new RMAppMoveEvent(applicationId, request.getTargetQueue(), future));
  
  try {
    Futures.get(future, YarnException.class);
  } catch (YarnException ex) {
    RMAuditLogger.logFailure(callerUGI.getShortUserName(),
        AuditConstants.MOVE_APP_REQUEST, "UNKNOWN", "ClientRMService",
        ex.getMessage());
    throw ex;
  }

  RMAuditLogger.logSuccess(callerUGI.getShortUserName(), 
      AuditConstants.MOVE_APP_REQUEST, "ClientRMService" , applicationId);
  MoveApplicationAcrossQueuesResponse response = recordFactory
      .newRecordInstance(MoveApplicationAcrossQueuesResponse.class);
  return response;
}
Example 15
Source File: AMSimulator.java    From big-c with Apache License 2.0 4 votes vote down vote up 
private void submitApp()
        throws YarnException, InterruptedException, IOException {
  // ask for new application
  GetNewApplicationRequest newAppRequest =
      Records.newRecord(GetNewApplicationRequest.class);
  GetNewApplicationResponse newAppResponse = 
      rm.getClientRMService().getNewApplication(newAppRequest);
  appId = newAppResponse.getApplicationId();
  
  // submit the application
  final SubmitApplicationRequest subAppRequest =
      Records.newRecord(SubmitApplicationRequest.class);
  ApplicationSubmissionContext appSubContext = 
      Records.newRecord(ApplicationSubmissionContext.class);
  appSubContext.setApplicationId(appId);
  appSubContext.setMaxAppAttempts(1);
  appSubContext.setQueue(queue);
  appSubContext.setPriority(Priority.newInstance(0));
  ContainerLaunchContext conLauContext = 
      Records.newRecord(ContainerLaunchContext.class);
  conLauContext.setApplicationACLs(
      new HashMap<ApplicationAccessType, String>());
  conLauContext.setCommands(new ArrayList<String>());
  conLauContext.setEnvironment(new HashMap<String, String>());
  conLauContext.setLocalResources(new HashMap<String, LocalResource>());
  conLauContext.setServiceData(new HashMap<String, ByteBuffer>());
  appSubContext.setAMContainerSpec(conLauContext);
  appSubContext.setUnmanagedAM(true);
  subAppRequest.setApplicationSubmissionContext(appSubContext);
  UserGroupInformation ugi = UserGroupInformation.createRemoteUser(user);
  ugi.doAs(new PrivilegedExceptionAction<Object>() {
    @Override
    public Object run() throws YarnException {
      rm.getClientRMService().submitApplication(subAppRequest);
      return null;
    }
  });
  LOG.info(MessageFormat.format("Submit a new application {0}", appId));
  
  // waiting until application ACCEPTED
  RMApp app = rm.getRMContext().getRMApps().get(appId);
  while(app.getState() != RMAppState.ACCEPTED) {
    Thread.sleep(10);
  }

  // Waiting until application attempt reach LAUNCHED
  // "Unmanaged AM must register after AM attempt reaches LAUNCHED state"
  this.appAttemptId = rm.getRMContext().getRMApps().get(appId)
      .getCurrentAppAttempt().getAppAttemptId();
  RMAppAttempt rmAppAttempt = rm.getRMContext().getRMApps().get(appId)
      .getCurrentAppAttempt();
  while (rmAppAttempt.getAppAttemptState() != RMAppAttemptState.LAUNCHED) {
    Thread.sleep(10);
  }
}