org.apache.hadoop.yarn.api.protocolrecords.FinishApplicationMasterRequest Java Examples

@Override
public void stopQueryMaster(QueryId queryId) {
  try {
    FinalApplicationStatus appStatus = FinalApplicationStatus.UNDEFINED;
    QueryInProgress queryInProgress = masterContext.getQueryJobManager().getQueryInProgress(queryId);
    if(queryInProgress == null) {
      return;
    }
    TajoProtos.QueryState state = queryInProgress.getQueryInfo().getQueryState();
    if (state == TajoProtos.QueryState.QUERY_SUCCEEDED) {
      appStatus = FinalApplicationStatus.SUCCEEDED;
    } else if (state == TajoProtos.QueryState.QUERY_FAILED || state == TajoProtos.QueryState.QUERY_ERROR) {
      appStatus = FinalApplicationStatus.FAILED;
    } else if (state == TajoProtos.QueryState.QUERY_ERROR) {
      appStatus = FinalApplicationStatus.FAILED;
    }
    FinishApplicationMasterRequest request = recordFactory
        .newRecordInstance(FinishApplicationMasterRequest.class);
    request.setFinalApplicationStatus(appStatus);
    request.setDiagnostics("QueryMaster shutdown by TajoMaster.");
    rmClient.finishApplicationMaster(request);
  } catch (Exception e) {
    LOG.error(e.getMessage(), e);
  }
}
 

private void finishApplication(FinalApplicationStatus finalStatus) throws YarnException, IOException
{
  LOG.info("Application completed. Signalling finish to RM");
  FinishApplicationMasterRequest finishReq = Records.newRecord(FinishApplicationMasterRequest.class);
  finishReq.setFinalApplicationStatus(finalStatus);

  if (finalStatus != FinalApplicationStatus.SUCCEEDED) {
    String diagnostics = "Diagnostics." + " completed=" + numCompletedContainers.get() + ", allocated=" + allocatedContainers.size() + ", failed=" + numFailedContainers.get();
    if (!StringUtils.isEmpty(dnmgr.shutdownDiagnosticsMessage)) {
      diagnostics += "\n";
      diagnostics += dnmgr.shutdownDiagnosticsMessage;
    }
    // YARN-208 - as of 2.0.1-alpha dropped by the RM
    finishReq.setDiagnostics(diagnostics);
    // expected termination of the master process
    // application status and diagnostics message are set above
  }
  LOG.info("diagnostics: " + finishReq.getDiagnostics());
  amRmClient.unregisterApplicationMaster(finishReq.getFinalApplicationStatus(), finishReq.getDiagnostics(), null);
}
 

public static void main(String[] args) throws Exception {
  if (args[0].equals("success")) {
    ApplicationMasterProtocol client = ClientRMProxy.createRMProxy(conf,
        ApplicationMasterProtocol.class);
    client.registerApplicationMaster(RegisterApplicationMasterRequest
        .newInstance(NetUtils.getHostname(), -1, ""));
    Thread.sleep(1000);
    FinishApplicationMasterResponse resp =
        client.finishApplicationMaster(FinishApplicationMasterRequest
          .newInstance(FinalApplicationStatus.SUCCEEDED, "success", null));
    assertTrue(resp.getIsUnregistered());
    System.exit(0);
  } else {
    System.exit(1);
  }
}
 

public void unregisterAppAttempt(final FinishApplicationMasterRequest req,
    boolean waitForStateRunning) throws Exception {
  if (waitForStateRunning) {
    waitForState(RMAppAttemptState.RUNNING);
  }
  if (ugi == null) {
    ugi =  UserGroupInformation.createRemoteUser(attemptId.toString());
    Token<AMRMTokenIdentifier> token =
        context.getRMApps()
            .get(attemptId.getApplicationId())
            .getRMAppAttempt(attemptId).getAMRMToken();
    ugi.addTokenIdentifier(token.decodeIdentifier());
  }
  try {
    ugi.doAs(new PrivilegedExceptionAction<Object>() {
      @Override
      public Object run() throws Exception {
        amRMProtocol.finishApplicationMaster(req);
        return null;
      }
    });
  } catch (UndeclaredThrowableException e) {
    throw (Exception) e.getCause();
  }
}
 

private void finishApplicationMaster(RMApp rmApp, MockRM rm, MockNM nm,
    MockAM am, FinishApplicationMasterRequest req) throws Exception {
  RMState rmState =
      ((MemoryRMStateStore) rm.getRMContext().getStateStore()).getState();
  Map<ApplicationId, ApplicationStateData> rmAppState =
      rmState.getApplicationState();
  am.unregisterAppAttempt(req,true);
  am.waitForState(RMAppAttemptState.FINISHING);
  nm.nodeHeartbeat(am.getApplicationAttemptId(), 1, ContainerState.COMPLETE);
  am.waitForState(RMAppAttemptState.FINISHED);
  rm.waitForState(rmApp.getApplicationId(), RMAppState.FINISHED);
  // check that app/attempt is saved with the final state
  ApplicationStateData appState = rmAppState.get(rmApp.getApplicationId());
  Assert
    .assertEquals(RMAppState.FINISHED, appState.getState());
  Assert.assertEquals(RMAppAttemptState.FINISHED,
    appState.getAttempt(am.getApplicationAttemptId()).getState());
}
 

private void finishApplicationMaster(RMApp rmApp, MockRM rm, MockNM nm,
    MockAM am, FinishApplicationMasterRequest req) throws Exception {
  RMState rmState =
      ((MemoryRMStateStore) rm.getRMContext().getStateStore()).getState();
  Map<ApplicationId, ApplicationStateData> rmAppState =
      rmState.getApplicationState();
  am.unregisterAppAttempt(req,true);
  am.waitForState(RMAppAttemptState.FINISHING);
  nm.nodeHeartbeat(am.getApplicationAttemptId(), 1, ContainerState.COMPLETE);
  am.waitForState(RMAppAttemptState.FINISHED);
  rm.waitForState(rmApp.getApplicationId(), RMAppState.FINISHED);
  // check that app/attempt is saved with the final state
  ApplicationStateData appState = rmAppState.get(rmApp.getApplicationId());
  Assert
    .assertEquals(RMAppState.FINISHED, appState.getState());
  Assert.assertEquals(RMAppAttemptState.FINISHED,
    appState.getAttempt(am.getApplicationAttemptId()).getState());
}
 

public void unregisterAppAttempt(final FinishApplicationMasterRequest req,
    boolean waitForStateRunning) throws Exception {
  if (waitForStateRunning) {
    waitForState(RMAppAttemptState.RUNNING);
  }
  if (ugi == null) {
    ugi =  UserGroupInformation.createRemoteUser(attemptId.toString());
    Token<AMRMTokenIdentifier> token =
        context.getRMApps()
            .get(attemptId.getApplicationId())
            .getRMAppAttempt(attemptId).getAMRMToken();
    ugi.addTokenIdentifier(token.decodeIdentifier());
  }
  try {
    ugi.doAs(new PrivilegedExceptionAction<Object>() {
      @Override
      public Object run() throws Exception {
        amRMProtocol.finishApplicationMaster(req);
        return null;
      }
    });
  } catch (UndeclaredThrowableException e) {
    throw (Exception) e.getCause();
  }
}
 

private void finishApplication(FinalApplicationStatus finalStatus) throws YarnException, IOException
{
  LOG.info("Application completed. Signalling finish to RM");
  FinishApplicationMasterRequest finishReq = Records.newRecord(FinishApplicationMasterRequest.class);
  finishReq.setFinalApplicationStatus(finalStatus);

  if (finalStatus != FinalApplicationStatus.SUCCEEDED) {
    String diagnostics = "Diagnostics." + " completed=" + numCompletedContainers.get() + ", allocated=" + allocatedContainers.size() + ", failed=" + numFailedContainers.get();
    if (!StringUtils.isEmpty(dnmgr.shutdownDiagnosticsMessage)) {
      diagnostics += "\n";
      diagnostics += dnmgr.shutdownDiagnosticsMessage;
    }
    // YARN-208 - as of 2.0.1-alpha dropped by the RM
    finishReq.setDiagnostics(diagnostics);
    // expected termination of the master process
    // application status and diagnostics message are set above
  }
  LOG.info("diagnostics: " + finishReq.getDiagnostics());
  amRmClient.unregisterApplicationMaster(finishReq.getFinalApplicationStatus(), finishReq.getDiagnostics(), null);
}
 

public static void main(String[] args) throws Exception {
  if (args[0].equals("success")) {
    ApplicationMasterProtocol client = ClientRMProxy.createRMProxy(conf,
        ApplicationMasterProtocol.class);
    client.registerApplicationMaster(RegisterApplicationMasterRequest
        .newInstance(NetUtils.getHostname(), -1, ""));
    Thread.sleep(1000);
    FinishApplicationMasterResponse resp =
        client.finishApplicationMaster(FinishApplicationMasterRequest
          .newInstance(FinalApplicationStatus.SUCCEEDED, "success", null));
    assertTrue(resp.getIsUnregistered());
    System.exit(0);
  } else {
    System.exit(1);
  }
}
 

@Override
public FinishApplicationMasterResponse finishApplicationMaster(
    FinishApplicationMasterRequest request) throws YarnException,
    IOException {
  FinishApplicationMasterRequestProto requestProto =
      ((FinishApplicationMasterRequestPBImpl) request).getProto();
  try {
    return new FinishApplicationMasterResponsePBImpl(
      proxy.finishApplicationMaster(null, requestProto));
  } catch (ServiceException e) {
    RPCUtil.unwrapAndThrowException(e);
    return null;
  }
}
 

@Test(timeout = 15000)
public void testFinishApplicationMasterOnHA() throws YarnException,
    IOException {
  FinishApplicationMasterRequest request =
      FinishApplicationMasterRequest.newInstance(
          FinalApplicationStatus.SUCCEEDED, "", "");
  FinishApplicationMasterResponse response =
      amClient.finishApplicationMaster(request);
  Assert.assertEquals(response,
      this.cluster.createFakeFinishApplicationMasterResponse());
}
 

@Override
public FinishApplicationMasterResponse finishApplicationMaster(
    FinishApplicationMasterRequest request) throws YarnException,
    IOException {
  resetStartFailoverFlag(true);
  // make sure failover has been triggered
  Assert.assertTrue(waittingForFailOver());
  return createFakeFinishApplicationMasterResponse();
}
 

public void unregisterAppAttempt(boolean waitForStateRunning)
    throws Exception {
  final FinishApplicationMasterRequest req =
      FinishApplicationMasterRequest.newInstance(
          FinalApplicationStatus.SUCCEEDED, "", "");
  unregisterAppAttempt(req, waitForStateRunning);
}
 

public static void finishAMAndVerifyAppState(RMApp rmApp, MockRM rm, MockNM nm,
    MockAM am) throws Exception {
  FinishApplicationMasterRequest req =
      FinishApplicationMasterRequest.newInstance(
        FinalApplicationStatus.SUCCEEDED, "", "");
  am.unregisterAppAttempt(req,true);
  am.waitForState(RMAppAttemptState.FINISHING);
  nm.nodeHeartbeat(am.getApplicationAttemptId(), 1, ContainerState.COMPLETE);
  am.waitForState(RMAppAttemptState.FINISHED);
  rm.waitForState(rmApp.getApplicationId(), RMAppState.FINISHED);
}
 

private void finishApplicationMaster(RMApp rmApp, MockRM rm, MockNM nm,
    MockAM am) throws Exception {
  final FinishApplicationMasterRequest req =
      FinishApplicationMasterRequest.newInstance(
        FinalApplicationStatus.SUCCEEDED, "", "");
  finishApplicationMaster(rmApp, rm, nm, am, req);
}
 

@Override
public FinishApplicationMasterResponse finishApplicationMaster(
    FinishApplicationMasterRequest request) throws YarnException,
    IOException {
  // TODO Auto-generated method stub
  return null;
}
 

public static void finishAMAndVerifyAppState(RMApp rmApp, MockRM rm, MockNM nm,
    MockAM am) throws Exception {
  FinishApplicationMasterRequest req =
      FinishApplicationMasterRequest.newInstance(
        FinalApplicationStatus.SUCCEEDED, "", "");
  am.unregisterAppAttempt(req,true);
  am.waitForState(RMAppAttemptState.FINISHING);
  nm.nodeHeartbeat(am.getApplicationAttemptId(), 1, ContainerState.COMPLETE);
  am.waitForState(RMAppAttemptState.FINISHED);
  rm.waitForState(rmApp.getApplicationId(), RMAppState.FINISHED);
}
 

@Override
public FinishApplicationMasterResponse finishApplicationMaster(
    FinishApplicationMasterRequest request) throws YarnException,
    IOException {
  FinishApplicationMasterRequestProto requestProto =
      ((FinishApplicationMasterRequestPBImpl) request).getProto();
  try {
    return new FinishApplicationMasterResponsePBImpl(
      proxy.finishApplicationMaster(null, requestProto));
  } catch (ServiceException e) {
    RPCUtil.unwrapAndThrowException(e);
    return null;
  }
}
 

@Override
public FinishApplicationMasterResponse finishApplicationMaster(
    FinishApplicationMasterRequest request) throws YarnException,
    IOException {
  // TODO Auto-generated method stub
  return null;
}
 

private void finishApplicationMaster(RMApp rmApp, MockRM rm, MockNM nm,
    MockAM am) throws Exception {
  final FinishApplicationMasterRequest req =
      FinishApplicationMasterRequest.newInstance(
        FinalApplicationStatus.SUCCEEDED, "", "");
  finishApplicationMaster(rmApp, rm, nm, am, req);
}
 

public void unregisterAppAttempt(boolean waitForStateRunning)
    throws Exception {
  final FinishApplicationMasterRequest req =
      FinishApplicationMasterRequest.newInstance(
          FinalApplicationStatus.SUCCEEDED, "", "");
  unregisterAppAttempt(req, waitForStateRunning);
}
 

@Override
public FinishApplicationMasterResponse finishApplicationMaster(
    FinishApplicationMasterRequest request) throws YarnException,
    IOException {
  resetStartFailoverFlag(true);
  // make sure failover has been triggered
  Assert.assertTrue(waittingForFailOver());
  return createFakeFinishApplicationMasterResponse();
}
 

@Test(timeout = 15000)
public void testFinishApplicationMasterOnHA() throws YarnException,
    IOException {
  FinishApplicationMasterRequest request =
      FinishApplicationMasterRequest.newInstance(
          FinalApplicationStatus.SUCCEEDED, "", "");
  FinishApplicationMasterResponse response =
      amClient.finishApplicationMaster(request);
  Assert.assertEquals(response,
      this.cluster.createFakeFinishApplicationMasterResponse());
}
 

@Test (timeout = 60000)
public void testRMRestartWaitForPreviousSucceededAttempt() throws Exception {
  conf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, 2);
  MemoryRMStateStore memStore = new MemoryRMStateStore() {
    int count = 0;

    @Override
    public void updateApplicationStateInternal(ApplicationId appId,
        ApplicationStateData appStateData) throws Exception {
      if (count == 0) {
        // do nothing; simulate app final state is not saved.
        LOG.info(appId + " final state is not saved.");
        count++;
      } else {
        super.updateApplicationStateInternal(appId, appStateData);
      }
    }
  };
  memStore.init(conf);
  RMState rmState = memStore.getState();
  Map<ApplicationId, ApplicationStateData> rmAppState =
      rmState.getApplicationState();

  // start RM
  MockRM rm1 = createMockRM(conf, memStore);
  rm1.start();
  MockNM nm1 = rm1.registerNode("127.0.0.1:1234", 15120);
  RMApp app0 = rm1.submitApp(200);
  MockAM am0 = MockRM.launchAndRegisterAM(app0, rm1, nm1);

  FinishApplicationMasterRequest req =
      FinishApplicationMasterRequest.newInstance(
        FinalApplicationStatus.SUCCEEDED, "", "");
  am0.unregisterAppAttempt(req, true);
  am0.waitForState(RMAppAttemptState.FINISHING);
  // app final state is not saved. This guarantees that RMApp cannot be
  // recovered via its own saved state, but only via the event notification
  // from the RMAppAttempt on recovery.
  Assert.assertNull(rmAppState.get(app0.getApplicationId()).getState());

  // start RM
  MockRM rm2 = createMockRM(conf, memStore);
  nm1.setResourceTrackerService(rm2.getResourceTrackerService());
  rm2.start();

  rm2.waitForState(app0.getCurrentAppAttempt().getAppAttemptId(),
    RMAppAttemptState.FINISHED);
  rm2.waitForState(app0.getApplicationId(), RMAppState.FINISHED);
  // app final state is saved via the finish event from attempt.
  Assert.assertEquals(RMAppState.FINISHED,
      rmAppState.get(app0.getApplicationId()).getState());
}
 

@Override
public FinishApplicationMasterResponse finishApplicationMaster(
    FinishApplicationMasterRequest request) throws YarnException,
    IOException {
  return null;
}
 

@Override
public FinishApplicationMasterResponse finishApplicationMaster(
    FinishApplicationMasterRequest request) throws YarnException,
    IOException {
  return FinishApplicationMasterResponse.newInstance(false);
}
 

@Override
public FinishApplicationMasterResponse finishApplicationMaster(
    FinishApplicationMasterRequest request) throws YarnException,
    IOException {
  return FinishApplicationMasterResponse.newInstance(false);
}
 

@Override
public FinishApplicationMasterResponse finishApplicationMaster(
    FinishApplicationMasterRequest request) throws YarnException,
    IOException {
  return null;
}
 

@VisibleForTesting
protected void doUnregistration()
    throws YarnException, IOException, InterruptedException {
  FinalApplicationStatus finishState = FinalApplicationStatus.UNDEFINED;
  JobImpl jobImpl = (JobImpl)job;
  if (jobImpl.getInternalState() == JobStateInternal.SUCCEEDED) {
    finishState = FinalApplicationStatus.SUCCEEDED;
  } else if (jobImpl.getInternalState() == JobStateInternal.KILLED
      || (jobImpl.getInternalState() == JobStateInternal.RUNNING && isSignalled)) {
    finishState = FinalApplicationStatus.KILLED;
  } else if (jobImpl.getInternalState() == JobStateInternal.FAILED
      || jobImpl.getInternalState() == JobStateInternal.ERROR) {
    finishState = FinalApplicationStatus.FAILED;
  }
  StringBuffer sb = new StringBuffer();
  for (String s : job.getDiagnostics()) {
    sb.append(s).append("\n");
  }
  LOG.info("Setting job diagnostics to " + sb.toString());

  String historyUrl =
      MRWebAppUtil.getApplicationWebURLOnJHSWithScheme(getConfig(),
          context.getApplicationID());
  LOG.info("History url is " + historyUrl);
  FinishApplicationMasterRequest request =
      FinishApplicationMasterRequest.newInstance(finishState,
        sb.toString(), historyUrl);
  try {
    while (true) {
      FinishApplicationMasterResponse response =
          scheduler.finishApplicationMaster(request);
      if (response.getIsUnregistered()) {
        // When excepting ClientService, other services are already stopped,
        // it is safe to let clients know the final states. ClientService
        // should wait for some time so clients have enough time to know the
        // final states.
        RunningAppContext raContext = (RunningAppContext) context;
        raContext.markSuccessfulUnregistration();
        break;
      }
      LOG.info("Waiting for application to be successfully unregistered.");
      Thread.sleep(rmPollInterval);
    }
  } catch (ApplicationMasterNotRegisteredException e) {
    // RM might have restarted or failed over and so lost the fact that AM had
    // registered before.
    register();
    doUnregistration();
  }
}
 

@Override
public FinishApplicationMasterResponse finishApplicationMaster(
    FinishApplicationMasterRequest request) throws YarnException,
    IOException {

  ApplicationAttemptId applicationAttemptId =
      authorizeRequest().getApplicationAttemptId();
  ApplicationId appId = applicationAttemptId.getApplicationId();

  RMApp rmApp =
      rmContext.getRMApps().get(applicationAttemptId.getApplicationId());
  // checking whether the app exits in RMStateStore at first not to throw
  // ApplicationDoesNotExistInCacheException before and after
  // RM work-preserving restart.
  if (rmApp.isAppFinalStateStored()) {
    LOG.info(rmApp.getApplicationId() + " unregistered successfully. ");
    return FinishApplicationMasterResponse.newInstance(true);
  }

  AllocateResponseLock lock = responseMap.get(applicationAttemptId);
  if (lock == null) {
    throwApplicationDoesNotExistInCacheException(applicationAttemptId);
  }

  // Allow only one thread in AM to do finishApp at a time.
  synchronized (lock) {
    if (!hasApplicationMasterRegistered(applicationAttemptId)) {
      String message =
          "Application Master is trying to unregister before registering for: "
              + appId;
      LOG.error(message);
      RMAuditLogger.logFailure(
          this.rmContext.getRMApps()
              .get(appId).getUser(),
          AuditConstants.UNREGISTER_AM, "", "ApplicationMasterService",
          message, appId,
          applicationAttemptId);
      throw new ApplicationMasterNotRegisteredException(message);
    }

    this.amLivelinessMonitor.receivedPing(applicationAttemptId);

    rmContext.getDispatcher().getEventHandler().handle(
        new RMAppAttemptUnregistrationEvent(applicationAttemptId, request
            .getTrackingUrl(), request.getFinalApplicationStatus(), request
            .getDiagnostics()));

    // For UnmanagedAMs, return true so they don't retry
    return FinishApplicationMasterResponse.newInstance(
        rmApp.getApplicationSubmissionContext().getUnmanagedAM());
  }
}