org.apache.hadoop.yarn.api.records.YarnApplicationState#FAILED

Source File: YARNRunner.java From hadoop with Apache License 2.0

7 votes

private void killUnFinishedApplication(ApplicationId appId)
    throws IOException {
  ApplicationReport application = null;
  try {
    application = resMgrDelegate.getApplicationReport(appId);
  } catch (YarnException e) {
    throw new IOException(e);
  }
  if (application.getYarnApplicationState() == YarnApplicationState.FINISHED
      || application.getYarnApplicationState() == YarnApplicationState.FAILED
      || application.getYarnApplicationState() == YarnApplicationState.KILLED) {
    return;
  }
  killApplication(appId);
}

Source File: ApplicationCLI.java From big-c with Apache License 2.0

6 votes

/**
 * Kills the application with the application id as appId
 * 
 * @param applicationId
 * @throws YarnException
 * @throws IOException
 */
private void killApplication(String applicationId) throws YarnException,
    IOException {
  ApplicationId appId = ConverterUtils.toApplicationId(applicationId);
  ApplicationReport  appReport = null;
  try {
    appReport = client.getApplicationReport(appId);
  } catch (ApplicationNotFoundException e) {
    sysout.println("Application with id '" + applicationId +
        "' doesn't exist in RM.");
    throw e;
  }

  if (appReport.getYarnApplicationState() == YarnApplicationState.FINISHED
      || appReport.getYarnApplicationState() == YarnApplicationState.KILLED
      || appReport.getYarnApplicationState() == YarnApplicationState.FAILED) {
    sysout.println("Application " + applicationId + " has already finished ");
  } else {
    sysout.println("Killing application " + applicationId);
    client.killApplication(appId);
  }
}

Source File: YarnApplicationStatusMonitor.java From flink with Apache License 2.0

6 votes

private void updateApplicationStatus() {
	if (yarnClient.isInState(Service.STATE.STARTED)) {
		final ApplicationReport applicationReport;

		try {
			applicationReport = yarnClient.getApplicationReport(yarnApplicationId);
		} catch (Exception e) {
			LOG.info("Could not retrieve the Yarn application report for {}.", yarnApplicationId);
			applicationStatus = ApplicationStatus.UNKNOWN;
			return;
		}

		YarnApplicationState yarnApplicationState = applicationReport.getYarnApplicationState();

		if (yarnApplicationState == YarnApplicationState.FAILED || yarnApplicationState == YarnApplicationState.KILLED) {
			applicationStatus = ApplicationStatus.FAILED;
		} else {
			applicationStatus = ApplicationStatus.SUCCEEDED;
		}
	} else {
		LOG.info("Yarn client is no longer in state STARTED. Stopping the Yarn application status monitor.");
		applicationStatusUpdateFuture.cancel(false);
	}
}

Source File: LocalClient.java From tez with Apache License 2.0

6 votes

protected YarnApplicationState convertDAGAppMasterState(DAGAppMasterState dagAppMasterState) {
  switch (dagAppMasterState) {
  case NEW:
    return YarnApplicationState.NEW;
  case INITED:
  case RECOVERING:
  case IDLE:
  case RUNNING:
    return YarnApplicationState.RUNNING;
  case SUCCEEDED:
    return YarnApplicationState.FINISHED;
  case FAILED:
    return YarnApplicationState.FAILED;
  case KILLED:
    return YarnApplicationState.KILLED;
  case ERROR:
    return YarnApplicationState.FAILED;
  default:
    return YarnApplicationState.SUBMITTED;
  }
}

Source File: RMServerUtils.java From hadoop with Apache License 2.0

6 votes

public static YarnApplicationState createApplicationState(
    RMAppState rmAppState) {
  switch (rmAppState) {
    case NEW:
      return YarnApplicationState.NEW;
    case NEW_SAVING:
      return YarnApplicationState.NEW_SAVING;
    case SUBMITTED:
      return YarnApplicationState.SUBMITTED;
    case ACCEPTED:
      return YarnApplicationState.ACCEPTED;
    case RUNNING:
      return YarnApplicationState.RUNNING;
    case FINISHING:
    case FINISHED:
      return YarnApplicationState.FINISHED;
    case KILLED:
      return YarnApplicationState.KILLED;
    case FAILED:
      return YarnApplicationState.FAILED;
    default:
      throw new YarnRuntimeException("Unknown state passed!");
    }
}

Source File: DAGClientImpl.java From tez with Apache License 2.0

5 votes

private void checkAndSetDagCompletionStatus() {
  ApplicationReport appReport = realClient.getApplicationReportInternal();
  if (appReport != null) {
    final YarnApplicationState appState = appReport.getYarnApplicationState();
    if (appState == YarnApplicationState.FINISHED || appState == YarnApplicationState.FAILED ||
        appState == YarnApplicationState.KILLED) {
      dagCompleted = true;
    }
  }
}

Source File: YARNRunner.java From big-c with Apache License 2.0

5 votes

@Override
public JobStatus submitJob(JobID jobId, String jobSubmitDir, Credentials ts)
throws IOException, InterruptedException {
  
  addHistoryToken(ts);
  
  // Construct necessary information to start the MR AM
  ApplicationSubmissionContext appContext =
    createApplicationSubmissionContext(conf, jobSubmitDir, ts);

  // Submit to ResourceManager
  try {
    ApplicationId applicationId =
        resMgrDelegate.submitApplication(appContext);

    ApplicationReport appMaster = resMgrDelegate
        .getApplicationReport(applicationId);
    String diagnostics =
        (appMaster == null ?
            "application report is null" : appMaster.getDiagnostics());
    if (appMaster == null
        || appMaster.getYarnApplicationState() == YarnApplicationState.FAILED
        || appMaster.getYarnApplicationState() == YarnApplicationState.KILLED) {
      throw new IOException("Failed to run job : " +
          diagnostics);
    }
    return clientCache.getClient(jobId).getJobStatus(jobId);
  } catch (YarnException e) {
    throw new IOException(e);
  }
}

Source File: ApplicationCLI.java From big-c with Apache License 2.0

5 votes

/**
 * Moves the application with the given ID to the given queue.
 */
private void moveApplicationAcrossQueues(String applicationId, String queue)
    throws YarnException, IOException {
  ApplicationId appId = ConverterUtils.toApplicationId(applicationId);
  ApplicationReport appReport = client.getApplicationReport(appId);
  if (appReport.getYarnApplicationState() == YarnApplicationState.FINISHED
      || appReport.getYarnApplicationState() == YarnApplicationState.KILLED
      || appReport.getYarnApplicationState() == YarnApplicationState.FAILED) {
    sysout.println("Application " + applicationId + " has already finished ");
  } else {
    sysout.println("Moving application " + applicationId + " to queue " + queue);
    client.moveApplicationAcrossQueues(appId, queue);
    sysout.println("Successfully completed move.");
  }
}

Source File: GobblinYarnAppLauncher.java From incubator-gobblin with Apache License 2.0

5 votes

@Subscribe
public void handleApplicationReportArrivalEvent(ApplicationReportArrivalEvent applicationReportArrivalEvent) {
  ApplicationReport applicationReport = applicationReportArrivalEvent.getApplicationReport();

  YarnApplicationState appState = applicationReport.getYarnApplicationState();
  LOGGER.info("Gobblin Yarn application state: " + appState.toString());

  // Reset the count on failures to get the ApplicationReport when there's one success
  this.getApplicationReportFailureCount.set(0);

  if (appState == YarnApplicationState.FINISHED ||
      appState == YarnApplicationState.FAILED ||
      appState == YarnApplicationState.KILLED) {

    applicationCompleted = true;

    LOGGER.info("Gobblin Yarn application finished with final status: " +
        applicationReport.getFinalApplicationStatus().toString());
    if (applicationReport.getFinalApplicationStatus() == FinalApplicationStatus.FAILED) {
      LOGGER.error("Gobblin Yarn application failed for the following reason: " + applicationReport.getDiagnostics());
    }

    try {
      GobblinYarnAppLauncher.this.stop();
    } catch (IOException ioe) {
      LOGGER.error("Failed to close the " + GobblinYarnAppLauncher.class.getSimpleName(), ioe);
    } catch (TimeoutException te) {
      LOGGER.error("Timeout in stopping the service manager", te);
    } finally {
      if (this.emailNotificationOnShutdown) {
        sendEmailOnShutdown(Optional.of(applicationReport));
      }
    }
  }
}

Source File: YARNRunner.java From hadoop with Apache License 2.0

5 votes

@Override
public JobStatus submitJob(JobID jobId, String jobSubmitDir, Credentials ts)
throws IOException, InterruptedException {
  
  addHistoryToken(ts);
  
  // Construct necessary information to start the MR AM
  ApplicationSubmissionContext appContext =
    createApplicationSubmissionContext(conf, jobSubmitDir, ts);

  // Submit to ResourceManager
  try {
    ApplicationId applicationId =
        resMgrDelegate.submitApplication(appContext);

    ApplicationReport appMaster = resMgrDelegate
        .getApplicationReport(applicationId);
    String diagnostics =
        (appMaster == null ?
            "application report is null" : appMaster.getDiagnostics());
    if (appMaster == null
        || appMaster.getYarnApplicationState() == YarnApplicationState.FAILED
        || appMaster.getYarnApplicationState() == YarnApplicationState.KILLED) {
      throw new IOException("Failed to run job : " +
          diagnostics);
    }
    return clientCache.getClient(jobId).getJobStatus(jobId);
  } catch (YarnException e) {
    throw new IOException(e);
  }
}

Source File: ApplicationCLI.java From hadoop with Apache License 2.0

5 votes

/**
 * Moves the application with the given ID to the given queue.
 */
private void moveApplicationAcrossQueues(String applicationId, String queue)
    throws YarnException, IOException {
  ApplicationId appId = ConverterUtils.toApplicationId(applicationId);
  ApplicationReport appReport = client.getApplicationReport(appId);
  if (appReport.getYarnApplicationState() == YarnApplicationState.FINISHED
      || appReport.getYarnApplicationState() == YarnApplicationState.KILLED
      || appReport.getYarnApplicationState() == YarnApplicationState.FAILED) {
    sysout.println("Application " + applicationId + " has already finished ");
  } else {
    sysout.println("Moving application " + applicationId + " to queue " + queue);
    client.moveApplicationAcrossQueues(appId, queue);
    sysout.println("Successfully completed move.");
  }
}

Source File: TezYarnClient.java From tez with Apache License 2.0

5 votes

@Override
 public ApplicationId submitApplication(ApplicationSubmissionContext appSubmissionContext)
     throws YarnException, IOException, TezException {
ApplicationId appId= yarnClient.submitApplication(appSubmissionContext);
   ApplicationReport appReport = getApplicationReport(appId);
   if (appReport.getYarnApplicationState() == YarnApplicationState.FAILED){
     throw new TezException("Failed to submit application to YARN"
         + ", applicationId=" + appId
         + ", diagnostics=" + appReport.getDiagnostics());
   }
   return appId;
 }

Source File: FlinkYarnSessionCli.java From flink with Apache License 2.0

5 votes

private void logApplicationReport(ApplicationReport appReport) {
	LOG.info("Application " + appReport.getApplicationId() + " finished with state " + appReport
		.getYarnApplicationState() + " and final state " + appReport
		.getFinalApplicationStatus() + " at " + appReport.getFinishTime());

	if (appReport.getYarnApplicationState() == YarnApplicationState.FAILED) {
		LOG.warn("Application failed. Diagnostics " + appReport.getDiagnostics());
		LOG.warn("If log aggregation is activated in the Hadoop cluster, we recommend to retrieve "
			+ "the full application log using this command:"
			+ System.lineSeparator()
			+ "\tyarn logs -applicationId " + appReport.getApplicationId()
			+ System.lineSeparator()
			+ "(It sometimes takes a few seconds until the logs are aggregated)");
	}
}

Source File: Client.java From metron with Apache License 2.0

4 votes

/**
 * Monitor the submitted application for completion.
 * Kill application if time expires.
 * @param appId Application Id of application to be monitored
 * @return true if application completed successfully
 * @throws YarnException
 * @throws IOException
 */
private boolean monitorApplication(ApplicationId appId)
        throws YarnException, IOException {

  while (true) {

    // Check app status every 1 second.
    try {
      Thread.sleep(1000);
    } catch (InterruptedException e) {
      LOG.debug("Thread sleep in monitoring loop interrupted");
    }

    // Get application report for the appId we are interested in
    ApplicationReport report = yarnClient.getApplicationReport(appId);

    LOG.info("Got application report from ASM for"
            + ", appId=" + appId.getId()
            + ", clientToAMToken=" + report.getClientToAMToken()
            + ", appDiagnostics=" + report.getDiagnostics()
            + ", appMasterHost=" + report.getHost()
            + ", appQueue=" + report.getQueue()
            + ", appMasterRpcPort=" + report.getRpcPort()
            + ", appStartTime=" + report.getStartTime()
            + ", yarnAppState=" + report.getYarnApplicationState().toString()
            + ", distributedFinalState=" + report.getFinalApplicationStatus().toString()
            + ", appTrackingUrl=" + report.getTrackingUrl()
            + ", appUser=" + report.getUser());

    YarnApplicationState state = report.getYarnApplicationState();
    FinalApplicationStatus dsStatus = report.getFinalApplicationStatus();
    if(YarnApplicationState.RUNNING == state) {
      LOG.info("Application is running...");
      return true;
    }
    if (YarnApplicationState.FINISHED == state) {
      if (FinalApplicationStatus.SUCCEEDED == dsStatus) {
        LOG.info("Application has completed successfully. Breaking monitoring loop");
        return true;
      }
      else {
        LOG.info("Application did finished unsuccessfully."
                + " YarnState=" + state.toString() + ", DSFinalStatus=" + dsStatus.toString()
                + ". Breaking monitoring loop");
        return false;
      }
    }
    else if (YarnApplicationState.KILLED == state
            || YarnApplicationState.FAILED == state) {
      LOG.info("Application did not finish."
              + " YarnState=" + state.toString() + ", DSFinalStatus=" + dsStatus.toString()
              + ". Breaking monitoring loop");
      return false;
    }

    if (System.currentTimeMillis() > (clientStartTime + clientTimeout)) {
      LOG.info("Reached client specified timeout for application. Killing application");
      forceKillApplication(appId);
      return false;
    }
  }

}

Source File: FlinkEngineServiceImpl.java From PoseidonX with Apache License 2.0

4 votes

public static void stopTask(Integer taskId, String taskName, Integer processId) throws Exception {
    if(taskId == null){
        throw new Exception("任务停止失败，参数异常，taskId为空！");
    }
    if(StringUtils.isBlank(taskName)){
        throw new Exception("任务提交失败，参数异常，任务名为空！");
    }
    if(processId == null){
        throw new Exception("任务停止失败，参数异常，processId为空！");
    }
    FlinkProcessPO flinkProcessPO = flinkEngineService.flinkProcessDao.getById(processId);
    if(flinkProcessPO == null){
        throw new Exception("任务停止失败，任务的执行信息不存在或者已经被删除！");
    }
    EngineContant.FLINK_HOME = ConfigProperty.getConfigValue(ConfigKeyEnum.FLINK_HOME);
    if(StringUtils.isBlank(EngineContant.FLINK_HOME)){
        throw new Exception("任务停止失败，参数异常，系统参数 FLINK_HOME 未进行配置！");
    }
    if(!FlinkOnYarnBusiness.stopJob(flinkProcessPO.getYarnAppId(),flinkProcessPO.getJobId())){
        throw new Exception("任务停止失败！");
    }

    ApplicationReport report = YarnClientProxy.getApplicationReportByAppId(flinkProcessPO.getYarnAppId());
    if(report != null){
        YarnApplicationState state = report.getYarnApplicationState();
        LOGGER.error("FlinkEngineServiceImpl stopTask report YarnApplicationState = " + state.toString() + " appId=" + flinkProcessPO.getYarnAppId());
        if (!(YarnApplicationState.FINISHED == state || YarnApplicationState.KILLED == state || YarnApplicationState.FAILED == state)) {
            try{
                YarnClientProxy.killApplicationByAppId(flinkProcessPO.getYarnAppId());
                LOGGER.error("FlinkEngineServiceImpl stopTask killApplicationByAppId is ok appId=" + flinkProcessPO.getYarnAppId());
            }catch (Exception e){
                LOGGER.error("FlinkEngineServiceImpl stopTask killApplicationByAppId is error appId=" + flinkProcessPO.getYarnAppId(),e);
                throw e;
            }
        }
    }

    TaskPO taskPO = new TaskPO();
    taskPO.setTaskStatus(TaskStatusEnum.STOP.getValue());
    taskPO.setTaskStopTime(new Date());
    taskPO.setId(flinkProcessPO.getTaskId());
    flinkEngineService.taskDao.update4Stop(taskPO);

    FlinkEngineCheckPointImpl.recoveryFail.remove(flinkProcessPO.getTaskId());
}

Source File: TestTezJobs.java From tez with Apache License 2.0

4 votes

@Test(timeout = 60000)
public void testSessionTimeout() throws Exception {
  Path stagingDirPath = new Path("/tmp/sessiontimeout-staging-dir");
  remoteFs.mkdirs(stagingDirPath);

  YarnClient yarnClient = YarnClient.createYarnClient();

  try {

    yarnClient.init(mrrTezCluster.getConfig());
    yarnClient.start();

    List<ApplicationReport> apps = yarnClient.getApplications();
    int appsBeforeCount = apps != null ? apps.size() : 0;

    TezConfiguration tezConf = new TezConfiguration(mrrTezCluster.getConfig());
    tezConf.set(TezConfiguration.TEZ_AM_STAGING_DIR, stagingDirPath.toString());
    tezConf.setInt(TezConfiguration.TEZ_SESSION_AM_DAG_SUBMIT_TIMEOUT_SECS, 5);
    TezClient tezClient = TezClient.create("testSessionTimeout", tezConf, true);
    tezClient.start();

    ApplicationId appId = tezClient.getAppMasterApplicationId();

    apps = yarnClient.getApplications();
    int appsAfterCount = apps != null ? apps.size() : 0;

    // Running in session mode. So should only create 1 more app.
    Assert.assertEquals(appsBeforeCount + 1, appsAfterCount);

    ApplicationReport report;
    while (true) {
      report = yarnClient.getApplicationReport(appId);
      if (report.getYarnApplicationState() == YarnApplicationState.FINISHED
          || report.getYarnApplicationState() == YarnApplicationState.FAILED
          || report.getYarnApplicationState() == YarnApplicationState.KILLED) {
        break;
      }
      Thread.sleep(1000);
    }
    // Add a sleep because YARN is not consistent in terms of reporting uptodate diagnostics
    Thread.sleep(2000);
    report = yarnClient.getApplicationReport(appId);
    LOG.info("App Report for appId=" + appId
        + ", report=" + report);
    Assert.assertTrue("Actual diagnostics: " + report.getDiagnostics(),
        report.getDiagnostics().contains("Session timed out"));

  } finally {
    remoteFs.delete(stagingDirPath, true);
    if (yarnClient != null) {
      yarnClient.stop();
    }
  }
}

Source File: YarnTestBase.java From flink with Apache License 2.0

4 votes

private static boolean isApplicationRunning(ApplicationReport app) {
	final YarnApplicationState yarnApplicationState = app.getYarnApplicationState();
	return yarnApplicationState != YarnApplicationState.FINISHED
		&& app.getYarnApplicationState() != YarnApplicationState.KILLED
		&& app.getYarnApplicationState() != YarnApplicationState.FAILED;
}

Source File: YarnTestBase.java From Flink-CEPplus with Apache License 2.0

4 votes

private static boolean isApplicationRunning(ApplicationReport app) {
	final YarnApplicationState yarnApplicationState = app.getYarnApplicationState();
	return yarnApplicationState != YarnApplicationState.FINISHED
		&& app.getYarnApplicationState() != YarnApplicationState.KILLED
		&& app.getYarnApplicationState() != YarnApplicationState.FAILED;
}

Source File: Client.java From big-c with Apache License 2.0

4 votes

/**
 * Monitor the submitted application for completion. 
 * Kill application if time expires. 
 * @param appId Application Id of application to be monitored
 * @return true if application completed successfully
 * @throws YarnException
 * @throws IOException
 */
private boolean monitorApplication(ApplicationId appId)
    throws YarnException, IOException {

  while (true) {

    // Check app status every 1 second.
    try {
      Thread.sleep(1000);
    } catch (InterruptedException e) {
      LOG.debug("Thread sleep in monitoring loop interrupted");
    }

    // Get application report for the appId we are interested in 
    ApplicationReport report = yarnClient.getApplicationReport(appId);

    LOG.info("Got application report from ASM for"
        + ", appId=" + appId.getId()
        + ", clientToAMToken=" + report.getClientToAMToken()
        + ", appDiagnostics=" + report.getDiagnostics()
        + ", appMasterHost=" + report.getHost()
        + ", appQueue=" + report.getQueue()
        + ", appMasterRpcPort=" + report.getRpcPort()
        + ", appStartTime=" + report.getStartTime()
        + ", yarnAppState=" + report.getYarnApplicationState().toString()
        + ", distributedFinalState=" + report.getFinalApplicationStatus().toString()
        + ", appTrackingUrl=" + report.getTrackingUrl()
        + ", appUser=" + report.getUser());

    YarnApplicationState state = report.getYarnApplicationState();
    FinalApplicationStatus dsStatus = report.getFinalApplicationStatus();
    if (YarnApplicationState.FINISHED == state) {
      if (FinalApplicationStatus.SUCCEEDED == dsStatus) {
        LOG.info("Application has completed successfully. Breaking monitoring loop");
        return true;        
      }
      else {
        LOG.info("Application did finished unsuccessfully."
            + " YarnState=" + state.toString() + ", DSFinalStatus=" + dsStatus.toString()
            + ". Breaking monitoring loop");
        return false;
      }			  
    }
    else if (YarnApplicationState.KILLED == state	
        || YarnApplicationState.FAILED == state) {
      LOG.info("Application did not finish."
          + " YarnState=" + state.toString() + ", DSFinalStatus=" + dsStatus.toString()
          + ". Breaking monitoring loop");
      return false;
    }			

    if (System.currentTimeMillis() > (clientStartTime + clientTimeout)) {
      LOG.info("Reached client specified timeout for application. Killing application");
      forceKillApplication(appId);
      return false;				
    }
  }			

}

Source File: YarnTestBase.java From flink with Apache License 2.0

4 votes

private static boolean isApplicationRunning(ApplicationReport app) {
	final YarnApplicationState yarnApplicationState = app.getYarnApplicationState();
	return yarnApplicationState != YarnApplicationState.FINISHED
		&& app.getYarnApplicationState() != YarnApplicationState.KILLED
		&& app.getYarnApplicationState() != YarnApplicationState.FAILED;
}

Java Code Examples for org.apache.hadoop.yarn.api.records.YarnApplicationState#FAILED