Java Code Examples for org.apache.hadoop.yarn.api.records.YarnApplicationState#KILLED
The following examples show how to use
org.apache.hadoop.yarn.api.records.YarnApplicationState#KILLED .
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: YARNRunner.java From hadoop with Apache License 2.0 | 7 votes |
private void killUnFinishedApplication(ApplicationId appId) throws IOException { ApplicationReport application = null; try { application = resMgrDelegate.getApplicationReport(appId); } catch (YarnException e) { throw new IOException(e); } if (application.getYarnApplicationState() == YarnApplicationState.FINISHED || application.getYarnApplicationState() == YarnApplicationState.FAILED || application.getYarnApplicationState() == YarnApplicationState.KILLED) { return; } killApplication(appId); }
Example 2
Source File: YarnApplicationStatusMonitor.java From Flink-CEPplus with Apache License 2.0 | 6 votes |
private void updateApplicationStatus() { if (yarnClient.isInState(Service.STATE.STARTED)) { final ApplicationReport applicationReport; try { applicationReport = yarnClient.getApplicationReport(yarnApplicationId); } catch (Exception e) { LOG.info("Could not retrieve the Yarn application report for {}.", yarnApplicationId); applicationStatus = ApplicationStatus.UNKNOWN; return; } YarnApplicationState yarnApplicationState = applicationReport.getYarnApplicationState(); if (yarnApplicationState == YarnApplicationState.FAILED || yarnApplicationState == YarnApplicationState.KILLED) { applicationStatus = ApplicationStatus.FAILED; } else { applicationStatus = ApplicationStatus.SUCCEEDED; } } else { LOG.info("Yarn client is no longer in state STARTED. Stopping the Yarn application status monitor."); applicationStatusUpdateFuture.cancel(false); } }
Example 3
Source File: RMServerUtils.java From big-c with Apache License 2.0 | 6 votes |
public static YarnApplicationState createApplicationState( RMAppState rmAppState) { switch (rmAppState) { case NEW: return YarnApplicationState.NEW; case NEW_SAVING: return YarnApplicationState.NEW_SAVING; case SUBMITTED: return YarnApplicationState.SUBMITTED; case ACCEPTED: return YarnApplicationState.ACCEPTED; case RUNNING: return YarnApplicationState.RUNNING; case FINISHING: case FINISHED: return YarnApplicationState.FINISHED; case KILLED: return YarnApplicationState.KILLED; case FAILED: return YarnApplicationState.FAILED; default: throw new YarnRuntimeException("Unknown state passed!"); } }
Example 4
Source File: RMServerUtils.java From hadoop with Apache License 2.0 | 6 votes |
public static YarnApplicationState createApplicationState( RMAppState rmAppState) { switch (rmAppState) { case NEW: return YarnApplicationState.NEW; case NEW_SAVING: return YarnApplicationState.NEW_SAVING; case SUBMITTED: return YarnApplicationState.SUBMITTED; case ACCEPTED: return YarnApplicationState.ACCEPTED; case RUNNING: return YarnApplicationState.RUNNING; case FINISHING: case FINISHED: return YarnApplicationState.FINISHED; case KILLED: return YarnApplicationState.KILLED; case FAILED: return YarnApplicationState.FAILED; default: throw new YarnRuntimeException("Unknown state passed!"); } }
Example 5
Source File: ApplicationCLI.java From hadoop with Apache License 2.0 | 5 votes |
/** * Moves the application with the given ID to the given queue. */ private void moveApplicationAcrossQueues(String applicationId, String queue) throws YarnException, IOException { ApplicationId appId = ConverterUtils.toApplicationId(applicationId); ApplicationReport appReport = client.getApplicationReport(appId); if (appReport.getYarnApplicationState() == YarnApplicationState.FINISHED || appReport.getYarnApplicationState() == YarnApplicationState.KILLED || appReport.getYarnApplicationState() == YarnApplicationState.FAILED) { sysout.println("Application " + applicationId + " has already finished "); } else { sysout.println("Moving application " + applicationId + " to queue " + queue); client.moveApplicationAcrossQueues(appId, queue); sysout.println("Successfully completed move."); } }
Example 6
Source File: YARNRunner.java From big-c with Apache License 2.0 | 5 votes |
private void killUnFinishedApplication(ApplicationId appId) throws IOException { ApplicationReport application = null; try { application = resMgrDelegate.getApplicationReport(appId); } catch (YarnException e) { throw new IOException(e); } if (application.getYarnApplicationState() == YarnApplicationState.FINISHED || application.getYarnApplicationState() == YarnApplicationState.FAILED || application.getYarnApplicationState() == YarnApplicationState.KILLED) { return; } killApplication(appId); }
Example 7
Source File: ApplicationCLI.java From big-c with Apache License 2.0 | 5 votes |
/** * Moves the application with the given ID to the given queue. */ private void moveApplicationAcrossQueues(String applicationId, String queue) throws YarnException, IOException { ApplicationId appId = ConverterUtils.toApplicationId(applicationId); ApplicationReport appReport = client.getApplicationReport(appId); if (appReport.getYarnApplicationState() == YarnApplicationState.FINISHED || appReport.getYarnApplicationState() == YarnApplicationState.KILLED || appReport.getYarnApplicationState() == YarnApplicationState.FAILED) { sysout.println("Application " + applicationId + " has already finished "); } else { sysout.println("Moving application " + applicationId + " to queue " + queue); client.moveApplicationAcrossQueues(appId, queue); sysout.println("Successfully completed move."); } }
Example 8
Source File: GobblinYarnAppLauncher.java From incubator-gobblin with Apache License 2.0 | 5 votes |
@Subscribe public void handleApplicationReportArrivalEvent(ApplicationReportArrivalEvent applicationReportArrivalEvent) { ApplicationReport applicationReport = applicationReportArrivalEvent.getApplicationReport(); YarnApplicationState appState = applicationReport.getYarnApplicationState(); LOGGER.info("Gobblin Yarn application state: " + appState.toString()); // Reset the count on failures to get the ApplicationReport when there's one success this.getApplicationReportFailureCount.set(0); if (appState == YarnApplicationState.FINISHED || appState == YarnApplicationState.FAILED || appState == YarnApplicationState.KILLED) { applicationCompleted = true; LOGGER.info("Gobblin Yarn application finished with final status: " + applicationReport.getFinalApplicationStatus().toString()); if (applicationReport.getFinalApplicationStatus() == FinalApplicationStatus.FAILED) { LOGGER.error("Gobblin Yarn application failed for the following reason: " + applicationReport.getDiagnostics()); } try { GobblinYarnAppLauncher.this.stop(); } catch (IOException ioe) { LOGGER.error("Failed to close the " + GobblinYarnAppLauncher.class.getSimpleName(), ioe); } catch (TimeoutException te) { LOGGER.error("Timeout in stopping the service manager", te); } finally { if (this.emailNotificationOnShutdown) { sendEmailOnShutdown(Optional.of(applicationReport)); } } } }
Example 9
Source File: TezClientUtils.java From incubator-tez with Apache License 2.0 | 5 votes |
static DAGClientAMProtocolBlockingPB getSessionAMProxy(YarnClient yarnClient, Configuration conf, ApplicationId applicationId) throws TezException, IOException { ApplicationReport appReport; try { appReport = yarnClient.getApplicationReport( applicationId); if(appReport == null) { throw new TezUncheckedException("Could not retrieve application report" + " from YARN, applicationId=" + applicationId); } YarnApplicationState appState = appReport.getYarnApplicationState(); if(appState != YarnApplicationState.RUNNING) { if (appState == YarnApplicationState.FINISHED || appState == YarnApplicationState.KILLED || appState == YarnApplicationState.FAILED) { throw new SessionNotRunning("Application not running" + ", applicationId=" + applicationId + ", yarnApplicationState=" + appReport.getYarnApplicationState() + ", finalApplicationStatus=" + appReport.getFinalApplicationStatus() + ", trackingUrl=" + appReport.getTrackingUrl()); } return null; } } catch (YarnException e) { throw new TezException(e); } return getAMProxy(conf, appReport.getHost(), appReport.getRpcPort(), appReport.getClientToAMToken()); }
Example 10
Source File: DAGClientImpl.java From tez with Apache License 2.0 | 5 votes |
private void checkAndSetDagCompletionStatus() { ApplicationReport appReport = realClient.getApplicationReportInternal(); if (appReport != null) { final YarnApplicationState appState = appReport.getYarnApplicationState(); if (appState == YarnApplicationState.FINISHED || appState == YarnApplicationState.FAILED || appState == YarnApplicationState.KILLED) { dagCompleted = true; } } }
Example 11
Source File: Client.java From Scribengin with GNU Affero General Public License v3.0 | 5 votes |
private boolean monitorApplication(ApplicationId appId) throws YarnException, IOException { boolean r = false; while (true) { try { Thread.sleep(1000); } catch (InterruptedException e) { r = false; break; } ApplicationReport report = yarnClient.getApplicationReport(appId); YarnApplicationState state = report.getYarnApplicationState(); FinalApplicationStatus status = report.getFinalApplicationStatus(); if (state == YarnApplicationState.FINISHED) { if (status == FinalApplicationStatus.SUCCEEDED) { LOG.info("Completed sucessfully."); r = true; break; } else { LOG.info("Application errored out. YarnState=" + state.toString() + ", finalStatue=" + status.toString()); r = false; break; } } else if (state == YarnApplicationState.KILLED || state == YarnApplicationState.FAILED) { LOG.info("Application errored out. YarnState=" + state.toString() + ", finalStatue=" + status.toString()); r = false; break; } }// while return r; }
Example 12
Source File: Client.java From hadoop-mini-clusters with Apache License 2.0 | 4 votes |
public void run(String[] args) throws Exception { final String command = args[0]; final int n = Integer.valueOf(args[1]); final Path jarPath = new Path(args[2]); final String resourceManagerAddress = args[3]; final String resourceManagerHostname = args[4]; final String resourceManagerSchedulerAddress = args[5]; final String resourceManagerResourceTrackerAddress = args[6]; // Create yarnClient YarnConfiguration conf = new YarnConfiguration(); conf.set("yarn.resourcemanager.address", resourceManagerAddress); conf.set("yarn.resourcemanager.hostname", resourceManagerHostname); conf.set("yarn.resourcemanager.scheduler.address", resourceManagerSchedulerAddress); conf.set("yarn.resourcemanager.resource-tracker.address", resourceManagerResourceTrackerAddress); YarnClient yarnClient = YarnClient.createYarnClient(); yarnClient.init(conf); yarnClient.start(); // Create application via yarnClient YarnClientApplication app = yarnClient.createApplication(); // Set up the container launch context for the application master ContainerLaunchContext amContainer = Records.newRecord(ContainerLaunchContext.class); amContainer.setCommands( Collections.singletonList( "$JAVA_HOME/bin/java" + " -Xmx256M" + " com.hortonworks.simpleyarnapp.ApplicationMaster" + " " + command + " " + String.valueOf(n) + " " + resourceManagerAddress + " " + resourceManagerHostname + " " + resourceManagerSchedulerAddress + " " + resourceManagerResourceTrackerAddress + " 1>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/stdout" + " 2>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/stderr" ) ); // Setup jar for ApplicationMaster LocalResource appMasterJar = Records.newRecord(LocalResource.class); setupAppMasterJar(jarPath, appMasterJar); amContainer.setLocalResources( Collections.singletonMap("simple-yarn-app-1.1.0.jar", appMasterJar)); // Setup CLASSPATH for ApplicationMaster Map<String, String> appMasterEnv = new HashMap<String, String>(); setupAppMasterEnv(appMasterEnv); amContainer.setEnvironment(appMasterEnv); // Set up resource type requirements for ApplicationMaster Resource capability = Records.newRecord(Resource.class); capability.setMemory(256); capability.setVirtualCores(1); // Finally, set-up ApplicationSubmissionContext for the application ApplicationSubmissionContext appContext = app.getApplicationSubmissionContext(); appContext.setApplicationName("simple-yarn-app"); // application name appContext.setAMContainerSpec(amContainer); appContext.setResource(capability); appContext.setQueue("default"); // queue // Submit application ApplicationId appId = appContext.getApplicationId(); System.out.println("Submitting application " + appId); yarnClient.submitApplication(appContext); ApplicationReport appReport = yarnClient.getApplicationReport(appId); YarnApplicationState appState = appReport.getYarnApplicationState(); while (appState != YarnApplicationState.FINISHED && appState != YarnApplicationState.KILLED && appState != YarnApplicationState.FAILED) { Thread.sleep(100); appReport = yarnClient.getApplicationReport(appId); appState = appReport.getYarnApplicationState(); } System.out.println( "Application " + appId + " finished with" + " state " + appState + " at " + appReport.getFinishTime()); }
Example 13
Source File: Client.java From hadoop with Apache License 2.0 | 4 votes |
/** * Monitor the submitted application for completion. * Kill application if time expires. * @param appId Application Id of application to be monitored * @return true if application completed successfully * @throws YarnException * @throws IOException */ private boolean monitorApplication(ApplicationId appId) throws YarnException, IOException { while (true) { // Check app status every 1 second. try { Thread.sleep(1000); } catch (InterruptedException e) { LOG.debug("Thread sleep in monitoring loop interrupted"); } // Get application report for the appId we are interested in ApplicationReport report = yarnClient.getApplicationReport(appId); LOG.info("Got application report from ASM for" + ", appId=" + appId.getId() + ", clientToAMToken=" + report.getClientToAMToken() + ", appDiagnostics=" + report.getDiagnostics() + ", appMasterHost=" + report.getHost() + ", appQueue=" + report.getQueue() + ", appMasterRpcPort=" + report.getRpcPort() + ", appStartTime=" + report.getStartTime() + ", yarnAppState=" + report.getYarnApplicationState().toString() + ", distributedFinalState=" + report.getFinalApplicationStatus().toString() + ", appTrackingUrl=" + report.getTrackingUrl() + ", appUser=" + report.getUser()); YarnApplicationState state = report.getYarnApplicationState(); FinalApplicationStatus dsStatus = report.getFinalApplicationStatus(); if (YarnApplicationState.FINISHED == state) { if (FinalApplicationStatus.SUCCEEDED == dsStatus) { LOG.info("Application has completed successfully. Breaking monitoring loop"); return true; } else { LOG.info("Application did finished unsuccessfully." + " YarnState=" + state.toString() + ", DSFinalStatus=" + dsStatus.toString() + ". Breaking monitoring loop"); return false; } } else if (YarnApplicationState.KILLED == state || YarnApplicationState.FAILED == state) { LOG.info("Application did not finish." + " YarnState=" + state.toString() + ", DSFinalStatus=" + dsStatus.toString() + ". Breaking monitoring loop"); return false; } if (System.currentTimeMillis() > (clientStartTime + clientTimeout)) { LOG.info("Reached client specified timeout for application. Killing application"); forceKillApplication(appId); return false; } } }
Example 14
Source File: FlinkEngineServiceImpl.java From PoseidonX with Apache License 2.0 | 4 votes |
public static void stopTask(Integer taskId, String taskName, Integer processId) throws Exception { if(taskId == null){ throw new Exception("任务停止失败,参数异常,taskId为空!"); } if(StringUtils.isBlank(taskName)){ throw new Exception("任务提交失败,参数异常,任务名为空!"); } if(processId == null){ throw new Exception("任务停止失败,参数异常,processId为空!"); } FlinkProcessPO flinkProcessPO = flinkEngineService.flinkProcessDao.getById(processId); if(flinkProcessPO == null){ throw new Exception("任务停止失败,任务的执行信息不存在或者已经被删除!"); } EngineContant.FLINK_HOME = ConfigProperty.getConfigValue(ConfigKeyEnum.FLINK_HOME); if(StringUtils.isBlank(EngineContant.FLINK_HOME)){ throw new Exception("任务停止失败,参数异常,系统参数 FLINK_HOME 未进行配置!"); } if(!FlinkOnYarnBusiness.stopJob(flinkProcessPO.getYarnAppId(),flinkProcessPO.getJobId())){ throw new Exception("任务停止失败!"); } ApplicationReport report = YarnClientProxy.getApplicationReportByAppId(flinkProcessPO.getYarnAppId()); if(report != null){ YarnApplicationState state = report.getYarnApplicationState(); LOGGER.error("FlinkEngineServiceImpl stopTask report YarnApplicationState = " + state.toString() + " appId=" + flinkProcessPO.getYarnAppId()); if (!(YarnApplicationState.FINISHED == state || YarnApplicationState.KILLED == state || YarnApplicationState.FAILED == state)) { try{ YarnClientProxy.killApplicationByAppId(flinkProcessPO.getYarnAppId()); LOGGER.error("FlinkEngineServiceImpl stopTask killApplicationByAppId is ok appId=" + flinkProcessPO.getYarnAppId()); }catch (Exception e){ LOGGER.error("FlinkEngineServiceImpl stopTask killApplicationByAppId is error appId=" + flinkProcessPO.getYarnAppId(),e); throw e; } } } TaskPO taskPO = new TaskPO(); taskPO.setTaskStatus(TaskStatusEnum.STOP.getValue()); taskPO.setTaskStopTime(new Date()); taskPO.setId(flinkProcessPO.getTaskId()); flinkEngineService.taskDao.update4Stop(taskPO); FlinkEngineCheckPointImpl.recoveryFail.remove(flinkProcessPO.getTaskId()); }
Example 15
Source File: StramClientUtils.java From attic-apex-core with Apache License 2.0 | 4 votes |
/** * Monitor the submitted application for completion. Kill application if time expires. * * @param appId Application Id of application to be monitored * @param callback * @param timeoutMillis * @return true if application completed successfully * @throws YarnException * @throws IOException */ @SuppressWarnings("SleepWhileInLoop") public boolean waitForCompletion(ApplicationId appId, AppStatusCallback callback, long timeoutMillis) throws YarnException, IOException { long startMillis = System.currentTimeMillis(); while (true) { // Check app status every 1 second. try { Thread.sleep(1000); } catch (InterruptedException e) { LOG.debug("Thread sleep in monitoring loop interrupted"); } ApplicationReport report = clientRM.getApplicationReport(appId); if (callback.exitLoop(report) == true) { return true; } YarnApplicationState state = report.getYarnApplicationState(); FinalApplicationStatus dsStatus = report.getFinalApplicationStatus(); if (YarnApplicationState.FINISHED == state) { if (FinalApplicationStatus.SUCCEEDED == dsStatus) { LOG.info("Application has completed successfully. Breaking monitoring loop"); return true; } else { LOG.info("Application finished unsuccessfully." + " YarnState=" + state.toString() + ", DSFinalStatus=" + dsStatus.toString() + ". Breaking monitoring loop"); return false; } } else if (YarnApplicationState.KILLED == state || YarnApplicationState.FAILED == state) { LOG.info("Application did not finish." + " YarnState=" + state.toString() + ", DSFinalStatus=" + dsStatus.toString() + ". Breaking monitoring loop"); return false; } if (System.currentTimeMillis() - startMillis > timeoutMillis) { LOG.info("Reached specified timeout. Killing application"); clientRM.killApplication(appId); return false; } } }
Example 16
Source File: YarnTestBase.java From flink with Apache License 2.0 | 4 votes |
private static boolean isApplicationRunning(ApplicationReport app) { final YarnApplicationState yarnApplicationState = app.getYarnApplicationState(); return yarnApplicationState != YarnApplicationState.FINISHED && app.getYarnApplicationState() != YarnApplicationState.KILLED && app.getYarnApplicationState() != YarnApplicationState.FAILED; }
Example 17
Source File: YarnTestBase.java From flink with Apache License 2.0 | 4 votes |
private static boolean isApplicationRunning(ApplicationReport app) { final YarnApplicationState yarnApplicationState = app.getYarnApplicationState(); return yarnApplicationState != YarnApplicationState.FINISHED && app.getYarnApplicationState() != YarnApplicationState.KILLED && app.getYarnApplicationState() != YarnApplicationState.FAILED; }
Example 18
Source File: YarnTestBase.java From Flink-CEPplus with Apache License 2.0 | 4 votes |
private static boolean isApplicationRunning(ApplicationReport app) { final YarnApplicationState yarnApplicationState = app.getYarnApplicationState(); return yarnApplicationState != YarnApplicationState.FINISHED && app.getYarnApplicationState() != YarnApplicationState.KILLED && app.getYarnApplicationState() != YarnApplicationState.FAILED; }
Example 19
Source File: Client.java From metron with Apache License 2.0 | 4 votes |
/** * Monitor the submitted application for completion. * Kill application if time expires. * @param appId Application Id of application to be monitored * @return true if application completed successfully * @throws YarnException * @throws IOException */ private boolean monitorApplication(ApplicationId appId) throws YarnException, IOException { while (true) { // Check app status every 1 second. try { Thread.sleep(1000); } catch (InterruptedException e) { LOG.debug("Thread sleep in monitoring loop interrupted"); } // Get application report for the appId we are interested in ApplicationReport report = yarnClient.getApplicationReport(appId); LOG.info("Got application report from ASM for" + ", appId=" + appId.getId() + ", clientToAMToken=" + report.getClientToAMToken() + ", appDiagnostics=" + report.getDiagnostics() + ", appMasterHost=" + report.getHost() + ", appQueue=" + report.getQueue() + ", appMasterRpcPort=" + report.getRpcPort() + ", appStartTime=" + report.getStartTime() + ", yarnAppState=" + report.getYarnApplicationState().toString() + ", distributedFinalState=" + report.getFinalApplicationStatus().toString() + ", appTrackingUrl=" + report.getTrackingUrl() + ", appUser=" + report.getUser()); YarnApplicationState state = report.getYarnApplicationState(); FinalApplicationStatus dsStatus = report.getFinalApplicationStatus(); if(YarnApplicationState.RUNNING == state) { LOG.info("Application is running..."); return true; } if (YarnApplicationState.FINISHED == state) { if (FinalApplicationStatus.SUCCEEDED == dsStatus) { LOG.info("Application has completed successfully. Breaking monitoring loop"); return true; } else { LOG.info("Application did finished unsuccessfully." + " YarnState=" + state.toString() + ", DSFinalStatus=" + dsStatus.toString() + ". Breaking monitoring loop"); return false; } } else if (YarnApplicationState.KILLED == state || YarnApplicationState.FAILED == state) { LOG.info("Application did not finish." + " YarnState=" + state.toString() + ", DSFinalStatus=" + dsStatus.toString() + ". Breaking monitoring loop"); return false; } if (System.currentTimeMillis() > (clientStartTime + clientTimeout)) { LOG.info("Reached client specified timeout for application. Killing application"); forceKillApplication(appId); return false; } } }
Example 20
Source File: TezClient.java From tez with Apache License 2.0 | 4 votes |
private boolean isJobInTerminalState(YarnApplicationState yarnApplicationState) { return (yarnApplicationState == YarnApplicationState.FINISHED || yarnApplicationState == YarnApplicationState.FAILED || yarnApplicationState == YarnApplicationState.KILLED); }