Java Code Examples for org.apache.hadoop.yarn.api.records.FinalApplicationStatus#SUCCEEDED
The following examples show how to use
org.apache.hadoop.yarn.api.records.FinalApplicationStatus#SUCCEEDED .
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
| Source File: TypeConverter.java From big-c with Apache License 2.0 | 6 votes |
|
public static State fromYarn(YarnApplicationState yarnApplicationState,
FinalApplicationStatus finalApplicationStatus) {
switch (yarnApplicationState) {
case NEW:
case NEW_SAVING:
case SUBMITTED:
case ACCEPTED:
return State.PREP;
case RUNNING:
return State.RUNNING;
case FINISHED:
if (finalApplicationStatus == FinalApplicationStatus.SUCCEEDED) {
return State.SUCCEEDED;
} else if (finalApplicationStatus == FinalApplicationStatus.KILLED) {
return State.KILLED;
}
case FAILED:
return State.FAILED;
case KILLED:
return State.KILLED;
}
throw new YarnRuntimeException("Unrecognized application state: " + yarnApplicationState);
}
Example 2
| Source File: TestRMAppAttemptTransitions.java From big-c with Apache License 2.0 | 6 votes |
|
@Test
public void testFinishingToFinishing() {
Container amContainer = allocateApplicationAttempt();
launchApplicationAttempt(amContainer);
runApplicationAttempt(amContainer, "host", 8042, "oldtrackingurl", false);
FinalApplicationStatus finalStatus = FinalApplicationStatus.SUCCEEDED;
String trackingUrl = "mytrackingurl";
String diagnostics = "Successful";
unregisterApplicationAttempt(amContainer, finalStatus, trackingUrl,
diagnostics);
// container must be AM container to move from FINISHING to FINISHED
NodeId anyNodeId = NodeId.newInstance("host", 1234);
applicationAttempt.handle(
new RMAppAttemptContainerFinishedEvent(
applicationAttempt.getAppAttemptId(),
BuilderUtils.newContainerStatus(
BuilderUtils.newContainerId(
applicationAttempt.getAppAttemptId(), 42),
ContainerState.COMPLETE, "", 0), anyNodeId));
testAppAttemptFinishingState(amContainer, finalStatus, trackingUrl,
diagnostics);
}
Example 3
| Source File: TypeConverter.java From hadoop with Apache License 2.0 | 6 votes |
|
public static State fromYarn(YarnApplicationState yarnApplicationState,
FinalApplicationStatus finalApplicationStatus) {
switch (yarnApplicationState) {
case NEW:
case NEW_SAVING:
case SUBMITTED:
case ACCEPTED:
return State.PREP;
case RUNNING:
return State.RUNNING;
case FINISHED:
if (finalApplicationStatus == FinalApplicationStatus.SUCCEEDED) {
return State.SUCCEEDED;
} else if (finalApplicationStatus == FinalApplicationStatus.KILLED) {
return State.KILLED;
}
case FAILED:
return State.FAILED;
case KILLED:
return State.KILLED;
}
throw new YarnRuntimeException("Unrecognized application state: " + yarnApplicationState);
}
Example 4
| Source File: LocalClient.java From tez with Apache License 2.0 | 6 votes |
|
protected FinalApplicationStatus convertDAGAppMasterStateToFinalYARNState(
DAGAppMasterState dagAppMasterState) {
switch (dagAppMasterState) {
case NEW:
case INITED:
case RECOVERING:
case IDLE:
case RUNNING:
return FinalApplicationStatus.UNDEFINED;
case SUCCEEDED:
return FinalApplicationStatus.SUCCEEDED;
case FAILED:
return FinalApplicationStatus.FAILED;
case KILLED:
return FinalApplicationStatus.KILLED;
case ERROR:
return FinalApplicationStatus.FAILED;
default:
return FinalApplicationStatus.UNDEFINED;
}
}
Example 5
| Source File: TestRMAppAttemptTransitions.java From hadoop with Apache License 2.0 | 6 votes |
|
@Test
public void testFinalSavingToFinishedWithExpire() {
Container amContainer = allocateApplicationAttempt();
launchApplicationAttempt(amContainer);
runApplicationAttempt(amContainer, "host", 8042, "oldtrackingurl", false);
FinalApplicationStatus finalStatus = FinalApplicationStatus.SUCCEEDED;
String trackingUrl = "mytrackingurl";
String diagnostics = "Successssseeeful";
applicationAttempt.handle(new RMAppAttemptUnregistrationEvent(
applicationAttempt.getAppAttemptId(), trackingUrl, finalStatus,
diagnostics));
assertEquals(RMAppAttemptState.FINAL_SAVING,
applicationAttempt.getAppAttemptState());
assertEquals(YarnApplicationAttemptState.RUNNING,
applicationAttempt.createApplicationAttemptState());
// Expire event comes before Attempt_saved event.
applicationAttempt.handle(new RMAppAttemptEvent(applicationAttempt
.getAppAttemptId(), RMAppAttemptEventType.EXPIRE));
assertEquals(RMAppAttemptState.FINAL_SAVING,
applicationAttempt.getAppAttemptState());
// send attempt_saved
sendAttemptUpdateSavedEvent(applicationAttempt);
testAppAttemptFinishedState(amContainer, finalStatus, trackingUrl,
diagnostics, 0, false);
}
Example 6
| Source File: StreamingAppMasterService.java From attic-apex-core with Apache License 2.0 | 6 votes |
|
private void finishApplication(FinalApplicationStatus finalStatus) throws YarnException, IOException
{
LOG.info("Application completed. Signalling finish to RM");
FinishApplicationMasterRequest finishReq = Records.newRecord(FinishApplicationMasterRequest.class);
finishReq.setFinalApplicationStatus(finalStatus);
if (finalStatus != FinalApplicationStatus.SUCCEEDED) {
String diagnostics = "Diagnostics." + " completed=" + numCompletedContainers.get() + ", allocated=" + allocatedContainers.size() + ", failed=" + numFailedContainers.get();
if (!StringUtils.isEmpty(dnmgr.shutdownDiagnosticsMessage)) {
diagnostics += "\n";
diagnostics += dnmgr.shutdownDiagnosticsMessage;
}
// YARN-208 - as of 2.0.1-alpha dropped by the RM
finishReq.setDiagnostics(diagnostics);
// expected termination of the master process
// application status and diagnostics message are set above
}
LOG.info("diagnostics: " + finishReq.getDiagnostics());
amRmClient.unregisterApplicationMaster(finishReq.getFinalApplicationStatus(), finishReq.getDiagnostics(), null);
}
Example 7
| Source File: YarnResourceManager.java From flink with Apache License 2.0 | 6 votes |
|
/**
* Converts a Flink application status enum to a YARN application status enum.
* @param status The Flink application status.
* @return The corresponding YARN application status.
*/
private FinalApplicationStatus getYarnStatus(ApplicationStatus status) {
if (status == null) {
return FinalApplicationStatus.UNDEFINED;
}
else {
switch (status) {
case SUCCEEDED:
return FinalApplicationStatus.SUCCEEDED;
case FAILED:
return FinalApplicationStatus.FAILED;
case CANCELED:
return FinalApplicationStatus.KILLED;
default:
return FinalApplicationStatus.UNDEFINED;
}
}
}
Example 8
| Source File: TestRMAppAttemptTransitions.java From hadoop with Apache License 2.0 | 6 votes |
|
@Test
public void testFinishingExpire() {
Container amContainer = allocateApplicationAttempt();
launchApplicationAttempt(amContainer);
runApplicationAttempt(amContainer, "host", 8042, "oldtrackingurl", false);
FinalApplicationStatus finalStatus = FinalApplicationStatus.SUCCEEDED;
String trackingUrl = "mytrackingurl";
String diagnostics = "Successful";
unregisterApplicationAttempt(amContainer, finalStatus, trackingUrl,
diagnostics);
applicationAttempt.handle(
new RMAppAttemptEvent(
applicationAttempt.getAppAttemptId(),
RMAppAttemptEventType.EXPIRE));
testAppAttemptFinishedState(amContainer, finalStatus, trackingUrl,
diagnostics, 0, false);
}
Example 9
| Source File: TestRMAppAttemptTransitions.java From hadoop with Apache License 2.0 | 5 votes |
|
@Test
public void
testFinalSavingToFinishedWithContainerFinished() {
Container amContainer = allocateApplicationAttempt();
launchApplicationAttempt(amContainer);
runApplicationAttempt(amContainer, "host", 8042, "oldtrackingurl", false);
FinalApplicationStatus finalStatus = FinalApplicationStatus.SUCCEEDED;
String trackingUrl = "mytrackingurl";
String diagnostics = "Successful";
applicationAttempt.handle(new RMAppAttemptUnregistrationEvent(
applicationAttempt.getAppAttemptId(), trackingUrl, finalStatus,
diagnostics));
assertEquals(RMAppAttemptState.FINAL_SAVING,
applicationAttempt.getAppAttemptState());
assertEquals(YarnApplicationAttemptState.RUNNING,
applicationAttempt.createApplicationAttemptState());
// Container_finished event comes before Attempt_Saved event.
NodeId anyNodeId = NodeId.newInstance("host", 1234);
applicationAttempt.handle(new RMAppAttemptContainerFinishedEvent(
applicationAttempt.getAppAttemptId(), BuilderUtils.newContainerStatus(
amContainer.getId(), ContainerState.COMPLETE, "", 0), anyNodeId));
assertEquals(RMAppAttemptState.FINAL_SAVING,
applicationAttempt.getAppAttemptState());
// send attempt_saved
sendAttemptUpdateSavedEvent(applicationAttempt);
testAppAttemptFinishedState(amContainer, finalStatus, trackingUrl,
diagnostics, 0, false);
}
Example 10
| Source File: ApplicationMaster.java From TensorFlowOnYARN with Apache License 2.0 | 5 votes |
|
private FinalApplicationStatus getFinalAppStatus() {
if (completedContainerNum.get() - failedContainerNum.get() >= args.totalContainerNum) {
return FinalApplicationStatus.SUCCEEDED;
} else {
return FinalApplicationStatus.FAILED;
}
}
Example 11
| Source File: YarnRMContainerAllocator.java From incubator-tajo with Apache License 2.0 | 5 votes |
|
public void stop() {
if(stopped.get()) {
return;
}
LOG.info("un-registering ApplicationMaster(QueryMaster):" + appAttemptId);
stopped.set(true);
try {
FinalApplicationStatus status = FinalApplicationStatus.UNDEFINED;
Query query = context.getQuery();
if (query != null) {
TajoProtos.QueryState state = query.getState();
if (state == TajoProtos.QueryState.QUERY_SUCCEEDED) {
status = FinalApplicationStatus.SUCCEEDED;
} else if (state == TajoProtos.QueryState.QUERY_FAILED || state == TajoProtos.QueryState.QUERY_ERROR) {
status = FinalApplicationStatus.FAILED;
} else if (state == TajoProtos.QueryState.QUERY_ERROR) {
status = FinalApplicationStatus.FAILED;
}
}
unregisterApplicationMaster(status, "tajo query finished", null);
} catch (Exception e) {
LOG.error(e.getMessage(), e);
}
allocatorThread.interrupt();
LOG.info("un-registered ApplicationMAster(QueryMaster) stopped:" + appAttemptId);
super.stop();
}
Example 12
| Source File: TaskSchedulerManager.java From tez with Apache License 2.0 | 5 votes |
|
public AppFinalStatus getFinalAppStatus() {
FinalApplicationStatus finishState = FinalApplicationStatus.UNDEFINED;
StringBuffer sb = new StringBuffer();
if (dagAppMaster == null) {
finishState = FinalApplicationStatus.UNDEFINED;
sb.append("App not yet initialized");
} else {
DAGAppMasterState appMasterState = dagAppMaster.getState();
if (appMasterState == DAGAppMasterState.SUCCEEDED) {
finishState = FinalApplicationStatus.SUCCEEDED;
} else if (appMasterState == DAGAppMasterState.KILLED
|| (appMasterState == DAGAppMasterState.RUNNING && isSignalled)) {
finishState = FinalApplicationStatus.KILLED;
} else if (appMasterState == DAGAppMasterState.FAILED
|| appMasterState == DAGAppMasterState.ERROR) {
finishState = FinalApplicationStatus.FAILED;
} else {
finishState = FinalApplicationStatus.UNDEFINED;
}
finishState = hadoopShim.applyFinalApplicationStatusCorrection(finishState,
dagAppMaster.isSession(), appMasterState == DAGAppMasterState.ERROR);
List<String> diagnostics = dagAppMaster.getDiagnostics();
if(diagnostics != null) {
for (String s : diagnostics) {
sb.append(s).append("\n");
}
}
}
if(LOG.isDebugEnabled()) {
LOG.debug("Setting job diagnostics to " + sb.toString());
}
// if history url is set use the same, if historyUrl is set to "" then rm ui disables the
// history url
return new AppFinalStatus(finishState, sb.toString(), historyUrl);
}
Example 13
| Source File: TaskSchedulerEventHandler.java From incubator-tez with Apache License 2.0 | 5 votes |
|
@Override
public AppFinalStatus getFinalAppStatus() {
FinalApplicationStatus finishState = FinalApplicationStatus.UNDEFINED;
StringBuffer sb = new StringBuffer();
if (dagAppMaster == null) {
finishState = FinalApplicationStatus.UNDEFINED;
sb.append("App not yet initialized");
} else {
DAGAppMasterState appMasterState = dagAppMaster.getState();
if (appMasterState == DAGAppMasterState.SUCCEEDED) {
finishState = FinalApplicationStatus.SUCCEEDED;
} else if (appMasterState == DAGAppMasterState.KILLED
|| (appMasterState == DAGAppMasterState.RUNNING && isSignalled)) {
finishState = FinalApplicationStatus.KILLED;
} else if (appMasterState == DAGAppMasterState.FAILED
|| appMasterState == DAGAppMasterState.ERROR) {
finishState = FinalApplicationStatus.FAILED;
} else {
finishState = FinalApplicationStatus.UNDEFINED;
}
List<String> diagnostics = dagAppMaster.getDiagnostics();
if(diagnostics != null) {
for (String s : diagnostics) {
sb.append(s).append("\n");
}
}
}
if(LOG.isDebugEnabled()) {
LOG.debug("Setting job diagnostics to " + sb.toString());
}
String historyUrl = "";
/*String historyUrl = JobHistoryUtils.getHistoryUrl(getConfig(),
appContext.getApplicationID());
LOG.info("History url is " + historyUrl);*/
return new AppFinalStatus(finishState, sb.toString(), historyUrl);
}
Example 14
| Source File: InlineAM.java From attic-apex-core with Apache License 2.0 | 4 votes |
|
public boolean run() throws Exception
{
LOG.info("Starting Client");
// Connect to ResourceManager
rmClient.start();
try {
// Get a new application id
YarnClientApplication newApp = rmClient.createApplication();
ApplicationId appId = newApp.getNewApplicationResponse().getApplicationId();
// Create launch context for app master
LOG.info("Setting up application submission context for ASM");
ApplicationSubmissionContext appContext = Records
.newRecord(ApplicationSubmissionContext.class);
// set the application id
appContext.setApplicationId(appId);
// set the application name
appContext.setApplicationName(appName);
// Set the priority for the application master
Priority pri = Records.newRecord(Priority.class);
pri.setPriority(amPriority);
appContext.setPriority(pri);
// Set the queue to which this application is to be submitted in the RM
appContext.setQueue(amQueue);
// Set up the container launch context for the application master
ContainerLaunchContext amContainer = Records
.newRecord(ContainerLaunchContext.class);
appContext.setAMContainerSpec(amContainer);
// unmanaged AM
appContext.setUnmanagedAM(true);
LOG.info("Setting unmanaged AM");
// Submit the application to the applications manager
LOG.info("Submitting application to ASM");
rmClient.submitApplication(appContext);
// Monitor the application to wait for launch state
ApplicationReport appReport = monitorApplication(appId,
EnumSet.of(YarnApplicationState.ACCEPTED));
ApplicationAttemptId attemptId = appReport.getCurrentApplicationAttemptId();
LOG.info("Launching application with id: " + attemptId);
// launch AM
runAM(attemptId);
// Monitor the application for end state
appReport = monitorApplication(appId, EnumSet.of(
YarnApplicationState.KILLED, YarnApplicationState.FAILED,
YarnApplicationState.FINISHED));
YarnApplicationState appState = appReport.getYarnApplicationState();
FinalApplicationStatus appStatus = appReport.getFinalApplicationStatus();
LOG.info("App ended with state: " + appReport.getYarnApplicationState()
+ " and status: " + appStatus);
boolean success;
if (YarnApplicationState.FINISHED == appState
&& FinalApplicationStatus.SUCCEEDED == appStatus) {
LOG.info("Application has completed successfully.");
success = true;
} else {
LOG.info("Application did finished unsuccessfully." + " YarnState="
+ appState.toString() + ", FinalStatus=" + appStatus.toString());
success = false;
}
return success;
} finally {
rmClient.stop();
}
}
Example 15
| Source File: YarnScribeConsumerManager.java From Scribengin with GNU Affero General Public License v3.0 | 4 votes |
|
@Override
public void monitorConsumers(){
LinkedList<ScribeConsumerConfig> toAdd = new LinkedList<ScribeConsumerConfig>();
synchronized(yarnApps){
Iterator<YarnInfo> it = yarnApps.iterator();
while (it.hasNext()) {
YarnInfo yarnInfo = it.next();
ApplicationReport report = null;
Client c = null;
try {
c = yarnInfo.client;
report = c.getYarnClient().getApplicationReport(c.getAppId());
} catch (YarnException | IOException e) {
e.printStackTrace();
};
YarnApplicationState state = report.getYarnApplicationState();
FinalApplicationStatus status = report.getFinalApplicationStatus();
if (state == YarnApplicationState.FINISHED || state == YarnApplicationState.KILLED ) {
if (status == FinalApplicationStatus.SUCCEEDED) {
LOG.info("Application completed successfully: "+c.getAppId().toString());
it.remove();
break;
}
else if(state == YarnApplicationState.ACCEPTED){
//Do nothing
}
else {
LOG.info("Application finished but errored out. YarnState=" + state.toString() + ", finalStatue=" + status.toString() + ", AppId: "+c.getAppId().toString());
yarnInfo.conf.cleanStart = false;
toAdd.add(new ScribeConsumerConfig(yarnInfo.conf));
it.remove();
break;
}
} else if (state == YarnApplicationState.FAILED) {
LOG.info("Application errored out. YarnState=" + state.toString() + ", finalStatue=" + status.toString() + ", AppId: "+c.getAppId().toString());
it.remove();
break;
}
}
}
for(ScribeConsumerConfig sci: toAdd){
startNewConsumer(sci);
}
}
Example 16
| Source File: RMCommunicator.java From hadoop with Apache License 2.0 | 4 votes |
|
@VisibleForTesting
protected void doUnregistration()
throws YarnException, IOException, InterruptedException {
FinalApplicationStatus finishState = FinalApplicationStatus.UNDEFINED;
JobImpl jobImpl = (JobImpl)job;
if (jobImpl.getInternalState() == JobStateInternal.SUCCEEDED) {
finishState = FinalApplicationStatus.SUCCEEDED;
} else if (jobImpl.getInternalState() == JobStateInternal.KILLED
|| (jobImpl.getInternalState() == JobStateInternal.RUNNING && isSignalled)) {
finishState = FinalApplicationStatus.KILLED;
} else if (jobImpl.getInternalState() == JobStateInternal.FAILED
|| jobImpl.getInternalState() == JobStateInternal.ERROR) {
finishState = FinalApplicationStatus.FAILED;
}
StringBuffer sb = new StringBuffer();
for (String s : job.getDiagnostics()) {
sb.append(s).append("\n");
}
LOG.info("Setting job diagnostics to " + sb.toString());
String historyUrl =
MRWebAppUtil.getApplicationWebURLOnJHSWithScheme(getConfig(),
context.getApplicationID());
LOG.info("History url is " + historyUrl);
FinishApplicationMasterRequest request =
FinishApplicationMasterRequest.newInstance(finishState,
sb.toString(), historyUrl);
try {
while (true) {
FinishApplicationMasterResponse response =
scheduler.finishApplicationMaster(request);
if (response.getIsUnregistered()) {
// When excepting ClientService, other services are already stopped,
// it is safe to let clients know the final states. ClientService
// should wait for some time so clients have enough time to know the
// final states.
RunningAppContext raContext = (RunningAppContext) context;
raContext.markSuccessfulUnregistration();
break;
}
LOG.info("Waiting for application to be successfully unregistered.");
Thread.sleep(rmPollInterval);
}
} catch (ApplicationMasterNotRegisteredException e) {
// RM might have restarted or failed over and so lost the fact that AM had
// registered before.
register();
doUnregistration();
}
}
Example 17
| Source File: StramClientUtils.java From attic-apex-core with Apache License 2.0 | 4 votes |
|
/**
* Monitor the submitted application for completion. Kill application if time expires.
*
* @param appId Application Id of application to be monitored
* @param callback
* @param timeoutMillis
* @return true if application completed successfully
* @throws YarnException
* @throws IOException
*/
@SuppressWarnings("SleepWhileInLoop")
public boolean waitForCompletion(ApplicationId appId, AppStatusCallback callback, long timeoutMillis) throws YarnException, IOException
{
long startMillis = System.currentTimeMillis();
while (true) {
// Check app status every 1 second.
try {
Thread.sleep(1000);
} catch (InterruptedException e) {
LOG.debug("Thread sleep in monitoring loop interrupted");
}
ApplicationReport report = clientRM.getApplicationReport(appId);
if (callback.exitLoop(report) == true) {
return true;
}
YarnApplicationState state = report.getYarnApplicationState();
FinalApplicationStatus dsStatus = report.getFinalApplicationStatus();
if (YarnApplicationState.FINISHED == state) {
if (FinalApplicationStatus.SUCCEEDED == dsStatus) {
LOG.info("Application has completed successfully. Breaking monitoring loop");
return true;
} else {
LOG.info("Application finished unsuccessfully."
+ " YarnState=" + state.toString() + ", DSFinalStatus=" + dsStatus.toString()
+ ". Breaking monitoring loop");
return false;
}
} else if (YarnApplicationState.KILLED == state
|| YarnApplicationState.FAILED == state) {
LOG.info("Application did not finish."
+ " YarnState=" + state.toString() + ", DSFinalStatus=" + dsStatus.toString()
+ ". Breaking monitoring loop");
return false;
}
if (System.currentTimeMillis() - startMillis > timeoutMillis) {
LOG.info("Reached specified timeout. Killing application");
clientRM.killApplication(appId);
return false;
}
}
}
Example 18
| Source File: Client.java From metron with Apache License 2.0 | 4 votes |
|
/**
* Monitor the submitted application for completion.
* Kill application if time expires.
* @param appId Application Id of application to be monitored
* @return true if application completed successfully
* @throws YarnException
* @throws IOException
*/
private boolean monitorApplication(ApplicationId appId)
throws YarnException, IOException {
while (true) {
// Check app status every 1 second.
try {
Thread.sleep(1000);
} catch (InterruptedException e) {
LOG.debug("Thread sleep in monitoring loop interrupted");
}
// Get application report for the appId we are interested in
ApplicationReport report = yarnClient.getApplicationReport(appId);
LOG.info("Got application report from ASM for"
+ ", appId=" + appId.getId()
+ ", clientToAMToken=" + report.getClientToAMToken()
+ ", appDiagnostics=" + report.getDiagnostics()
+ ", appMasterHost=" + report.getHost()
+ ", appQueue=" + report.getQueue()
+ ", appMasterRpcPort=" + report.getRpcPort()
+ ", appStartTime=" + report.getStartTime()
+ ", yarnAppState=" + report.getYarnApplicationState().toString()
+ ", distributedFinalState=" + report.getFinalApplicationStatus().toString()
+ ", appTrackingUrl=" + report.getTrackingUrl()
+ ", appUser=" + report.getUser());
YarnApplicationState state = report.getYarnApplicationState();
FinalApplicationStatus dsStatus = report.getFinalApplicationStatus();
if(YarnApplicationState.RUNNING == state) {
LOG.info("Application is running...");
return true;
}
if (YarnApplicationState.FINISHED == state) {
if (FinalApplicationStatus.SUCCEEDED == dsStatus) {
LOG.info("Application has completed successfully. Breaking monitoring loop");
return true;
}
else {
LOG.info("Application did finished unsuccessfully."
+ " YarnState=" + state.toString() + ", DSFinalStatus=" + dsStatus.toString()
+ ". Breaking monitoring loop");
return false;
}
}
else if (YarnApplicationState.KILLED == state
|| YarnApplicationState.FAILED == state) {
LOG.info("Application did not finish."
+ " YarnState=" + state.toString() + ", DSFinalStatus=" + dsStatus.toString()
+ ". Breaking monitoring loop");
return false;
}
if (System.currentTimeMillis() > (clientStartTime + clientTimeout)) {
LOG.info("Reached client specified timeout for application. Killing application");
forceKillApplication(appId);
return false;
}
}
}
Example 19
| Source File: UnmanagedAMLauncher.java From big-c with Apache License 2.0 | 4 votes |
|
public boolean run() throws IOException, YarnException {
LOG.info("Starting Client");
// Connect to ResourceManager
rmClient.start();
try {
// Create launch context for app master
LOG.info("Setting up application submission context for ASM");
ApplicationSubmissionContext appContext = rmClient.createApplication()
.getApplicationSubmissionContext();
ApplicationId appId = appContext.getApplicationId();
// set the application name
appContext.setApplicationName(appName);
// Set the priority for the application master
Priority pri = Records.newRecord(Priority.class);
pri.setPriority(amPriority);
appContext.setPriority(pri);
// Set the queue to which this application is to be submitted in the RM
appContext.setQueue(amQueue);
// Set up the container launch context for the application master
ContainerLaunchContext amContainer = Records
.newRecord(ContainerLaunchContext.class);
appContext.setAMContainerSpec(amContainer);
// unmanaged AM
appContext.setUnmanagedAM(true);
LOG.info("Setting unmanaged AM");
// Submit the application to the applications manager
LOG.info("Submitting application to ASM");
rmClient.submitApplication(appContext);
ApplicationReport appReport =
monitorApplication(appId, EnumSet.of(YarnApplicationState.ACCEPTED,
YarnApplicationState.KILLED, YarnApplicationState.FAILED,
YarnApplicationState.FINISHED));
if (appReport.getYarnApplicationState() == YarnApplicationState.ACCEPTED) {
// Monitor the application attempt to wait for launch state
ApplicationAttemptReport attemptReport =
monitorCurrentAppAttempt(appId,
YarnApplicationAttemptState.LAUNCHED);
ApplicationAttemptId attemptId =
attemptReport.getApplicationAttemptId();
LOG.info("Launching AM with application attempt id " + attemptId);
// launch AM
launchAM(attemptId);
// Monitor the application for end state
appReport =
monitorApplication(appId, EnumSet.of(YarnApplicationState.KILLED,
YarnApplicationState.FAILED, YarnApplicationState.FINISHED));
}
YarnApplicationState appState = appReport.getYarnApplicationState();
FinalApplicationStatus appStatus = appReport.getFinalApplicationStatus();
LOG.info("App ended with state: " + appReport.getYarnApplicationState()
+ " and status: " + appStatus);
boolean success;
if (YarnApplicationState.FINISHED == appState
&& FinalApplicationStatus.SUCCEEDED == appStatus) {
LOG.info("Application has completed successfully.");
success = true;
} else {
LOG.info("Application did finished unsuccessfully." + " YarnState="
+ appState.toString() + ", FinalStatus=" + appStatus.toString());
success = false;
}
return success;
} finally {
rmClient.stop();
}
}
Example 20
| Source File: Client.java From big-c with Apache License 2.0 | 4 votes |
|
/**
* Monitor the submitted application for completion.
* Kill application if time expires.
* @param appId Application Id of application to be monitored
* @return true if application completed successfully
* @throws YarnException
* @throws IOException
*/
private boolean monitorApplication(ApplicationId appId)
throws YarnException, IOException {
while (true) {
// Check app status every 1 second.
try {
Thread.sleep(1000);
} catch (InterruptedException e) {
LOG.debug("Thread sleep in monitoring loop interrupted");
}
// Get application report for the appId we are interested in
ApplicationReport report = yarnClient.getApplicationReport(appId);
LOG.info("Got application report from ASM for"
+ ", appId=" + appId.getId()
+ ", clientToAMToken=" + report.getClientToAMToken()
+ ", appDiagnostics=" + report.getDiagnostics()
+ ", appMasterHost=" + report.getHost()
+ ", appQueue=" + report.getQueue()
+ ", appMasterRpcPort=" + report.getRpcPort()
+ ", appStartTime=" + report.getStartTime()
+ ", yarnAppState=" + report.getYarnApplicationState().toString()
+ ", distributedFinalState=" + report.getFinalApplicationStatus().toString()
+ ", appTrackingUrl=" + report.getTrackingUrl()
+ ", appUser=" + report.getUser());
YarnApplicationState state = report.getYarnApplicationState();
FinalApplicationStatus dsStatus = report.getFinalApplicationStatus();
if (YarnApplicationState.FINISHED == state) {
if (FinalApplicationStatus.SUCCEEDED == dsStatus) {
LOG.info("Application has completed successfully. Breaking monitoring loop");
return true;
}
else {
LOG.info("Application did finished unsuccessfully."
+ " YarnState=" + state.toString() + ", DSFinalStatus=" + dsStatus.toString()
+ ". Breaking monitoring loop");
return false;
}
}
else if (YarnApplicationState.KILLED == state
|| YarnApplicationState.FAILED == state) {
LOG.info("Application did not finish."
+ " YarnState=" + state.toString() + ", DSFinalStatus=" + dsStatus.toString()
+ ". Breaking monitoring loop");
return false;
}
if (System.currentTimeMillis() > (clientStartTime + clientTimeout)) {
LOG.info("Reached client specified timeout for application. Killing application");
forceKillApplication(appId);
return false;
}
}
}
