Java Code Examples for org.apache.hadoop.yarn.api.records.FinalApplicationStatus#UNDEFINED

private JSONObject getResource(StramAgent.StramUriSpec uriSpec, ApplicationReport appReport, WebServicesClient.WebServicesHandler handler)
{

  if (appReport == null) {
    throw new CliException("No application selected");
  }

  if (StringUtils.isEmpty(appReport.getTrackingUrl()) || appReport.getFinalApplicationStatus() != FinalApplicationStatus.UNDEFINED) {
    appReport = null;
    throw new CliException("Application terminated");
  }

  WebServicesClient wsClient = new WebServicesClient();
  try {
    return stramAgent.issueStramWebRequest(wsClient, appReport.getApplicationId().toString(), uriSpec, handler);
  } catch (Exception e) {
    // check the application status as above may have failed due application termination etc.
    if (appReport == currentApp) {
      currentApp = assertRunningApp(appReport);
    }
    throw new CliException("Failed to request web service for appid " + appReport.getApplicationId().toString(), e);
  }
}
 

protected FinalApplicationStatus convertDAGAppMasterStateToFinalYARNState(
    DAGAppMasterState dagAppMasterState) {
  switch (dagAppMasterState) {
    case NEW:
    case INITED:
    case RECOVERING:
    case IDLE:
    case RUNNING:
      return FinalApplicationStatus.UNDEFINED;
    case SUCCEEDED:
      return FinalApplicationStatus.SUCCEEDED;
    case FAILED:
      return FinalApplicationStatus.FAILED;
    case KILLED:
      return FinalApplicationStatus.KILLED;
    case ERROR:
      return FinalApplicationStatus.FAILED;
    default:
      return FinalApplicationStatus.UNDEFINED;
  }
}
 

/**
 * Converts a Flink application status enum to a YARN application status enum.
 * @param status The Flink application status.
 * @return The corresponding YARN application status.
 */
private FinalApplicationStatus getYarnStatus(ApplicationStatus status) {
	if (status == null) {
		return FinalApplicationStatus.UNDEFINED;
	}
	else {
		switch (status) {
			case SUCCEEDED:
				return FinalApplicationStatus.SUCCEEDED;
			case FAILED:
				return FinalApplicationStatus.FAILED;
			case CANCELED:
				return FinalApplicationStatus.KILLED;
			default:
				return FinalApplicationStatus.UNDEFINED;
		}
	}
}
 

private JSONObject getResource(StramAgent.StramUriSpec uriSpec, ApplicationReport appReport, WebServicesClient.WebServicesHandler handler)
{

  if (appReport == null) {
    throw new CliException("No application selected");
  }

  if (StringUtils.isEmpty(appReport.getTrackingUrl()) || appReport.getFinalApplicationStatus() != FinalApplicationStatus.UNDEFINED) {
    appReport = null;
    throw new CliException("Application terminated");
  }

  WebServicesClient wsClient = new WebServicesClient();
  try {
    return stramAgent.issueStramWebRequest(wsClient, appReport.getApplicationId().toString(), uriSpec, handler);
  } catch (Exception e) {
    // check the application status as above may have failed due application termination etc.
    if (appReport == currentApp) {
      currentApp = assertRunningApp(appReport);
    }
    throw new CliException("Failed to request web service for appid " + appReport.getApplicationId().toString(), e);
  }
}
 

@Override
public boolean isFinished()
{
  try {
    ApplicationReport appReport = yarnClient.getApplicationReport(appId);
    if (appReport != null) {
      if (appReport.getFinalApplicationStatus() == null
          || appReport.getFinalApplicationStatus() == FinalApplicationStatus.UNDEFINED) {
        return false;
      }
    }
    return true;
  } catch (YarnException | IOException e) {
    throw Throwables.propagate(e);
  }
}
 

private FinalApplicationStatus createFinalApplicationStatus(RMAppState state) {
  switch(state) {
  case NEW:
  case NEW_SAVING:
  case SUBMITTED:
  case ACCEPTED:
  case RUNNING:
  case FINAL_SAVING:
  case KILLING:
    return FinalApplicationStatus.UNDEFINED;    
  // finished without a proper final state is the same as failed  
  case FINISHING:
  case FINISHED:
  case FAILED:
    return FinalApplicationStatus.FAILED;
  case KILLED:
    return FinalApplicationStatus.KILLED;
  }
  throw new YarnRuntimeException("Unknown state passed!");
}
 

private FinalApplicationStatus createFinalApplicationStatus(RMAppState state) {
  switch(state) {
  case NEW:
  case NEW_SAVING:
  case SUBMITTED:
  case ACCEPTED:
  case RUNNING:
  case FINAL_SAVING:
  case KILLING:
    return FinalApplicationStatus.UNDEFINED;    
  // finished without a proper final state is the same as failed  
  case FINISHING:
  case FINISHED:
  case FAILED:
    return FinalApplicationStatus.FAILED;
  case KILLED:
    return FinalApplicationStatus.KILLED;
  }
  throw new YarnRuntimeException("Unknown state passed!");
}
 

/**
 * Converts a Flink application status enum to a YARN application status enum.
 * @param status The Flink application status.
 * @return The corresponding YARN application status.
 */
private FinalApplicationStatus getYarnStatus(ApplicationStatus status) {
	if (status == null) {
		return FinalApplicationStatus.UNDEFINED;
	}
	else {
		switch (status) {
			case SUCCEEDED:
				return FinalApplicationStatus.SUCCEEDED;
			case FAILED:
				return FinalApplicationStatus.FAILED;
			case CANCELED:
				return FinalApplicationStatus.KILLED;
			default:
				return FinalApplicationStatus.UNDEFINED;
		}
	}
}
 

/**
 * Converts a Flink application status enum to a YARN application status enum.
 * @param status The Flink application status.
 * @return The corresponding YARN application status.
 */
private FinalApplicationStatus getYarnStatus(ApplicationStatus status) {
	if (status == null) {
		return FinalApplicationStatus.UNDEFINED;
	}
	else {
		switch (status) {
			case SUCCEEDED:
				return FinalApplicationStatus.SUCCEEDED;
			case FAILED:
				return FinalApplicationStatus.FAILED;
			case CANCELED:
				return FinalApplicationStatus.KILLED;
			default:
				return FinalApplicationStatus.UNDEFINED;
		}
	}
}
 

@Override
public boolean isFinished()
{
  try {
    ApplicationReport appReport = yarnClient.getApplicationReport(appId);
    if (appReport != null) {
      if (appReport.getFinalApplicationStatus() == null
          || appReport.getFinalApplicationStatus() == FinalApplicationStatus.UNDEFINED) {
        return false;
      }
    }
    return true;
  } catch (YarnException | IOException e) {
    throw Throwables.propagate(e);
  }
}
 

public AppFinalStatus getFinalAppStatus() {
  FinalApplicationStatus finishState = FinalApplicationStatus.UNDEFINED;
  StringBuffer sb = new StringBuffer();
  if (dagAppMaster == null) {
    finishState = FinalApplicationStatus.UNDEFINED;
    sb.append("App not yet initialized");
  } else {
    DAGAppMasterState appMasterState = dagAppMaster.getState();
    if (appMasterState == DAGAppMasterState.SUCCEEDED) {
      finishState = FinalApplicationStatus.SUCCEEDED;
    } else if (appMasterState == DAGAppMasterState.KILLED
        || (appMasterState == DAGAppMasterState.RUNNING && isSignalled)) {
      finishState = FinalApplicationStatus.KILLED;
    } else if (appMasterState == DAGAppMasterState.FAILED
        || appMasterState == DAGAppMasterState.ERROR) {
      finishState = FinalApplicationStatus.FAILED;
    } else {
      finishState = FinalApplicationStatus.UNDEFINED;
    }
    finishState = hadoopShim.applyFinalApplicationStatusCorrection(finishState,
        dagAppMaster.isSession(), appMasterState == DAGAppMasterState.ERROR);
    List<String> diagnostics = dagAppMaster.getDiagnostics();
    if(diagnostics != null) {
      for (String s : diagnostics) {
        sb.append(s).append("\n");
      }
    }
  }
  if(LOG.isDebugEnabled()) {
    LOG.debug("Setting job diagnostics to " + sb.toString());
  }

  // if history url is set use the same, if historyUrl is set to "" then rm ui disables the
  // history url
  return new AppFinalStatus(finishState, sb.toString(), historyUrl);
}
 

public void stop() {
  if(stopped.get()) {
    return;
  }
  LOG.info("un-registering ApplicationMaster(QueryMaster):" + appAttemptId);
  stopped.set(true);

  try {
    FinalApplicationStatus status = FinalApplicationStatus.UNDEFINED;
    Query query = context.getQuery();
    if (query != null) {
      TajoProtos.QueryState state = query.getState();
      if (state == TajoProtos.QueryState.QUERY_SUCCEEDED) {
        status = FinalApplicationStatus.SUCCEEDED;
      } else if (state == TajoProtos.QueryState.QUERY_FAILED || state == TajoProtos.QueryState.QUERY_ERROR) {
        status = FinalApplicationStatus.FAILED;
      } else if (state == TajoProtos.QueryState.QUERY_ERROR) {
        status = FinalApplicationStatus.FAILED;
      }
    }
    unregisterApplicationMaster(status, "tajo query finished", null);
  } catch (Exception e) {
    LOG.error(e.getMessage(), e);
  }

  allocatorThread.interrupt();
  LOG.info("un-registered ApplicationMAster(QueryMaster) stopped:" + appAttemptId);

  super.stop();
}
 

@Override
public AppFinalStatus getFinalAppStatus() {
  FinalApplicationStatus finishState = FinalApplicationStatus.UNDEFINED;
  StringBuffer sb = new StringBuffer();
  if (dagAppMaster == null) {
    finishState = FinalApplicationStatus.UNDEFINED;
    sb.append("App not yet initialized");
  } else {
    DAGAppMasterState appMasterState = dagAppMaster.getState();
    if (appMasterState == DAGAppMasterState.SUCCEEDED) {
      finishState = FinalApplicationStatus.SUCCEEDED;
    } else if (appMasterState == DAGAppMasterState.KILLED
        || (appMasterState == DAGAppMasterState.RUNNING && isSignalled)) {
      finishState = FinalApplicationStatus.KILLED;
    } else if (appMasterState == DAGAppMasterState.FAILED
        || appMasterState == DAGAppMasterState.ERROR) {
      finishState = FinalApplicationStatus.FAILED;
    } else {
      finishState = FinalApplicationStatus.UNDEFINED;
    }
    List<String> diagnostics = dagAppMaster.getDiagnostics();
    if(diagnostics != null) {
      for (String s : diagnostics) {
        sb.append(s).append("\n");
      }
    }
  }
  if(LOG.isDebugEnabled()) {
    LOG.debug("Setting job diagnostics to " + sb.toString());
  }

  String historyUrl = "";
  /*String historyUrl = JobHistoryUtils.getHistoryUrl(getConfig(),
      appContext.getApplicationID());
  LOG.info("History url is " + historyUrl);*/

  return new AppFinalStatus(finishState, sb.toString(), historyUrl);
}
 

private String clairfyAppFinalStatus(FinalApplicationStatus status) {
  if (status == FinalApplicationStatus.UNDEFINED) {
    return "Application has not completed yet.";
  }
  return status.toString();
}
 

@Override
public FinalApplicationStatus getFinalApplicationStatus() {
  return FinalApplicationStatus.UNDEFINED;
}
 

@VisibleForTesting
protected void doUnregistration()
    throws YarnException, IOException, InterruptedException {
  FinalApplicationStatus finishState = FinalApplicationStatus.UNDEFINED;
  JobImpl jobImpl = (JobImpl)job;
  if (jobImpl.getInternalState() == JobStateInternal.SUCCEEDED) {
    finishState = FinalApplicationStatus.SUCCEEDED;
  } else if (jobImpl.getInternalState() == JobStateInternal.KILLED
      || (jobImpl.getInternalState() == JobStateInternal.RUNNING && isSignalled)) {
    finishState = FinalApplicationStatus.KILLED;
  } else if (jobImpl.getInternalState() == JobStateInternal.FAILED
      || jobImpl.getInternalState() == JobStateInternal.ERROR) {
    finishState = FinalApplicationStatus.FAILED;
  }
  StringBuffer sb = new StringBuffer();
  for (String s : job.getDiagnostics()) {
    sb.append(s).append("\n");
  }
  LOG.info("Setting job diagnostics to " + sb.toString());

  String historyUrl =
      MRWebAppUtil.getApplicationWebURLOnJHSWithScheme(getConfig(),
          context.getApplicationID());
  LOG.info("History url is " + historyUrl);
  FinishApplicationMasterRequest request =
      FinishApplicationMasterRequest.newInstance(finishState,
        sb.toString(), historyUrl);
  try {
    while (true) {
      FinishApplicationMasterResponse response =
          scheduler.finishApplicationMaster(request);
      if (response.getIsUnregistered()) {
        // When excepting ClientService, other services are already stopped,
        // it is safe to let clients know the final states. ClientService
        // should wait for some time so clients have enough time to know the
        // final states.
        RunningAppContext raContext = (RunningAppContext) context;
        raContext.markSuccessfulUnregistration();
        break;
      }
      LOG.info("Waiting for application to be successfully unregistered.");
      Thread.sleep(rmPollInterval);
    }
  } catch (ApplicationMasterNotRegisteredException e) {
    // RM might have restarted or failed over and so lost the fact that AM had
    // registered before.
    register();
    doUnregistration();
  }
}
 

private String clairfyAppFinalStatus(FinalApplicationStatus status) {
  if (status == FinalApplicationStatus.UNDEFINED) {
    return "Application has not completed yet.";
  }
  return status.toString();
}
 

@Override
public ClusterClient<ApplicationId> retrieve(ApplicationId applicationId) throws ClusterRetrieveException {

	try {
		// check if required Hadoop environment variables are set. If not, warn user
		if (System.getenv("HADOOP_CONF_DIR") == null &&
			System.getenv("YARN_CONF_DIR") == null) {
			LOG.warn("Neither the HADOOP_CONF_DIR nor the YARN_CONF_DIR environment variable is set." +
				"The Flink YARN Client needs one of these to be set to properly load the Hadoop " +
				"configuration for accessing YARN.");
		}

		final ApplicationReport appReport = yarnClient.getApplicationReport(applicationId);

		if (appReport.getFinalApplicationStatus() != FinalApplicationStatus.UNDEFINED) {
			// Flink cluster is not running anymore
			LOG.error("The application {} doesn't run anymore. It has previously completed with final status: {}",
				applicationId, appReport.getFinalApplicationStatus());
			throw new RuntimeException("The Yarn application " + applicationId + " doesn't run anymore.");
		}

		final String host = appReport.getHost();
		final int rpcPort = appReport.getRpcPort();

		LOG.info("Found application JobManager host name '{}' and port '{}' from supplied application id '{}'",
			host, rpcPort, applicationId);

		flinkConfiguration.setString(JobManagerOptions.ADDRESS, host);
		flinkConfiguration.setInteger(JobManagerOptions.PORT, rpcPort);

		flinkConfiguration.setString(RestOptions.ADDRESS, host);
		flinkConfiguration.setInteger(RestOptions.PORT, rpcPort);

		return createYarnClusterClient(
			this,
			-1, // we don't know the number of task managers of a started Flink cluster
			-1, // we don't know how many slots each task manager has for a started Flink cluster
			appReport,
			flinkConfiguration,
			false);
	} catch (Exception e) {
		throw new ClusterRetrieveException("Couldn't retrieve Yarn cluster", e);
	}
}
 

@Override
public FinalApplicationStatus getFinalApplicationStatus() {
  return FinalApplicationStatus.UNDEFINED;
}
 

@Override
public ClusterClient<ApplicationId> retrieve(ApplicationId applicationId) throws ClusterRetrieveException {

	try {
		// check if required Hadoop environment variables are set. If not, warn user
		if (System.getenv("HADOOP_CONF_DIR") == null &&
			System.getenv("YARN_CONF_DIR") == null) {
			LOG.warn("Neither the HADOOP_CONF_DIR nor the YARN_CONF_DIR environment variable is set." +
				"The Flink YARN Client needs one of these to be set to properly load the Hadoop " +
				"configuration for accessing YARN.");
		}

		final ApplicationReport appReport = yarnClient.getApplicationReport(applicationId);

		if (appReport.getFinalApplicationStatus() != FinalApplicationStatus.UNDEFINED) {
			// Flink cluster is not running anymore
			LOG.error("The application {} doesn't run anymore. It has previously completed with final status: {}",
				applicationId, appReport.getFinalApplicationStatus());
			throw new RuntimeException("The Yarn application " + applicationId + " doesn't run anymore.");
		}

		final String host = appReport.getHost();
		final int rpcPort = appReport.getRpcPort();

		LOG.info("Found application JobManager host name '{}' and port '{}' from supplied application id '{}'",
			host, rpcPort, applicationId);

		flinkConfiguration.setString(JobManagerOptions.ADDRESS, host);
		flinkConfiguration.setInteger(JobManagerOptions.PORT, rpcPort);

		flinkConfiguration.setString(RestOptions.ADDRESS, host);
		flinkConfiguration.setInteger(RestOptions.PORT, rpcPort);

		return createYarnClusterClient(
			this,
			-1, // we don't know the number of task managers of a started Flink cluster
			-1, // we don't know how many slots each task manager has for a started Flink cluster
			appReport,
			flinkConfiguration,
			false);
	} catch (Exception e) {
		throw new ClusterRetrieveException("Couldn't retrieve Yarn cluster", e);
	}
}