com.amazonaws.services.elasticmapreduce.model.DescribeStepRequest Java Examples

The following examples show how to use com.amazonaws.services.elasticmapreduce.model.DescribeStepRequest. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example #1

Source File: LambdaContainer.java From aws-big-data-blog with Apache License 2.0

6 votes

public void monitorEMRStep() throws Exception {
	List<String> stepIds = new ArrayList<String>();
	Connection conn  = new com.mysql.jdbc.Driver().connect(props.getProperty("url"), props);
	ResultSet openStepsRS = conn.createStatement().executeQuery(props.getProperty("sql.retrieveOpenSteps"));
	AmazonElasticMapReduceClient emr = new AmazonElasticMapReduceClient();
	DescribeStepRequest stepReq=new  DescribeStepRequest();
	PreparedStatement ps = conn.prepareStatement(props.getProperty("sql.updateStepStatus"));
	while(openStepsRS.next()){
		
		stepReq.setClusterId(openStepsRS.getString("cluster_id"));
		stepReq.setStepId(openStepsRS.getString("step_id"));
		String stepState = emr.describeStep(stepReq).getStep().getStatus().getState();
		
			if(stepState.equals(StepState.COMPLETED.toString())){
				ps.setString(1,StepState.COMPLETED.toString());
			}else if (stepState.equals(StepState.FAILED.toString())){
				ps.setString(1,StepState.FAILED.toString());					
			}
			ps.setString(2,openStepsRS.getString("job_config_id"));
			ps.addBatch();
	}
	
	ps.executeBatch();
	ps.close();
	conn.close();
}

Example #2

Source File: EmrDaoImpl.java From herd with Apache License 2.0

4 votes

@Override
public Step getClusterStep(String clusterId, String stepId, AwsParamsDto awsParamsDto)
{
    DescribeStepRequest describeStepRequest = new DescribeStepRequest().withClusterId(clusterId).withStepId(stepId);
    return emrOperations.describeStepRequest(getEmrClient(awsParamsDto), describeStepRequest).getStep();
}

Example #3

Source File: EmrClusterJob.java From datacollector with Apache License 2.0

4 votes

@Override
public Properties getJobStatus(Properties jobProps) throws IOException {
  EMRJobConfig emrJobConfig = new EMRJobConfig(jobProps);
  Utils.checkNotNull(emrJobConfig.getClusterId(), "EMR Cluster Id");
  String state;
  String message = null;
  DescribeStepResult res = getEmrClient(emrClusterConfig).describeStep(new DescribeStepRequest().withClusterId(
      emrJobConfig.getClusterId()).withStepId(emrJobConfig.getStepId()));
  StepStatus status = res.getStep().getStatus();
  ApplicationId appId = null;
  LOG.debug(Utils.format("Status of step: {} is {}", emrJobConfig.getStepId(), status.getState()));
  if ("PENDING".equals(status.getState())) {
    state = status.getState();
    if (status.getStateChangeReason() != null) {
      message = status.getStateChangeReason().getMessage();
    }
  } else if (!"COMPLETED".equals(status.getState()) && !"RUNNING".equals(status.getState())) {
    state = status.getState();
    if (status.getFailureDetails() != null) {
      message = status.getFailureDetails().getReason();
    }
  } else {
    YarnClient yarnClient = getYarnClient(emrJobConfig.getClusterId(), emrClusterConfig);
    ApplicationReport report = null;
    try {
      for (ApplicationReport applicationReport : yarnClient.getApplications()) {
        if (applicationReport.getName().contains(emrJobConfig.getUniquePrefix())) {
          appId = applicationReport.getApplicationId();
          break;
        }
      }
      if (appId != null) {
        report = yarnClient.getApplicationReport(appId);
      }
    } catch (YarnException ex) {
      throw new IOException("Failed to fetch yarn app report " + ex);
    }
    if (report != null) {
      YarnApplicationState yarnState = report.getYarnApplicationState();
      FinalApplicationStatus finalApplicationStatus = report.getFinalApplicationStatus();
      LOG.info(Utils.format("Application state for app id: {} is {} ", appId, yarnState));
      state = yarnState.name();
      if (YarnApplicationState.FINISHED == yarnState) {
        // override with final application state if yarnState is FINISHED
        state = finalApplicationStatus.name();
      }
      message = report.getDiagnostics();
    } else {
      state = "STARTING"; // a situation where step was in RUNNING but yarn application not yet created.
      message = "Yarn application not yet created";
    }
  }
  EmrState emrJobState = new EmrState();
  emrJobState.setState(state);
  emrJobState.setMessage(message);
  emrJobState.setAppId(appId != null ? appId.toString() : null);
  return emrJobState.toProperties();
}