Java Code Examples for org.apache.hadoop.mapreduce.v2.api.protocolrecords.GetJobReportRequest#setJobId()

The following examples show how to use org.apache.hadoop.mapreduce.v2.api.protocolrecords.GetJobReportRequest#setJobId() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: ClientServiceDelegate.java    From hadoop with Apache License 2.0 6 votes vote down vote up 
public JobStatus getJobStatus(JobID oldJobID) throws IOException {
  org.apache.hadoop.mapreduce.v2.api.records.JobId jobId =
    TypeConverter.toYarn(oldJobID);
  GetJobReportRequest request =
      recordFactory.newRecordInstance(GetJobReportRequest.class);
  request.setJobId(jobId);
  JobReport report = ((GetJobReportResponse) invoke("getJobReport",
      GetJobReportRequest.class, request)).getJobReport();
  JobStatus jobStatus = null;
  if (report != null) {
    if (StringUtils.isEmpty(report.getJobFile())) {
      String jobFile = MRApps.getJobFile(conf, report.getUser(), oldJobID);
      report.setJobFile(jobFile);
    }
    String historyTrackingUrl = report.getTrackingUrl();
    String url = StringUtils.isNotEmpty(historyTrackingUrl)
        ? historyTrackingUrl : trackingUrl;
    jobStatus = TypeConverter.fromYarn(report, url);
  }
  return jobStatus;
}
Example 2
Source File: ClientServiceDelegate.java    From big-c with Apache License 2.0 6 votes vote down vote up 
public JobStatus getJobStatus(JobID oldJobID) throws IOException {
  org.apache.hadoop.mapreduce.v2.api.records.JobId jobId =
    TypeConverter.toYarn(oldJobID);
  GetJobReportRequest request =
      recordFactory.newRecordInstance(GetJobReportRequest.class);
  request.setJobId(jobId);
  JobReport report = ((GetJobReportResponse) invoke("getJobReport",
      GetJobReportRequest.class, request)).getJobReport();
  JobStatus jobStatus = null;
  if (report != null) {
    if (StringUtils.isEmpty(report.getJobFile())) {
      String jobFile = MRApps.getJobFile(conf, report.getUser(), oldJobID);
      report.setJobFile(jobFile);
    }
    String historyTrackingUrl = report.getTrackingUrl();
    String url = StringUtils.isNotEmpty(historyTrackingUrl)
        ? historyTrackingUrl : trackingUrl;
    jobStatus = TypeConverter.fromYarn(report, url);
  }
  return jobStatus;
}
Example 3
Source File: MRCommunicator.java    From jumbune with GNU Lesser General Public License v3.0 5 votes vote down vote up 
/**
 * Given the required details (application id and suffix id) for JobId it gives the JobReport
 * @param appid, the Application Id instance
 * @param id, the suffix id
 * @return the Job Report
 * @throws IOException
 */
public JobReport getJobReport(ApplicationId appId, int id) throws IOException{
	JobId jobId = YarnCommunicatorUtil.getJobId(appId, (int)1);
	GetJobReportRequestProto proto = GetJobReportRequestProto.getDefaultInstance();
	GetJobReportRequest request = new GetJobReportRequestPBImpl(proto);
	request.setJobId(jobId);
	GetJobReportResponse jobReportResponse = proxy.getJobReport(request);
	return jobReportResponse.getJobReport();
}
Example 4
Source File: MRCommunicator.java    From jumbune with GNU Lesser General Public License v3.0 5 votes vote down vote up 
public JobReport getJobReport(JobId jobId) throws IOException {
	GetJobReportRequestProto proto = GetJobReportRequestProto.getDefaultInstance();
	GetJobReportRequest request = new GetJobReportRequestPBImpl(proto);
	request.setJobId(jobId);
	GetJobReportResponse jobReportResponse = proxy.getJobReport(request);
	return jobReportResponse.getJobReport();
}
Example 5
Source File: ClientServiceDelegate.java    From hadoop with Apache License 2.0 4 votes vote down vote up 
public LogParams getLogFilePath(JobID oldJobID, TaskAttemptID oldTaskAttemptID)
    throws IOException {
  org.apache.hadoop.mapreduce.v2.api.records.JobId jobId =
      TypeConverter.toYarn(oldJobID);
  GetJobReportRequest request =
      recordFactory.newRecordInstance(GetJobReportRequest.class);
  request.setJobId(jobId);

  JobReport report =
      ((GetJobReportResponse) invoke("getJobReport",
          GetJobReportRequest.class, request)).getJobReport();
  if (EnumSet.of(JobState.SUCCEEDED, JobState.FAILED, JobState.KILLED,
      JobState.ERROR).contains(report.getJobState())) {
    if (oldTaskAttemptID != null) {
      GetTaskAttemptReportRequest taRequest =
          recordFactory.newRecordInstance(GetTaskAttemptReportRequest.class);
      taRequest.setTaskAttemptId(TypeConverter.toYarn(oldTaskAttemptID));
      TaskAttemptReport taReport =
          ((GetTaskAttemptReportResponse) invoke("getTaskAttemptReport",
              GetTaskAttemptReportRequest.class, taRequest))
              .getTaskAttemptReport();
      if (taReport.getContainerId() == null
          || taReport.getNodeManagerHost() == null) {
        throw new IOException("Unable to get log information for task: "
            + oldTaskAttemptID);
      }
      return new LogParams(
          taReport.getContainerId().toString(),
          taReport.getContainerId().getApplicationAttemptId()
              .getApplicationId().toString(),
          NodeId.newInstance(taReport.getNodeManagerHost(),
              taReport.getNodeManagerPort()).toString(), report.getUser());
    } else {
      if (report.getAMInfos() == null || report.getAMInfos().size() == 0) {
        throw new IOException("Unable to get log information for job: "
            + oldJobID);
      }
      AMInfo amInfo = report.getAMInfos().get(report.getAMInfos().size() - 1);
      return new LogParams(
          amInfo.getContainerId().toString(),
          amInfo.getAppAttemptId().getApplicationId().toString(),
          NodeId.newInstance(amInfo.getNodeManagerHost(),
              amInfo.getNodeManagerPort()).toString(), report.getUser());
    }
  } else {
    throw new IOException("Cannot get log path for a in-progress job");
  }
}
Example 6
Source File: TestClientServiceDelegate.java    From hadoop with Apache License 2.0 4 votes vote down vote up 
private GetJobReportRequest getJobReportRequest() {
  GetJobReportRequest request = Records.newRecord(GetJobReportRequest.class);
  request.setJobId(jobId);
  return request;
}
Example 7
Source File: TestMRJobsWithHistoryService.java    From hadoop with Apache License 2.0 4 votes vote down vote up 
@Test (timeout = 90000)
public void testJobHistoryData() throws IOException, InterruptedException,
    AvroRemoteException, ClassNotFoundException {
  if (!(new File(MiniMRYarnCluster.APPJAR)).exists()) {
    LOG.info("MRAppJar " + MiniMRYarnCluster.APPJAR
        + " not found. Not running test.");
    return;
  }


  
  SleepJob sleepJob = new SleepJob();
  sleepJob.setConf(mrCluster.getConfig());
  // Job with 3 maps and 2 reduces
  Job job = sleepJob.createJob(3, 2, 1000, 1, 500, 1);
  job.setJarByClass(SleepJob.class);
  job.addFileToClassPath(APP_JAR); // The AppMaster jar itself.
  job.waitForCompletion(true);
  Counters counterMR = job.getCounters();
  JobId jobId = TypeConverter.toYarn(job.getJobID());
  ApplicationId appID = jobId.getAppId();
  int pollElapsed = 0;
  while (true) {
    Thread.sleep(1000);
    pollElapsed += 1000;

    if (TERMINAL_RM_APP_STATES.contains(
        mrCluster.getResourceManager().getRMContext().getRMApps().get(appID)
        .getState())) {
      break;
    }

    if (pollElapsed >= 60000) {
      LOG.warn("application did not reach terminal state within 60 seconds");
      break;
    }
  }
  Assert.assertEquals(RMAppState.FINISHED, mrCluster.getResourceManager()
    .getRMContext().getRMApps().get(appID).getState());
  Counters counterHS = job.getCounters();
  //TODO the Assert below worked. need to check
  //Should we compare each field or convert to V2 counter and compare
  LOG.info("CounterHS " + counterHS);
  LOG.info("CounterMR " + counterMR);
  Assert.assertEquals(counterHS, counterMR);
  
  HSClientProtocol historyClient = instantiateHistoryProxy();
  GetJobReportRequest gjReq = Records.newRecord(GetJobReportRequest.class);
  gjReq.setJobId(jobId);
  JobReport jobReport = historyClient.getJobReport(gjReq).getJobReport();
  verifyJobReport(jobReport, jobId);
}
Example 8
Source File: ClientServiceDelegate.java    From big-c with Apache License 2.0 4 votes vote down vote up 
public LogParams getLogFilePath(JobID oldJobID, TaskAttemptID oldTaskAttemptID)
    throws IOException {
  org.apache.hadoop.mapreduce.v2.api.records.JobId jobId =
      TypeConverter.toYarn(oldJobID);
  GetJobReportRequest request =
      recordFactory.newRecordInstance(GetJobReportRequest.class);
  request.setJobId(jobId);

  JobReport report =
      ((GetJobReportResponse) invoke("getJobReport",
          GetJobReportRequest.class, request)).getJobReport();
  if (EnumSet.of(JobState.SUCCEEDED, JobState.FAILED, JobState.KILLED,
      JobState.ERROR).contains(report.getJobState())) {
    if (oldTaskAttemptID != null) {
      GetTaskAttemptReportRequest taRequest =
          recordFactory.newRecordInstance(GetTaskAttemptReportRequest.class);
      taRequest.setTaskAttemptId(TypeConverter.toYarn(oldTaskAttemptID));
      TaskAttemptReport taReport =
          ((GetTaskAttemptReportResponse) invoke("getTaskAttemptReport",
              GetTaskAttemptReportRequest.class, taRequest))
              .getTaskAttemptReport();
      if (taReport.getContainerId() == null
          || taReport.getNodeManagerHost() == null) {
        throw new IOException("Unable to get log information for task: "
            + oldTaskAttemptID);
      }
      return new LogParams(
          taReport.getContainerId().toString(),
          taReport.getContainerId().getApplicationAttemptId()
              .getApplicationId().toString(),
          NodeId.newInstance(taReport.getNodeManagerHost(),
              taReport.getNodeManagerPort()).toString(), report.getUser());
    } else {
      if (report.getAMInfos() == null || report.getAMInfos().size() == 0) {
        throw new IOException("Unable to get log information for job: "
            + oldJobID);
      }
      AMInfo amInfo = report.getAMInfos().get(report.getAMInfos().size() - 1);
      return new LogParams(
          amInfo.getContainerId().toString(),
          amInfo.getAppAttemptId().getApplicationId().toString(),
          NodeId.newInstance(amInfo.getNodeManagerHost(),
              amInfo.getNodeManagerPort()).toString(), report.getUser());
    }
  } else {
    throw new IOException("Cannot get log path for a in-progress job");
  }
}
Example 9
Source File: TestClientServiceDelegate.java    From big-c with Apache License 2.0 4 votes vote down vote up 
private GetJobReportRequest getJobReportRequest() {
  GetJobReportRequest request = Records.newRecord(GetJobReportRequest.class);
  request.setJobId(jobId);
  return request;
}
Example 10
Source File: TestMRJobsWithHistoryService.java    From big-c with Apache License 2.0 4 votes vote down vote up 
@Test (timeout = 90000)
public void testJobHistoryData() throws IOException, InterruptedException,
    AvroRemoteException, ClassNotFoundException {
  if (!(new File(MiniMRYarnCluster.APPJAR)).exists()) {
    LOG.info("MRAppJar " + MiniMRYarnCluster.APPJAR
        + " not found. Not running test.");
    return;
  }


  
  SleepJob sleepJob = new SleepJob();
  sleepJob.setConf(mrCluster.getConfig());
  // Job with 3 maps and 2 reduces
  Job job = sleepJob.createJob(3, 2, 1000, 1, 500, 1);
  job.setJarByClass(SleepJob.class);
  job.addFileToClassPath(APP_JAR); // The AppMaster jar itself.
  job.waitForCompletion(true);
  Counters counterMR = job.getCounters();
  JobId jobId = TypeConverter.toYarn(job.getJobID());
  ApplicationId appID = jobId.getAppId();
  int pollElapsed = 0;
  while (true) {
    Thread.sleep(1000);
    pollElapsed += 1000;

    if (TERMINAL_RM_APP_STATES.contains(
        mrCluster.getResourceManager().getRMContext().getRMApps().get(appID)
        .getState())) {
      break;
    }

    if (pollElapsed >= 60000) {
      LOG.warn("application did not reach terminal state within 60 seconds");
      break;
    }
  }
  Assert.assertEquals(RMAppState.FINISHED, mrCluster.getResourceManager()
    .getRMContext().getRMApps().get(appID).getState());
  Counters counterHS = job.getCounters();
  //TODO the Assert below worked. need to check
  //Should we compare each field or convert to V2 counter and compare
  LOG.info("CounterHS " + counterHS);
  LOG.info("CounterMR " + counterMR);
  Assert.assertEquals(counterHS, counterMR);
  
  HSClientProtocol historyClient = instantiateHistoryProxy();
  GetJobReportRequest gjReq = Records.newRecord(GetJobReportRequest.class);
  gjReq.setJobId(jobId);
  JobReport jobReport = historyClient.getJobReport(gjReq).getJobReport();
  verifyJobReport(jobReport, jobId);
}