org.apache.hadoop.mapreduce.v2.api.records.JobReport Java Examples
The following examples show how to use
org.apache.hadoop.mapreduce.v2.api.records.JobReport.
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
| Source File: MRApp.java From big-c with Apache License 2.0 | 6 votes |
|
public void waitForState(Job job, JobState finalState) throws Exception {
int timeoutSecs = 0;
JobReport report = job.getReport();
while (!finalState.equals(report.getJobState()) &&
timeoutSecs++ < 20) {
System.out.println("Job State is : " + report.getJobState() +
" Waiting for state : " + finalState +
" map progress : " + report.getMapProgress() +
" reduce progress : " + report.getReduceProgress());
report = job.getReport();
Thread.sleep(500);
}
System.out.println("Job State is : " + report.getJobState());
Assert.assertEquals("Job state is not correct (timedout)", finalState,
job.getState());
}
Example #2
| Source File: TestJobHistoryEntities.java From hadoop with Apache License 2.0 | 6 votes |
|
@Test (timeout=100000)
public void testCompletedJob() throws Exception {
HistoryFileInfo info = mock(HistoryFileInfo.class);
when(info.getConfFile()).thenReturn(fullConfPath);
//Re-initialize to verify the delayed load.
completedJob =
new CompletedJob(conf, jobId, fullHistoryPath, loadTasks, "user",
info, jobAclsManager);
//Verify tasks loaded based on loadTask parameter.
assertEquals(loadTasks, completedJob.tasksLoaded.get());
assertEquals(1, completedJob.getAMInfos().size());
assertEquals(10, completedJob.getCompletedMaps());
assertEquals(1, completedJob.getCompletedReduces());
assertEquals(12, completedJob.getTasks().size());
//Verify tasks loaded at this point.
assertEquals(true, completedJob.tasksLoaded.get());
assertEquals(10, completedJob.getTasks(TaskType.MAP).size());
assertEquals(2, completedJob.getTasks(TaskType.REDUCE).size());
assertEquals("user", completedJob.getUserName());
assertEquals(JobState.SUCCEEDED, completedJob.getState());
JobReport jobReport = completedJob.getReport();
assertEquals("user", jobReport.getUser());
assertEquals(JobState.SUCCEEDED, jobReport.getJobState());
}
Example #3
| Source File: VerifyJobsUtils.java From big-c with Apache License 2.0 | 6 votes |
|
public static void verifyHsJobGeneric(Job job, String id, String user,
String name, String state, String queue, long startTime, long finishTime,
int mapsTotal, int mapsCompleted, int reducesTotal, int reducesCompleted) {
JobReport report = job.getReport();
WebServicesTestUtils.checkStringMatch("id", MRApps.toString(job.getID()),
id);
WebServicesTestUtils.checkStringMatch("user", job.getUserName().toString(),
user);
WebServicesTestUtils.checkStringMatch("name", job.getName(), name);
WebServicesTestUtils.checkStringMatch("state", job.getState().toString(),
state);
WebServicesTestUtils.checkStringMatch("queue", job.getQueueName(), queue);
assertEquals("startTime incorrect", report.getStartTime(), startTime);
assertEquals("finishTime incorrect", report.getFinishTime(), finishTime);
assertEquals("mapsTotal incorrect", job.getTotalMaps(), mapsTotal);
assertEquals("mapsCompleted incorrect", job.getCompletedMaps(),
mapsCompleted);
assertEquals("reducesTotal incorrect", job.getTotalReduces(), reducesTotal);
assertEquals("reducesCompleted incorrect", job.getCompletedReduces(),
reducesCompleted);
}
Example #4
| Source File: TestBlocks.java From big-c with Apache License 2.0 | 6 votes |
|
private Job getJob() {
Job job = mock(Job.class);
JobId jobId = new JobIdPBImpl();
ApplicationId appId = ApplicationIdPBImpl.newInstance(System.currentTimeMillis(),4);
jobId.setAppId(appId);
jobId.setId(1);
when(job.getID()).thenReturn(jobId);
JobReport report = mock(JobReport.class);
when(report.getStartTime()).thenReturn(100010L);
when(report.getFinishTime()).thenReturn(100015L);
when(job.getReport()).thenReturn(report);
when(job.getName()).thenReturn("JobName");
when(job.getUserName()).thenReturn("UserName");
when(job.getQueueName()).thenReturn("QueueName");
when(job.getState()).thenReturn(JobState.SUCCEEDED);
when(job.getTotalMaps()).thenReturn(3);
when(job.getCompletedMaps()).thenReturn(2);
when(job.getTotalReduces()).thenReturn(2);
when(job.getCompletedReduces()).thenReturn(1);
when(job.getCompletedReduces()).thenReturn(1);
return job;
}
Example #5
| Source File: MockHistoryJobs.java From big-c with Apache License 2.0 | 6 votes |
|
private static JobsPair split(Map<JobId, Job> mocked) throws IOException {
JobsPair ret = new JobsPair();
ret.full = Maps.newHashMap();
ret.partial = Maps.newHashMap();
for(Map.Entry<JobId, Job> entry: mocked.entrySet()) {
JobId id = entry.getKey();
Job j = entry.getValue();
MockCompletedJob mockJob = new MockCompletedJob(j);
// use MockCompletedJob to set everything below to make sure
// consistent with what history server would do
ret.full.put(id, mockJob);
JobReport report = mockJob.getReport();
JobIndexInfo info = new JobIndexInfo(report.getStartTime(),
report.getFinishTime(), mockJob.getUserName(), mockJob.getName(), id,
mockJob.getCompletedMaps(), mockJob.getCompletedReduces(),
String.valueOf(mockJob.getState()));
info.setJobStartTime(report.getStartTime());
info.setQueueName(mockJob.getQueueName());
ret.partial.put(id, new PartialJob(info, id));
}
return ret;
}
Example #6
| Source File: MRApp.java From hadoop with Apache License 2.0 | 6 votes |
|
public void waitForState(Job job, JobState finalState) throws Exception {
int timeoutSecs = 0;
JobReport report = job.getReport();
while (!finalState.equals(report.getJobState()) &&
timeoutSecs++ < 20) {
System.out.println("Job State is : " + report.getJobState() +
" Waiting for state : " + finalState +
" map progress : " + report.getMapProgress() +
" reduce progress : " + report.getReduceProgress());
report = job.getReport();
Thread.sleep(500);
}
System.out.println("Job State is : " + report.getJobState());
Assert.assertEquals("Job state is not correct (timedout)", finalState,
job.getState());
}
Example #7
| Source File: TestJobHistoryEntities.java From big-c with Apache License 2.0 | 6 votes |
|
@Test (timeout=100000)
public void testCompletedJob() throws Exception {
HistoryFileInfo info = mock(HistoryFileInfo.class);
when(info.getConfFile()).thenReturn(fullConfPath);
//Re-initialize to verify the delayed load.
completedJob =
new CompletedJob(conf, jobId, fullHistoryPath, loadTasks, "user",
info, jobAclsManager);
//Verify tasks loaded based on loadTask parameter.
assertEquals(loadTasks, completedJob.tasksLoaded.get());
assertEquals(1, completedJob.getAMInfos().size());
assertEquals(10, completedJob.getCompletedMaps());
assertEquals(1, completedJob.getCompletedReduces());
assertEquals(12, completedJob.getTasks().size());
//Verify tasks loaded at this point.
assertEquals(true, completedJob.tasksLoaded.get());
assertEquals(10, completedJob.getTasks(TaskType.MAP).size());
assertEquals(2, completedJob.getTasks(TaskType.REDUCE).size());
assertEquals("user", completedJob.getUserName());
assertEquals(JobState.SUCCEEDED, completedJob.getState());
JobReport jobReport = completedJob.getReport();
assertEquals("user", jobReport.getUser());
assertEquals(JobState.SUCCEEDED, jobReport.getJobState());
}
Example #8
| Source File: MRApp.java From hadoop with Apache License 2.0 | 6 votes |
|
public void verifyCompleted() {
for (Job job : getContext().getAllJobs().values()) {
JobReport jobReport = job.getReport();
System.out.println("Job start time :" + jobReport.getStartTime());
System.out.println("Job finish time :" + jobReport.getFinishTime());
Assert.assertTrue("Job start time is not less than finish time",
jobReport.getStartTime() <= jobReport.getFinishTime());
Assert.assertTrue("Job finish time is in future",
jobReport.getFinishTime() <= System.currentTimeMillis());
for (Task task : job.getTasks().values()) {
TaskReport taskReport = task.getReport();
System.out.println("Task start time : " + taskReport.getStartTime());
System.out.println("Task finish time : " + taskReport.getFinishTime());
Assert.assertTrue("Task start time is not less than finish time",
taskReport.getStartTime() <= taskReport.getFinishTime());
for (TaskAttempt attempt : task.getAttempts().values()) {
TaskAttemptReport attemptReport = attempt.getReport();
Assert.assertTrue("Attempt start time is not less than finish time",
attemptReport.getStartTime() <= attemptReport.getFinishTime());
}
}
}
}
Example #9
| Source File: MockHistoryJobs.java From hadoop with Apache License 2.0 | 6 votes |
|
private static JobsPair split(Map<JobId, Job> mocked) throws IOException {
JobsPair ret = new JobsPair();
ret.full = Maps.newHashMap();
ret.partial = Maps.newHashMap();
for(Map.Entry<JobId, Job> entry: mocked.entrySet()) {
JobId id = entry.getKey();
Job j = entry.getValue();
MockCompletedJob mockJob = new MockCompletedJob(j);
// use MockCompletedJob to set everything below to make sure
// consistent with what history server would do
ret.full.put(id, mockJob);
JobReport report = mockJob.getReport();
JobIndexInfo info = new JobIndexInfo(report.getStartTime(),
report.getFinishTime(), mockJob.getUserName(), mockJob.getName(), id,
mockJob.getCompletedMaps(), mockJob.getCompletedReduces(),
String.valueOf(mockJob.getState()));
info.setJobStartTime(report.getStartTime());
info.setQueueName(mockJob.getQueueName());
ret.partial.put(id, new PartialJob(info, id));
}
return ret;
}
Example #10
| Source File: NotRunningJob.java From hadoop with Apache License 2.0 | 6 votes |
|
@Override
public GetJobReportResponse getJobReport(GetJobReportRequest request)
throws IOException {
JobReport jobReport =
recordFactory.newRecordInstance(JobReport.class);
jobReport.setJobId(request.getJobId());
jobReport.setJobState(jobState);
jobReport.setUser(applicationReport.getUser());
jobReport.setStartTime(applicationReport.getStartTime());
jobReport.setDiagnostics(applicationReport.getDiagnostics());
jobReport.setJobName(applicationReport.getName());
jobReport.setTrackingUrl(applicationReport.getTrackingUrl());
jobReport.setFinishTime(applicationReport.getFinishTime());
GetJobReportResponse resp =
recordFactory.newRecordInstance(GetJobReportResponse.class);
resp.setJobReport(jobReport);
return resp;
}
Example #11
| Source File: TestTypeConverter.java From big-c with Apache License 2.0 | 6 votes |
|
@Test
public void testFromYarnJobReport() throws Exception {
int jobStartTime = 612354;
int jobFinishTime = 612355;
JobState state = JobState.RUNNING;
JobId jobId = Records.newRecord(JobId.class);
JobReport jobReport = Records.newRecord(JobReport.class);
ApplicationId applicationId = ApplicationId.newInstance(0, 0);
jobId.setAppId(applicationId);
jobId.setId(0);
jobReport.setJobId(jobId);
jobReport.setJobState(state);
jobReport.setStartTime(jobStartTime);
jobReport.setFinishTime(jobFinishTime);
jobReport.setUser("TestTypeConverter-user");
JobStatus jobStatus = TypeConverter.fromYarn(jobReport, "dummy-jobfile");
Assert.assertEquals(jobStartTime, jobStatus.getStartTime());
Assert.assertEquals(jobFinishTime, jobStatus.getFinishTime());
Assert.assertEquals(state.toString(), jobStatus.getState().toString());
}
Example #12
| Source File: ClientServiceDelegate.java From hadoop with Apache License 2.0 | 6 votes |
|
public JobStatus getJobStatus(JobID oldJobID) throws IOException {
org.apache.hadoop.mapreduce.v2.api.records.JobId jobId =
TypeConverter.toYarn(oldJobID);
GetJobReportRequest request =
recordFactory.newRecordInstance(GetJobReportRequest.class);
request.setJobId(jobId);
JobReport report = ((GetJobReportResponse) invoke("getJobReport",
GetJobReportRequest.class, request)).getJobReport();
JobStatus jobStatus = null;
if (report != null) {
if (StringUtils.isEmpty(report.getJobFile())) {
String jobFile = MRApps.getJobFile(conf, report.getUser(), oldJobID);
report.setJobFile(jobFile);
}
String historyTrackingUrl = report.getTrackingUrl();
String url = StringUtils.isNotEmpty(historyTrackingUrl)
? historyTrackingUrl : trackingUrl;
jobStatus = TypeConverter.fromYarn(report, url);
}
return jobStatus;
}
Example #13
| Source File: MRBuilderUtils.java From big-c with Apache License 2.0 | 6 votes |
|
public static JobReport newJobReport(JobId jobId, String jobName,
String userName, JobState state, long submitTime, long startTime, long finishTime,
float setupProgress, float mapProgress, float reduceProgress,
float cleanupProgress, String jobFile, List<AMInfo> amInfos,
boolean isUber, String diagnostics) {
JobReport report = Records.newRecord(JobReport.class);
report.setJobId(jobId);
report.setJobName(jobName);
report.setUser(userName);
report.setJobState(state);
report.setSubmitTime(submitTime);
report.setStartTime(startTime);
report.setFinishTime(finishTime);
report.setSetupProgress(setupProgress);
report.setCleanupProgress(cleanupProgress);
report.setMapProgress(mapProgress);
report.setReduceProgress(reduceProgress);
report.setJobFile(jobFile);
report.setAMInfos(amInfos);
report.setIsUber(isUber);
report.setDiagnostics(diagnostics);
return report;
}
Example #14
| Source File: TestClientRedirect.java From hadoop with Apache License 2.0 | 6 votes |
|
@Override
public GetJobReportResponse getJobReport(GetJobReportRequest request)
throws IOException {
amContact = true;
JobReport jobReport = recordFactory.newRecordInstance(JobReport.class);
jobReport.setJobId(request.getJobId());
jobReport.setJobState(JobState.RUNNING);
jobReport.setJobName("TestClientRedirect-jobname");
jobReport.setUser("TestClientRedirect-user");
jobReport.setStartTime(0L);
jobReport.setFinishTime(1L);
GetJobReportResponse response = recordFactory
.newRecordInstance(GetJobReportResponse.class);
response.setJobReport(jobReport);
return response;
}
Example #15
| Source File: TestBlocks.java From hadoop with Apache License 2.0 | 6 votes |
|
private Job getJob() {
Job job = mock(Job.class);
JobId jobId = new JobIdPBImpl();
ApplicationId appId = ApplicationIdPBImpl.newInstance(System.currentTimeMillis(),4);
jobId.setAppId(appId);
jobId.setId(1);
when(job.getID()).thenReturn(jobId);
JobReport report = mock(JobReport.class);
when(report.getStartTime()).thenReturn(100010L);
when(report.getFinishTime()).thenReturn(100015L);
when(job.getReport()).thenReturn(report);
when(job.getName()).thenReturn("JobName");
when(job.getUserName()).thenReturn("UserName");
when(job.getQueueName()).thenReturn("QueueName");
when(job.getState()).thenReturn(JobState.SUCCEEDED);
when(job.getTotalMaps()).thenReturn(3);
when(job.getCompletedMaps()).thenReturn(2);
when(job.getTotalReduces()).thenReturn(2);
when(job.getCompletedReduces()).thenReturn(1);
when(job.getCompletedReduces()).thenReturn(1);
return job;
}
Example #16
| Source File: VerifyJobsUtils.java From hadoop with Apache License 2.0 | 6 votes |
|
public static void verifyHsJobGeneric(Job job, String id, String user,
String name, String state, String queue, long startTime, long finishTime,
int mapsTotal, int mapsCompleted, int reducesTotal, int reducesCompleted) {
JobReport report = job.getReport();
WebServicesTestUtils.checkStringMatch("id", MRApps.toString(job.getID()),
id);
WebServicesTestUtils.checkStringMatch("user", job.getUserName().toString(),
user);
WebServicesTestUtils.checkStringMatch("name", job.getName(), name);
WebServicesTestUtils.checkStringMatch("state", job.getState().toString(),
state);
WebServicesTestUtils.checkStringMatch("queue", job.getQueueName(), queue);
assertEquals("startTime incorrect", report.getStartTime(), startTime);
assertEquals("finishTime incorrect", report.getFinishTime(), finishTime);
assertEquals("mapsTotal incorrect", job.getTotalMaps(), mapsTotal);
assertEquals("mapsCompleted incorrect", job.getCompletedMaps(),
mapsCompleted);
assertEquals("reducesTotal incorrect", job.getTotalReduces(), reducesTotal);
assertEquals("reducesCompleted incorrect", job.getCompletedReduces(),
reducesCompleted);
}
Example #17
| Source File: TestClientRedirect.java From big-c with Apache License 2.0 | 6 votes |
|
@Override
public GetJobReportResponse getJobReport(GetJobReportRequest request)
throws IOException {
amContact = true;
JobReport jobReport = recordFactory.newRecordInstance(JobReport.class);
jobReport.setJobId(request.getJobId());
jobReport.setJobState(JobState.RUNNING);
jobReport.setJobName("TestClientRedirect-jobname");
jobReport.setUser("TestClientRedirect-user");
jobReport.setStartTime(0L);
jobReport.setFinishTime(1L);
GetJobReportResponse response = recordFactory
.newRecordInstance(GetJobReportResponse.class);
response.setJobReport(jobReport);
return response;
}
Example #18
| Source File: ClientServiceDelegate.java From big-c with Apache License 2.0 | 6 votes |
|
public JobStatus getJobStatus(JobID oldJobID) throws IOException {
org.apache.hadoop.mapreduce.v2.api.records.JobId jobId =
TypeConverter.toYarn(oldJobID);
GetJobReportRequest request =
recordFactory.newRecordInstance(GetJobReportRequest.class);
request.setJobId(jobId);
JobReport report = ((GetJobReportResponse) invoke("getJobReport",
GetJobReportRequest.class, request)).getJobReport();
JobStatus jobStatus = null;
if (report != null) {
if (StringUtils.isEmpty(report.getJobFile())) {
String jobFile = MRApps.getJobFile(conf, report.getUser(), oldJobID);
report.setJobFile(jobFile);
}
String historyTrackingUrl = report.getTrackingUrl();
String url = StringUtils.isNotEmpty(historyTrackingUrl)
? historyTrackingUrl : trackingUrl;
jobStatus = TypeConverter.fromYarn(report, url);
}
return jobStatus;
}
Example #19
| Source File: NotRunningJob.java From big-c with Apache License 2.0 | 6 votes |
|
@Override
public GetJobReportResponse getJobReport(GetJobReportRequest request)
throws IOException {
JobReport jobReport =
recordFactory.newRecordInstance(JobReport.class);
jobReport.setJobId(request.getJobId());
jobReport.setJobState(jobState);
jobReport.setUser(applicationReport.getUser());
jobReport.setStartTime(applicationReport.getStartTime());
jobReport.setDiagnostics(applicationReport.getDiagnostics());
jobReport.setJobName(applicationReport.getName());
jobReport.setTrackingUrl(applicationReport.getTrackingUrl());
jobReport.setFinishTime(applicationReport.getFinishTime());
GetJobReportResponse resp =
recordFactory.newRecordInstance(GetJobReportResponse.class);
resp.setJobReport(jobReport);
return resp;
}
Example #20
| Source File: MRBuilderUtils.java From hadoop with Apache License 2.0 | 6 votes |
|
public static JobReport newJobReport(JobId jobId, String jobName,
String userName, JobState state, long submitTime, long startTime, long finishTime,
float setupProgress, float mapProgress, float reduceProgress,
float cleanupProgress, String jobFile, List<AMInfo> amInfos,
boolean isUber, String diagnostics) {
JobReport report = Records.newRecord(JobReport.class);
report.setJobId(jobId);
report.setJobName(jobName);
report.setUser(userName);
report.setJobState(state);
report.setSubmitTime(submitTime);
report.setStartTime(startTime);
report.setFinishTime(finishTime);
report.setSetupProgress(setupProgress);
report.setCleanupProgress(cleanupProgress);
report.setMapProgress(mapProgress);
report.setReduceProgress(reduceProgress);
report.setJobFile(jobFile);
report.setAMInfos(amInfos);
report.setIsUber(isUber);
report.setDiagnostics(diagnostics);
return report;
}
Example #21
| Source File: TestTypeConverter.java From hadoop with Apache License 2.0 | 6 votes |
|
@Test
public void testFromYarnJobReport() throws Exception {
int jobStartTime = 612354;
int jobFinishTime = 612355;
JobState state = JobState.RUNNING;
JobId jobId = Records.newRecord(JobId.class);
JobReport jobReport = Records.newRecord(JobReport.class);
ApplicationId applicationId = ApplicationId.newInstance(0, 0);
jobId.setAppId(applicationId);
jobId.setId(0);
jobReport.setJobId(jobId);
jobReport.setJobState(state);
jobReport.setStartTime(jobStartTime);
jobReport.setFinishTime(jobFinishTime);
jobReport.setUser("TestTypeConverter-user");
JobStatus jobStatus = TypeConverter.fromYarn(jobReport, "dummy-jobfile");
Assert.assertEquals(jobStartTime, jobStatus.getStartTime());
Assert.assertEquals(jobFinishTime, jobStatus.getFinishTime());
Assert.assertEquals(state.toString(), jobStatus.getState().toString());
}
Example #22
| Source File: NotRunningJob.java From tez with Apache License 2.0 | 6 votes |
|
@Override
public GetJobReportResponse getJobReport(GetJobReportRequest request)
throws IOException {
JobReport jobReport =
recordFactory.newRecordInstance(JobReport.class);
jobReport.setJobId(request.getJobId());
jobReport.setJobState(jobState);
jobReport.setUser(applicationReport.getUser());
jobReport.setStartTime(applicationReport.getStartTime());
jobReport.setDiagnostics(applicationReport.getDiagnostics());
jobReport.setJobName(applicationReport.getName());
jobReport.setTrackingUrl(applicationReport.getTrackingUrl());
jobReport.setFinishTime(applicationReport.getFinishTime());
GetJobReportResponse resp =
recordFactory.newRecordInstance(GetJobReportResponse.class);
resp.setJobReport(jobReport);
return resp;
}
Example #23
| Source File: MRApp.java From big-c with Apache License 2.0 | 6 votes |
|
public void verifyCompleted() {
for (Job job : getContext().getAllJobs().values()) {
JobReport jobReport = job.getReport();
System.out.println("Job start time :" + jobReport.getStartTime());
System.out.println("Job finish time :" + jobReport.getFinishTime());
Assert.assertTrue("Job start time is not less than finish time",
jobReport.getStartTime() <= jobReport.getFinishTime());
Assert.assertTrue("Job finish time is in future",
jobReport.getFinishTime() <= System.currentTimeMillis());
for (Task task : job.getTasks().values()) {
TaskReport taskReport = task.getReport();
System.out.println("Task start time : " + taskReport.getStartTime());
System.out.println("Task finish time : " + taskReport.getFinishTime());
Assert.assertTrue("Task start time is not less than finish time",
taskReport.getStartTime() <= taskReport.getFinishTime());
for (TaskAttempt attempt : task.getAttempts().values()) {
TaskAttemptReport attemptReport = attempt.getReport();
Assert.assertTrue("Attempt start time is not less than finish time",
attemptReport.getStartTime() <= attemptReport.getFinishTime());
}
}
}
}
Example #24
| Source File: NotRunningJob.java From incubator-tez with Apache License 2.0 | 6 votes |
|
@Override
public GetJobReportResponse getJobReport(GetJobReportRequest request)
throws IOException {
JobReport jobReport =
recordFactory.newRecordInstance(JobReport.class);
jobReport.setJobId(request.getJobId());
jobReport.setJobState(jobState);
jobReport.setUser(applicationReport.getUser());
jobReport.setStartTime(applicationReport.getStartTime());
jobReport.setDiagnostics(applicationReport.getDiagnostics());
jobReport.setJobName(applicationReport.getName());
jobReport.setTrackingUrl(applicationReport.getTrackingUrl());
jobReport.setFinishTime(applicationReport.getFinishTime());
GetJobReportResponse resp =
recordFactory.newRecordInstance(GetJobReportResponse.class);
resp.setJobReport(jobReport);
return resp;
}
Example #25
| Source File: MockJobs.java From big-c with Apache License 2.0 | 5 votes |
|
public static JobReport newJobReport(JobId id) {
JobReport report = Records.newRecord(JobReport.class);
report.setJobId(id);
report.setSubmitTime(System.currentTimeMillis()-DT);
report
.setStartTime(System.currentTimeMillis() - (int) (Math.random() * DT));
report.setFinishTime(System.currentTimeMillis()
+ (int) (Math.random() * DT) + 1);
report.setMapProgress((float) Math.random());
report.setReduceProgress((float) Math.random());
report.setJobState(JOB_STATES.next());
return report;
}
Example #26
| Source File: MRCommunicator.java From jumbune with GNU Lesser General Public License v3.0 | 5 votes |
|
/**
* Given the required details (application id and suffix id) for JobId it gives the JobReport
* @param appid, the Application Id instance
* @param id, the suffix id
* @return the Job Report
* @throws IOException
*/
public JobReport getJobReport(ApplicationId appId, int id) throws IOException{
JobId jobId = YarnCommunicatorUtil.getJobId(appId, (int)1);
GetJobReportRequestProto proto = GetJobReportRequestProto.getDefaultInstance();
GetJobReportRequest request = new GetJobReportRequestPBImpl(proto);
request.setJobId(jobId);
GetJobReportResponse jobReportResponse = proxy.getJobReport(request);
return jobReportResponse.getJobReport();
}
Example #27
| Source File: TestMRClientService.java From big-c with Apache License 2.0 | 5 votes |
|
private void verifyJobReport(JobReport jr) {
Assert.assertNotNull("JobReport is null", jr);
List<AMInfo> amInfos = jr.getAMInfos();
Assert.assertEquals(1, amInfos.size());
Assert.assertEquals(JobState.RUNNING, jr.getJobState());
AMInfo amInfo = amInfos.get(0);
Assert.assertEquals(MRApp.NM_HOST, amInfo.getNodeManagerHost());
Assert.assertEquals(MRApp.NM_PORT, amInfo.getNodeManagerPort());
Assert.assertEquals(MRApp.NM_HTTP_PORT, amInfo.getNodeManagerHttpPort());
Assert.assertEquals(1, amInfo.getAppAttemptId().getAttemptId());
Assert.assertEquals(1, amInfo.getContainerId().getApplicationAttemptId()
.getAttemptId());
Assert.assertTrue(amInfo.getStartTime() > 0);
Assert.assertEquals(false, jr.isUber());
}
Example #28
| Source File: JobImpl.java From big-c with Apache License 2.0 | 5 votes |
|
@Override
public JobReport getReport() {
readLock.lock();
try {
JobState state = getState();
// jobFile can be null if the job is not yet inited.
String jobFile =
remoteJobConfFile == null ? "" : remoteJobConfFile.toString();
StringBuilder diagsb = new StringBuilder();
for (String s : getDiagnostics()) {
diagsb.append(s).append("\n");
}
if (getInternalState() == JobStateInternal.NEW) {
return MRBuilderUtils.newJobReport(jobId, jobName, username, state,
appSubmitTime, startTime, finishTime, setupProgress, 0.0f, 0.0f,
cleanupProgress, jobFile, amInfos, isUber, diagsb.toString());
}
computeProgress();
JobReport report = MRBuilderUtils.newJobReport(jobId, jobName, username,
state, appSubmitTime, startTime, finishTime, setupProgress,
this.mapProgress, this.reduceProgress,
cleanupProgress, jobFile, amInfos, isUber, diagsb.toString());
return report;
} finally {
readLock.unlock();
}
}
Example #29
| Source File: TestAMWebServicesJobs.java From big-c with Apache License 2.0 | 5 votes |
|
public void verifyAMJobGeneric(Job job, String id, String user, String name,
String state, long startTime, long finishTime, long elapsedTime,
int mapsTotal, int mapsCompleted, int reducesTotal, int reducesCompleted,
float reduceProgress, float mapProgress) {
JobReport report = job.getReport();
WebServicesTestUtils.checkStringMatch("id", MRApps.toString(job.getID()),
id);
WebServicesTestUtils.checkStringMatch("user", job.getUserName().toString(),
user);
WebServicesTestUtils.checkStringMatch("name", job.getName(), name);
WebServicesTestUtils.checkStringMatch("state", job.getState().toString(),
state);
assertEquals("startTime incorrect", report.getStartTime(), startTime);
assertEquals("finishTime incorrect", report.getFinishTime(), finishTime);
assertEquals("elapsedTime incorrect",
Times.elapsed(report.getStartTime(), report.getFinishTime()),
elapsedTime);
assertEquals("mapsTotal incorrect", job.getTotalMaps(), mapsTotal);
assertEquals("mapsCompleted incorrect", job.getCompletedMaps(),
mapsCompleted);
assertEquals("reducesTotal incorrect", job.getTotalReduces(), reducesTotal);
assertEquals("reducesCompleted incorrect", job.getCompletedReduces(),
reducesCompleted);
assertEquals("mapProgress incorrect", report.getMapProgress() * 100,
mapProgress, 0);
assertEquals("reduceProgress incorrect", report.getReduceProgress() * 100,
reduceProgress, 0);
}
Example #30
| Source File: JobImpl.java From hadoop with Apache License 2.0 | 5 votes |
|
@Override
public JobReport getReport() {
readLock.lock();
try {
JobState state = getState();
// jobFile can be null if the job is not yet inited.
String jobFile =
remoteJobConfFile == null ? "" : remoteJobConfFile.toString();
StringBuilder diagsb = new StringBuilder();
for (String s : getDiagnostics()) {
diagsb.append(s).append("\n");
}
if (getInternalState() == JobStateInternal.NEW) {
return MRBuilderUtils.newJobReport(jobId, jobName, username, state,
appSubmitTime, startTime, finishTime, setupProgress, 0.0f, 0.0f,
cleanupProgress, jobFile, amInfos, isUber, diagsb.toString());
}
computeProgress();
JobReport report = MRBuilderUtils.newJobReport(jobId, jobName, username,
state, appSubmitTime, startTime, finishTime, setupProgress,
this.mapProgress, this.reduceProgress,
cleanupProgress, jobFile, amInfos, isUber, diagsb.toString());
return report;
} finally {
readLock.unlock();
}
}
