Java Code Examples for org.apache.hadoop.mapreduce.v2.api.records.JobId#setId()

The following examples show how to use org.apache.hadoop.mapreduce.v2.api.records.JobId#setId() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: TestBlocks.java    From big-c with Apache License 2.0 6 votes vote down vote up 
private Task getTask(long timestamp) {
  
  JobId jobId = new JobIdPBImpl();
  jobId.setId(0);
  jobId.setAppId(ApplicationIdPBImpl.newInstance(timestamp,1));

  TaskId taskId = new TaskIdPBImpl();
  taskId.setId(0);
  taskId.setTaskType(TaskType.REDUCE);
  taskId.setJobId(jobId);
  Task task = mock(Task.class);
  when(task.getID()).thenReturn(taskId);
  TaskReport report = mock(TaskReport.class);
  when(report.getProgress()).thenReturn(0.7f);
  when(report.getTaskState()).thenReturn(TaskState.SUCCEEDED);
  when(report.getStartTime()).thenReturn(100001L);
  when(report.getFinishTime()).thenReturn(100011L);

  when(task.getReport()).thenReturn(report);
  when(task.getType()).thenReturn(TaskType.REDUCE);
  return task;
}
Example 2
Source File: TestBlocks.java    From big-c with Apache License 2.0 6 votes vote down vote up 
private Job getJob() {
  Job job = mock(Job.class);

  JobId jobId = new JobIdPBImpl();

  ApplicationId appId = ApplicationIdPBImpl.newInstance(System.currentTimeMillis(),4);
  jobId.setAppId(appId);
  jobId.setId(1);
  when(job.getID()).thenReturn(jobId);

  JobReport report = mock(JobReport.class);
  when(report.getStartTime()).thenReturn(100010L);
  when(report.getFinishTime()).thenReturn(100015L);

  when(job.getReport()).thenReturn(report);
  when(job.getName()).thenReturn("JobName");
  when(job.getUserName()).thenReturn("UserName");
  when(job.getQueueName()).thenReturn("QueueName");
  when(job.getState()).thenReturn(JobState.SUCCEEDED);
  when(job.getTotalMaps()).thenReturn(3);
  when(job.getCompletedMaps()).thenReturn(2);
  when(job.getTotalReduces()).thenReturn(2);
  when(job.getCompletedReduces()).thenReturn(1);
  when(job.getCompletedReduces()).thenReturn(1);
  return job;
}
Example 3
Source File: TestTypeConverter.java    From hadoop with Apache License 2.0 6 votes vote down vote up 
@Test
public void testFromYarnJobReport() throws Exception {
  int jobStartTime = 612354;
  int jobFinishTime = 612355;
  JobState state = JobState.RUNNING;
  JobId jobId = Records.newRecord(JobId.class);
  JobReport jobReport = Records.newRecord(JobReport.class);
  ApplicationId applicationId = ApplicationId.newInstance(0, 0);
  jobId.setAppId(applicationId);
  jobId.setId(0);    
  jobReport.setJobId(jobId);
  jobReport.setJobState(state);
  jobReport.setStartTime(jobStartTime);
  jobReport.setFinishTime(jobFinishTime);
  jobReport.setUser("TestTypeConverter-user");    
  JobStatus jobStatus = TypeConverter.fromYarn(jobReport, "dummy-jobfile");
  Assert.assertEquals(jobStartTime, jobStatus.getStartTime());
  Assert.assertEquals(jobFinishTime, jobStatus.getFinishTime());    
  Assert.assertEquals(state.toString(), jobStatus.getState().toString());
}
Example 4
Source File: TestJobHistoryParsing.java    From hadoop with Apache License 2.0 6 votes vote down vote up 
/**
 * Simple test PartialJob
 */
@Test(timeout = 3000)
public void testPartialJob() throws Exception {
  JobId jobId = new JobIdPBImpl();
  jobId.setId(0);
  JobIndexInfo jii = new JobIndexInfo(0L, System.currentTimeMillis(), "user",
      "jobName", jobId, 3, 2, "JobStatus");
  PartialJob test = new PartialJob(jii, jobId);
  assertEquals(1.0f, test.getProgress(), 0.001);
  assertNull(test.getAllCounters());
  assertNull(test.getTasks());
  assertNull(test.getTasks(TaskType.MAP));
  assertNull(test.getTask(new TaskIdPBImpl()));

  assertNull(test.getTaskAttemptCompletionEvents(0, 100));
  assertNull(test.getMapAttemptCompletionEvents(0, 100));
  assertTrue(test.checkAccess(UserGroupInformation.getCurrentUser(), null));
  assertNull(test.getAMInfos());

}
Example 5
Source File: TestBlocks.java    From hadoop with Apache License 2.0 6 votes vote down vote up 
private Job getJob() {
  Job job = mock(Job.class);

  JobId jobId = new JobIdPBImpl();

  ApplicationId appId = ApplicationIdPBImpl.newInstance(System.currentTimeMillis(),4);
  jobId.setAppId(appId);
  jobId.setId(1);
  when(job.getID()).thenReturn(jobId);

  JobReport report = mock(JobReport.class);
  when(report.getStartTime()).thenReturn(100010L);
  when(report.getFinishTime()).thenReturn(100015L);

  when(job.getReport()).thenReturn(report);
  when(job.getName()).thenReturn("JobName");
  when(job.getUserName()).thenReturn("UserName");
  when(job.getQueueName()).thenReturn("QueueName");
  when(job.getState()).thenReturn(JobState.SUCCEEDED);
  when(job.getTotalMaps()).thenReturn(3);
  when(job.getCompletedMaps()).thenReturn(2);
  when(job.getTotalReduces()).thenReturn(2);
  when(job.getCompletedReduces()).thenReturn(1);
  when(job.getCompletedReduces()).thenReturn(1);
  return job;
}
Example 6
Source File: TestBlocks.java    From hadoop with Apache License 2.0 6 votes vote down vote up 
private Task getTask(long timestamp) {
  
  JobId jobId = new JobIdPBImpl();
  jobId.setId(0);
  jobId.setAppId(ApplicationIdPBImpl.newInstance(timestamp,1));

  TaskId taskId = new TaskIdPBImpl();
  taskId.setId(0);
  taskId.setTaskType(TaskType.REDUCE);
  taskId.setJobId(jobId);
  Task task = mock(Task.class);
  when(task.getID()).thenReturn(taskId);
  TaskReport report = mock(TaskReport.class);
  when(report.getProgress()).thenReturn(0.7f);
  when(report.getTaskState()).thenReturn(TaskState.SUCCEEDED);
  when(report.getStartTime()).thenReturn(100001L);
  when(report.getFinishTime()).thenReturn(100011L);

  when(task.getReport()).thenReturn(report);
  when(task.getType()).thenReturn(TaskType.REDUCE);
  return task;
}
Example 7
Source File: TestJobHistoryParsing.java    From big-c with Apache License 2.0 6 votes vote down vote up 
/**
 * Simple test PartialJob
 */
@Test(timeout = 3000)
public void testPartialJob() throws Exception {
  JobId jobId = new JobIdPBImpl();
  jobId.setId(0);
  JobIndexInfo jii = new JobIndexInfo(0L, System.currentTimeMillis(), "user",
      "jobName", jobId, 3, 2, "JobStatus");
  PartialJob test = new PartialJob(jii, jobId);
  assertEquals(1.0f, test.getProgress(), 0.001);
  assertNull(test.getAllCounters());
  assertNull(test.getTasks());
  assertNull(test.getTasks(TaskType.MAP));
  assertNull(test.getTask(new TaskIdPBImpl()));

  assertNull(test.getTaskAttemptCompletionEvents(0, 100));
  assertNull(test.getMapAttemptCompletionEvents(0, 100));
  assertTrue(test.checkAccess(UserGroupInformation.getCurrentUser(), null));
  assertNull(test.getAMInfos());

}
Example 8
Source File: TestTypeConverter.java    From big-c with Apache License 2.0 6 votes vote down vote up 
@Test
public void testFromYarnJobReport() throws Exception {
  int jobStartTime = 612354;
  int jobFinishTime = 612355;
  JobState state = JobState.RUNNING;
  JobId jobId = Records.newRecord(JobId.class);
  JobReport jobReport = Records.newRecord(JobReport.class);
  ApplicationId applicationId = ApplicationId.newInstance(0, 0);
  jobId.setAppId(applicationId);
  jobId.setId(0);    
  jobReport.setJobId(jobId);
  jobReport.setJobState(state);
  jobReport.setStartTime(jobStartTime);
  jobReport.setFinishTime(jobFinishTime);
  jobReport.setUser("TestTypeConverter-user");    
  JobStatus jobStatus = TypeConverter.fromYarn(jobReport, "dummy-jobfile");
  Assert.assertEquals(jobStartTime, jobStatus.getStartTime());
  Assert.assertEquals(jobFinishTime, jobStatus.getFinishTime());    
  Assert.assertEquals(state.toString(), jobStatus.getState().toString());
}
Example 9
Source File: TypeConverter.java    From hadoop with Apache License 2.0 5 votes vote down vote up 
public static JobId toYarn(org.apache.hadoop.mapreduce.JobID id) {
  JobId jobId = recordFactory.newRecordInstance(JobId.class);
  jobId.setId(id.getId()); //currently there is 1-1 mapping between appid and jobid

  ApplicationId appId = ApplicationId.newInstance(
      toClusterTimeStamp(id.getJtIdentifier()), id.getId());
  jobId.setAppId(appId);
  return jobId;
}
Example 10
Source File: TypeConverter.java    From big-c with Apache License 2.0 5 votes vote down vote up 
public static JobId toYarn(org.apache.hadoop.mapreduce.JobID id) {
  JobId jobId = recordFactory.newRecordInstance(JobId.class);
  jobId.setId(id.getId()); //currently there is 1-1 mapping between appid and jobid

  ApplicationId appId = ApplicationId.newInstance(
      toClusterTimeStamp(id.getJtIdentifier()), id.getId());
  jobId.setAppId(appId);
  return jobId;
}
Example 11
Source File: TestUnnecessaryBlockingOnHistoryFileInfo.java    From hadoop with Apache License 2.0 4 votes vote down vote up 
private static JobId createJobId(int id) {
  JobId jobId = new JobIdPBImpl();
  jobId.setId(id);
  jobId.setAppId(ApplicationIdPBImpl.newInstance(0, id));
  return jobId;
}
Example 12
Source File: MRBuilderUtils.java    From big-c with Apache License 2.0 4 votes vote down vote up 
public static JobId newJobId(ApplicationId appId, int id) {
  JobId jobId = Records.newRecord(JobId.class);
  jobId.setAppId(appId);
  jobId.setId(id);
  return jobId;
}
Example 13
Source File: MockJobs.java    From big-c with Apache License 2.0 4 votes vote down vote up 
public static JobId newJobID(ApplicationId appID, int i) {
  JobId id = Records.newRecord(JobId.class);
  id.setAppId(appID);
  id.setId(i);
  return id;
}
Example 14
Source File: TestBlocks.java    From big-c with Apache License 2.0 4 votes vote down vote up 
/**
 * Test rendering for TasksBlock
 */
@Test
public void testTasksBlock() throws Exception {

  ApplicationId appId = ApplicationIdPBImpl.newInstance(0, 1);
  JobId jobId = new JobIdPBImpl();
  jobId.setId(0);
  jobId.setAppId(appId);

  TaskId taskId = new TaskIdPBImpl();
  taskId.setId(0);
  taskId.setTaskType(TaskType.MAP);
  taskId.setJobId(jobId);
  Task task = mock(Task.class);
  when(task.getID()).thenReturn(taskId);
  TaskReport report = mock(TaskReport.class);
  when(report.getProgress()).thenReturn(0.7f);
  when(report.getTaskState()).thenReturn(TaskState.SUCCEEDED);
  when(report.getStartTime()).thenReturn(100001L);
  when(report.getFinishTime()).thenReturn(100011L);
  when(report.getStatus()).thenReturn("Dummy Status \n*");


  when(task.getReport()).thenReturn(report);
  when(task.getType()).thenReturn(TaskType.MAP);


  Map<TaskId, Task> tasks = new HashMap<TaskId, Task>();
  tasks.put(taskId, task);
  AppContext ctx = mock(AppContext.class);
  Job job = mock(Job.class);
  when(job.getTasks()).thenReturn(tasks);


  App app = new App(ctx);
  app.setJob(job);
  TasksBlockForTest taskBlock = new TasksBlockForTest(app);
  taskBlock.addParameter(AMParams.TASK_TYPE, "m");

  PrintWriter pWriter = new PrintWriter(data);
  Block html = new BlockForTest(new HtmlBlockForTest(), pWriter, 0, false);

  taskBlock.render(html);
  pWriter.flush();
  assertTrue(data.toString().contains("task_0_0001_m_000000"));
  assertTrue(data.toString().contains("70.00"));
  assertTrue(data.toString().contains("SUCCEEDED"));
  assertTrue(data.toString().contains("100001"));
  assertTrue(data.toString().contains("100011"));
  assertFalse(data.toString().contains("Dummy Status \n*"));
  assertTrue(data.toString().contains("Dummy Status \\n*"));


}
Example 15
Source File: TestBlocks.java    From hadoop with Apache License 2.0 4 votes vote down vote up 
/**
 * Test rendering for TasksBlock
 */
@Test
public void testTasksBlock() throws Exception {

  ApplicationId appId = ApplicationIdPBImpl.newInstance(0, 1);
  JobId jobId = new JobIdPBImpl();
  jobId.setId(0);
  jobId.setAppId(appId);

  TaskId taskId = new TaskIdPBImpl();
  taskId.setId(0);
  taskId.setTaskType(TaskType.MAP);
  taskId.setJobId(jobId);
  Task task = mock(Task.class);
  when(task.getID()).thenReturn(taskId);
  TaskReport report = mock(TaskReport.class);
  when(report.getProgress()).thenReturn(0.7f);
  when(report.getTaskState()).thenReturn(TaskState.SUCCEEDED);
  when(report.getStartTime()).thenReturn(100001L);
  when(report.getFinishTime()).thenReturn(100011L);
  when(report.getStatus()).thenReturn("Dummy Status \n*");


  when(task.getReport()).thenReturn(report);
  when(task.getType()).thenReturn(TaskType.MAP);


  Map<TaskId, Task> tasks = new HashMap<TaskId, Task>();
  tasks.put(taskId, task);
  AppContext ctx = mock(AppContext.class);
  Job job = mock(Job.class);
  when(job.getTasks()).thenReturn(tasks);


  App app = new App(ctx);
  app.setJob(job);
  TasksBlockForTest taskBlock = new TasksBlockForTest(app);
  taskBlock.addParameter(AMParams.TASK_TYPE, "m");

  PrintWriter pWriter = new PrintWriter(data);
  Block html = new BlockForTest(new HtmlBlockForTest(), pWriter, 0, false);

  taskBlock.render(html);
  pWriter.flush();
  assertTrue(data.toString().contains("task_0_0001_m_000000"));
  assertTrue(data.toString().contains("70.00"));
  assertTrue(data.toString().contains("SUCCEEDED"));
  assertTrue(data.toString().contains("100001"));
  assertTrue(data.toString().contains("100011"));
  assertFalse(data.toString().contains("Dummy Status \n*"));
  assertTrue(data.toString().contains("Dummy Status \\n*"));


}
Example 16
Source File: MRBuilderUtils.java    From hadoop with Apache License 2.0 4 votes vote down vote up 
public static JobId newJobId(ApplicationId appId, int id) {
  JobId jobId = Records.newRecord(JobId.class);
  jobId.setAppId(appId);
  jobId.setId(id);
  return jobId;
}
Example 17
Source File: MockJobs.java    From hadoop with Apache License 2.0 4 votes vote down vote up 
public static JobId newJobID(ApplicationId appID, int i) {
  JobId id = Records.newRecord(JobId.class);
  id.setAppId(appID);
  id.setId(i);
  return id;
}