Java Code Examples for org.apache.hadoop.mapreduce.v2.api.records.JobId#setAppId()
The following examples show how to use
org.apache.hadoop.mapreduce.v2.api.records.JobId#setAppId() .
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
| Source File: TestStagingCleanup.java From big-c with Apache License 2.0 | 6 votes |
|
public void testDeletionofStagingOnKillLastTry() throws IOException {
conf.set(MRJobConfig.MAPREDUCE_JOB_DIR, stagingJobDir);
fs = mock(FileSystem.class);
when(fs.delete(any(Path.class), anyBoolean())).thenReturn(true);
//Staging Dir exists
String user = UserGroupInformation.getCurrentUser().getShortUserName();
Path stagingDir = MRApps.getStagingAreaDir(conf, user);
when(fs.exists(stagingDir)).thenReturn(true);
ApplicationId appId = ApplicationId.newInstance(System.currentTimeMillis(),
0);
ApplicationAttemptId attemptId = ApplicationAttemptId.newInstance(appId, 1);
JobId jobid = recordFactory.newRecordInstance(JobId.class);
jobid.setAppId(appId);
ContainerAllocator mockAlloc = mock(ContainerAllocator.class);
MRAppMaster appMaster = new TestMRApp(attemptId, mockAlloc); //no retry
appMaster.init(conf);
assertTrue("appMaster.isLastAMRetry() is false", appMaster.isLastAMRetry());
//simulate the process being killed
MRAppMaster.MRAppMasterShutdownHook hook =
new MRAppMaster.MRAppMasterShutdownHook(appMaster);
hook.run();
assertTrue("MRAppMaster isn't stopped",
appMaster.isInState(Service.STATE.STOPPED));
verify(fs).delete(stagingJobPath, true);
}
Example 2
| Source File: TestBlocks.java From big-c with Apache License 2.0 | 6 votes |
|
private Job getJob() {
Job job = mock(Job.class);
JobId jobId = new JobIdPBImpl();
ApplicationId appId = ApplicationIdPBImpl.newInstance(System.currentTimeMillis(),4);
jobId.setAppId(appId);
jobId.setId(1);
when(job.getID()).thenReturn(jobId);
JobReport report = mock(JobReport.class);
when(report.getStartTime()).thenReturn(100010L);
when(report.getFinishTime()).thenReturn(100015L);
when(job.getReport()).thenReturn(report);
when(job.getName()).thenReturn("JobName");
when(job.getUserName()).thenReturn("UserName");
when(job.getQueueName()).thenReturn("QueueName");
when(job.getState()).thenReturn(JobState.SUCCEEDED);
when(job.getTotalMaps()).thenReturn(3);
when(job.getCompletedMaps()).thenReturn(2);
when(job.getTotalReduces()).thenReturn(2);
when(job.getCompletedReduces()).thenReturn(1);
when(job.getCompletedReduces()).thenReturn(1);
return job;
}
Example 3
| Source File: TestStagingCleanup.java From hadoop with Apache License 2.0 | 6 votes |
|
@Test
public void testDeletionofStaging() throws IOException {
conf.set(MRJobConfig.MAPREDUCE_JOB_DIR, stagingJobDir);
fs = mock(FileSystem.class);
when(fs.delete(any(Path.class), anyBoolean())).thenReturn(true);
//Staging Dir exists
String user = UserGroupInformation.getCurrentUser().getShortUserName();
Path stagingDir = MRApps.getStagingAreaDir(conf, user);
when(fs.exists(stagingDir)).thenReturn(true);
ApplicationId appId = ApplicationId.newInstance(System.currentTimeMillis(),
0);
ApplicationAttemptId attemptId = ApplicationAttemptId.newInstance(appId, 1);
JobId jobid = recordFactory.newRecordInstance(JobId.class);
jobid.setAppId(appId);
ContainerAllocator mockAlloc = mock(ContainerAllocator.class);
Assert.assertTrue(MRJobConfig.DEFAULT_MR_AM_MAX_ATTEMPTS > 1);
MRAppMaster appMaster = new TestMRApp(attemptId, mockAlloc,
JobStateInternal.RUNNING, MRJobConfig.DEFAULT_MR_AM_MAX_ATTEMPTS);
appMaster.init(conf);
appMaster.start();
appMaster.shutDownJob();
//test whether notifyIsLastAMRetry called
Assert.assertEquals(true, ((TestMRApp)appMaster).getTestIsLastAMRetry());
verify(fs).delete(stagingJobPath, true);
}
Example 4
| Source File: TestStagingCleanup.java From hadoop with Apache License 2.0 | 6 votes |
|
@Test (timeout = 30000)
public void testDeletionofStagingOnKill() throws IOException {
conf.set(MRJobConfig.MAPREDUCE_JOB_DIR, stagingJobDir);
fs = mock(FileSystem.class);
when(fs.delete(any(Path.class), anyBoolean())).thenReturn(true);
//Staging Dir exists
String user = UserGroupInformation.getCurrentUser().getShortUserName();
Path stagingDir = MRApps.getStagingAreaDir(conf, user);
when(fs.exists(stagingDir)).thenReturn(true);
ApplicationId appId = ApplicationId.newInstance(System.currentTimeMillis(),
0);
ApplicationAttemptId attemptId = ApplicationAttemptId.newInstance(appId, 0);
JobId jobid = recordFactory.newRecordInstance(JobId.class);
jobid.setAppId(appId);
ContainerAllocator mockAlloc = mock(ContainerAllocator.class);
MRAppMaster appMaster = new TestMRApp(attemptId, mockAlloc);
appMaster.init(conf);
//simulate the process being killed
MRAppMaster.MRAppMasterShutdownHook hook =
new MRAppMaster.MRAppMasterShutdownHook(appMaster);
hook.run();
verify(fs, times(0)).delete(stagingJobPath, true);
}
Example 5
| Source File: TestStagingCleanup.java From hadoop with Apache License 2.0 | 6 votes |
|
public void testDeletionofStagingOnKillLastTry() throws IOException {
conf.set(MRJobConfig.MAPREDUCE_JOB_DIR, stagingJobDir);
fs = mock(FileSystem.class);
when(fs.delete(any(Path.class), anyBoolean())).thenReturn(true);
//Staging Dir exists
String user = UserGroupInformation.getCurrentUser().getShortUserName();
Path stagingDir = MRApps.getStagingAreaDir(conf, user);
when(fs.exists(stagingDir)).thenReturn(true);
ApplicationId appId = ApplicationId.newInstance(System.currentTimeMillis(),
0);
ApplicationAttemptId attemptId = ApplicationAttemptId.newInstance(appId, 1);
JobId jobid = recordFactory.newRecordInstance(JobId.class);
jobid.setAppId(appId);
ContainerAllocator mockAlloc = mock(ContainerAllocator.class);
MRAppMaster appMaster = new TestMRApp(attemptId, mockAlloc); //no retry
appMaster.init(conf);
assertTrue("appMaster.isLastAMRetry() is false", appMaster.isLastAMRetry());
//simulate the process being killed
MRAppMaster.MRAppMasterShutdownHook hook =
new MRAppMaster.MRAppMasterShutdownHook(appMaster);
hook.run();
assertTrue("MRAppMaster isn't stopped",
appMaster.isInState(Service.STATE.STOPPED));
verify(fs).delete(stagingJobPath, true);
}
Example 6
| Source File: TestStagingCleanup.java From big-c with Apache License 2.0 | 6 votes |
|
@Test (timeout = 30000)
public void testDeletionofStagingOnKill() throws IOException {
conf.set(MRJobConfig.MAPREDUCE_JOB_DIR, stagingJobDir);
fs = mock(FileSystem.class);
when(fs.delete(any(Path.class), anyBoolean())).thenReturn(true);
//Staging Dir exists
String user = UserGroupInformation.getCurrentUser().getShortUserName();
Path stagingDir = MRApps.getStagingAreaDir(conf, user);
when(fs.exists(stagingDir)).thenReturn(true);
ApplicationId appId = ApplicationId.newInstance(System.currentTimeMillis(),
0);
ApplicationAttemptId attemptId = ApplicationAttemptId.newInstance(appId, 0);
JobId jobid = recordFactory.newRecordInstance(JobId.class);
jobid.setAppId(appId);
ContainerAllocator mockAlloc = mock(ContainerAllocator.class);
MRAppMaster appMaster = new TestMRApp(attemptId, mockAlloc);
appMaster.init(conf);
//simulate the process being killed
MRAppMaster.MRAppMasterShutdownHook hook =
new MRAppMaster.MRAppMasterShutdownHook(appMaster);
hook.run();
verify(fs, times(0)).delete(stagingJobPath, true);
}
Example 7
| Source File: TestStagingCleanup.java From big-c with Apache License 2.0 | 6 votes |
|
@Test
public void testDeletionofStaging() throws IOException {
conf.set(MRJobConfig.MAPREDUCE_JOB_DIR, stagingJobDir);
fs = mock(FileSystem.class);
when(fs.delete(any(Path.class), anyBoolean())).thenReturn(true);
//Staging Dir exists
String user = UserGroupInformation.getCurrentUser().getShortUserName();
Path stagingDir = MRApps.getStagingAreaDir(conf, user);
when(fs.exists(stagingDir)).thenReturn(true);
ApplicationId appId = ApplicationId.newInstance(System.currentTimeMillis(),
0);
ApplicationAttemptId attemptId = ApplicationAttemptId.newInstance(appId, 1);
JobId jobid = recordFactory.newRecordInstance(JobId.class);
jobid.setAppId(appId);
ContainerAllocator mockAlloc = mock(ContainerAllocator.class);
Assert.assertTrue(MRJobConfig.DEFAULT_MR_AM_MAX_ATTEMPTS > 1);
MRAppMaster appMaster = new TestMRApp(attemptId, mockAlloc,
JobStateInternal.RUNNING, MRJobConfig.DEFAULT_MR_AM_MAX_ATTEMPTS);
appMaster.init(conf);
appMaster.start();
appMaster.shutDownJob();
//test whether notifyIsLastAMRetry called
Assert.assertEquals(true, ((TestMRApp)appMaster).getTestIsLastAMRetry());
verify(fs).delete(stagingJobPath, true);
}
Example 8
| Source File: TestTypeConverter.java From hadoop with Apache License 2.0 | 6 votes |
|
@Test
public void testFromYarnJobReport() throws Exception {
int jobStartTime = 612354;
int jobFinishTime = 612355;
JobState state = JobState.RUNNING;
JobId jobId = Records.newRecord(JobId.class);
JobReport jobReport = Records.newRecord(JobReport.class);
ApplicationId applicationId = ApplicationId.newInstance(0, 0);
jobId.setAppId(applicationId);
jobId.setId(0);
jobReport.setJobId(jobId);
jobReport.setJobState(state);
jobReport.setStartTime(jobStartTime);
jobReport.setFinishTime(jobFinishTime);
jobReport.setUser("TestTypeConverter-user");
JobStatus jobStatus = TypeConverter.fromYarn(jobReport, "dummy-jobfile");
Assert.assertEquals(jobStartTime, jobStatus.getStartTime());
Assert.assertEquals(jobFinishTime, jobStatus.getFinishTime());
Assert.assertEquals(state.toString(), jobStatus.getState().toString());
}
Example 9
| Source File: TestBlocks.java From hadoop with Apache License 2.0 | 6 votes |
|
private Job getJob() {
Job job = mock(Job.class);
JobId jobId = new JobIdPBImpl();
ApplicationId appId = ApplicationIdPBImpl.newInstance(System.currentTimeMillis(),4);
jobId.setAppId(appId);
jobId.setId(1);
when(job.getID()).thenReturn(jobId);
JobReport report = mock(JobReport.class);
when(report.getStartTime()).thenReturn(100010L);
when(report.getFinishTime()).thenReturn(100015L);
when(job.getReport()).thenReturn(report);
when(job.getName()).thenReturn("JobName");
when(job.getUserName()).thenReturn("UserName");
when(job.getQueueName()).thenReturn("QueueName");
when(job.getState()).thenReturn(JobState.SUCCEEDED);
when(job.getTotalMaps()).thenReturn(3);
when(job.getCompletedMaps()).thenReturn(2);
when(job.getTotalReduces()).thenReturn(2);
when(job.getCompletedReduces()).thenReturn(1);
when(job.getCompletedReduces()).thenReturn(1);
return job;
}
Example 10
| Source File: TestTypeConverter.java From big-c with Apache License 2.0 | 6 votes |
|
@Test
public void testFromYarnJobReport() throws Exception {
int jobStartTime = 612354;
int jobFinishTime = 612355;
JobState state = JobState.RUNNING;
JobId jobId = Records.newRecord(JobId.class);
JobReport jobReport = Records.newRecord(JobReport.class);
ApplicationId applicationId = ApplicationId.newInstance(0, 0);
jobId.setAppId(applicationId);
jobId.setId(0);
jobReport.setJobId(jobId);
jobReport.setJobState(state);
jobReport.setStartTime(jobStartTime);
jobReport.setFinishTime(jobFinishTime);
jobReport.setUser("TestTypeConverter-user");
JobStatus jobStatus = TypeConverter.fromYarn(jobReport, "dummy-jobfile");
Assert.assertEquals(jobStartTime, jobStatus.getStartTime());
Assert.assertEquals(jobFinishTime, jobStatus.getFinishTime());
Assert.assertEquals(state.toString(), jobStatus.getState().toString());
}
Example 11
| Source File: TypeConverter.java From big-c with Apache License 2.0 | 5 votes |
|
public static JobId toYarn(org.apache.hadoop.mapreduce.JobID id) {
JobId jobId = recordFactory.newRecordInstance(JobId.class);
jobId.setId(id.getId()); //currently there is 1-1 mapping between appid and jobid
ApplicationId appId = ApplicationId.newInstance(
toClusterTimeStamp(id.getJtIdentifier()), id.getId());
jobId.setAppId(appId);
return jobId;
}
Example 12
| Source File: TestStagingCleanup.java From big-c with Apache License 2.0 | 5 votes |
|
@SuppressWarnings("resource")
private void testDeletionofStagingOnUnregistrationFailure(
int maxAttempts, boolean shouldHaveDeleted) throws IOException {
conf.set(MRJobConfig.MAPREDUCE_JOB_DIR, stagingJobDir);
fs = mock(FileSystem.class);
when(fs.delete(any(Path.class), anyBoolean())).thenReturn(true);
//Staging Dir exists
String user = UserGroupInformation.getCurrentUser().getShortUserName();
Path stagingDir = MRApps.getStagingAreaDir(conf, user);
when(fs.exists(stagingDir)).thenReturn(true);
ApplicationId appId = ApplicationId.newInstance(0, 1);
ApplicationAttemptId attemptId = ApplicationAttemptId.newInstance(appId, 1);
JobId jobid = recordFactory.newRecordInstance(JobId.class);
jobid.setAppId(appId);
TestMRApp appMaster = new TestMRApp(attemptId, null,
JobStateInternal.RUNNING, maxAttempts);
appMaster.crushUnregistration = true;
appMaster.init(conf);
appMaster.start();
appMaster.shutDownJob();
((RunningAppContext) appMaster.getContext()).resetIsLastAMRetry();
if (shouldHaveDeleted) {
Assert.assertEquals(new Boolean(true), appMaster.isLastAMRetry());
verify(fs).delete(stagingJobPath, true);
} else {
Assert.assertEquals(new Boolean(false), appMaster.isLastAMRetry());
verify(fs, never()).delete(stagingJobPath, true);
}
}
Example 13
| Source File: TestMRApps.java From big-c with Apache License 2.0 | 4 votes |
|
@Test (timeout = 120000)
public void testJobIDtoString() {
JobId jid = RecordFactoryProvider.getRecordFactory(null).newRecordInstance(JobId.class);
jid.setAppId(ApplicationId.newInstance(0, 0));
assertEquals("job_0_0000", MRApps.toString(jid));
}
Example 14
| Source File: TestBlocks.java From big-c with Apache License 2.0 | 4 votes |
|
/**
* Test rendering for TasksBlock
*/
@Test
public void testTasksBlock() throws Exception {
ApplicationId appId = ApplicationIdPBImpl.newInstance(0, 1);
JobId jobId = new JobIdPBImpl();
jobId.setId(0);
jobId.setAppId(appId);
TaskId taskId = new TaskIdPBImpl();
taskId.setId(0);
taskId.setTaskType(TaskType.MAP);
taskId.setJobId(jobId);
Task task = mock(Task.class);
when(task.getID()).thenReturn(taskId);
TaskReport report = mock(TaskReport.class);
when(report.getProgress()).thenReturn(0.7f);
when(report.getTaskState()).thenReturn(TaskState.SUCCEEDED);
when(report.getStartTime()).thenReturn(100001L);
when(report.getFinishTime()).thenReturn(100011L);
when(report.getStatus()).thenReturn("Dummy Status \n*");
when(task.getReport()).thenReturn(report);
when(task.getType()).thenReturn(TaskType.MAP);
Map<TaskId, Task> tasks = new HashMap<TaskId, Task>();
tasks.put(taskId, task);
AppContext ctx = mock(AppContext.class);
Job job = mock(Job.class);
when(job.getTasks()).thenReturn(tasks);
App app = new App(ctx);
app.setJob(job);
TasksBlockForTest taskBlock = new TasksBlockForTest(app);
taskBlock.addParameter(AMParams.TASK_TYPE, "m");
PrintWriter pWriter = new PrintWriter(data);
Block html = new BlockForTest(new HtmlBlockForTest(), pWriter, 0, false);
taskBlock.render(html);
pWriter.flush();
assertTrue(data.toString().contains("task_0_0001_m_000000"));
assertTrue(data.toString().contains("70.00"));
assertTrue(data.toString().contains("SUCCEEDED"));
assertTrue(data.toString().contains("100001"));
assertTrue(data.toString().contains("100011"));
assertFalse(data.toString().contains("Dummy Status \n*"));
assertTrue(data.toString().contains("Dummy Status \\n*"));
}
Example 15
| Source File: MockJobs.java From big-c with Apache License 2.0 | 4 votes |
|
public static JobId newJobID(ApplicationId appID, int i) {
JobId id = Records.newRecord(JobId.class);
id.setAppId(appID);
id.setId(i);
return id;
}
Example 16
| Source File: TestMRApps.java From hadoop with Apache License 2.0 | 4 votes |
|
@Test (timeout = 120000)
public void testJobIDtoString() {
JobId jid = RecordFactoryProvider.getRecordFactory(null).newRecordInstance(JobId.class);
jid.setAppId(ApplicationId.newInstance(0, 0));
assertEquals("job_0_0000", MRApps.toString(jid));
}
Example 17
| Source File: MRBuilderUtils.java From hadoop with Apache License 2.0 | 4 votes |
|
public static JobId newJobId(ApplicationId appId, int id) {
JobId jobId = Records.newRecord(JobId.class);
jobId.setAppId(appId);
jobId.setId(id);
return jobId;
}
Example 18
| Source File: MRBuilderUtils.java From big-c with Apache License 2.0 | 4 votes |
|
public static JobId newJobId(ApplicationId appId, int id) {
JobId jobId = Records.newRecord(JobId.class);
jobId.setAppId(appId);
jobId.setId(id);
return jobId;
}
Example 19
| Source File: MockJobs.java From hadoop with Apache License 2.0 | 4 votes |
|
public static JobId newJobID(ApplicationId appID, int i) {
JobId id = Records.newRecord(JobId.class);
id.setAppId(appID);
id.setId(i);
return id;
}
Example 20
| Source File: TestBlocks.java From hadoop with Apache License 2.0 | 4 votes |
|
/**
* Test rendering for TasksBlock
*/
@Test
public void testTasksBlock() throws Exception {
ApplicationId appId = ApplicationIdPBImpl.newInstance(0, 1);
JobId jobId = new JobIdPBImpl();
jobId.setId(0);
jobId.setAppId(appId);
TaskId taskId = new TaskIdPBImpl();
taskId.setId(0);
taskId.setTaskType(TaskType.MAP);
taskId.setJobId(jobId);
Task task = mock(Task.class);
when(task.getID()).thenReturn(taskId);
TaskReport report = mock(TaskReport.class);
when(report.getProgress()).thenReturn(0.7f);
when(report.getTaskState()).thenReturn(TaskState.SUCCEEDED);
when(report.getStartTime()).thenReturn(100001L);
when(report.getFinishTime()).thenReturn(100011L);
when(report.getStatus()).thenReturn("Dummy Status \n*");
when(task.getReport()).thenReturn(report);
when(task.getType()).thenReturn(TaskType.MAP);
Map<TaskId, Task> tasks = new HashMap<TaskId, Task>();
tasks.put(taskId, task);
AppContext ctx = mock(AppContext.class);
Job job = mock(Job.class);
when(job.getTasks()).thenReturn(tasks);
App app = new App(ctx);
app.setJob(job);
TasksBlockForTest taskBlock = new TasksBlockForTest(app);
taskBlock.addParameter(AMParams.TASK_TYPE, "m");
PrintWriter pWriter = new PrintWriter(data);
Block html = new BlockForTest(new HtmlBlockForTest(), pWriter, 0, false);
taskBlock.render(html);
pWriter.flush();
assertTrue(data.toString().contains("task_0_0001_m_000000"));
assertTrue(data.toString().contains("70.00"));
assertTrue(data.toString().contains("SUCCEEDED"));
assertTrue(data.toString().contains("100001"));
assertTrue(data.toString().contains("100011"));
assertFalse(data.toString().contains("Dummy Status \n*"));
assertTrue(data.toString().contains("Dummy Status \\n*"));
}
