Java Code Examples for org.apache.hadoop.mapreduce.JobStatus.State#RUNNING

The following examples show how to use org.apache.hadoop.mapreduce.JobStatus.State#RUNNING . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: TypeConverter.java    From hadoop with Apache License 2.0 6 votes vote down vote up 
public static State fromYarn(YarnApplicationState yarnApplicationState,
    FinalApplicationStatus finalApplicationStatus) {
  switch (yarnApplicationState) {
  case NEW:
  case NEW_SAVING:
  case SUBMITTED:
  case ACCEPTED:
    return State.PREP;
  case RUNNING:
    return State.RUNNING;
  case FINISHED:
    if (finalApplicationStatus == FinalApplicationStatus.SUCCEEDED) {
      return State.SUCCEEDED;
    } else if (finalApplicationStatus == FinalApplicationStatus.KILLED) {
      return State.KILLED;
    }
  case FAILED:
    return State.FAILED;
  case KILLED:
    return State.KILLED;
  }
  throw new YarnRuntimeException("Unrecognized application state: " + yarnApplicationState);
}
Example 2
Source File: TypeConverter.java    From big-c with Apache License 2.0 6 votes vote down vote up 
public static State fromYarn(YarnApplicationState yarnApplicationState,
    FinalApplicationStatus finalApplicationStatus) {
  switch (yarnApplicationState) {
  case NEW:
  case NEW_SAVING:
  case SUBMITTED:
  case ACCEPTED:
    return State.PREP;
  case RUNNING:
    return State.RUNNING;
  case FINISHED:
    if (finalApplicationStatus == FinalApplicationStatus.SUCCEEDED) {
      return State.SUCCEEDED;
    } else if (finalApplicationStatus == FinalApplicationStatus.KILLED) {
      return State.KILLED;
    }
  case FAILED:
    return State.FAILED;
  case KILLED:
    return State.KILLED;
  }
  throw new YarnRuntimeException("Unrecognized application state: " + yarnApplicationState);
}
Example 3
Source File: TestJobMonitorAndPrint.java    From hadoop with Apache License 2.0 5 votes vote down vote up 
@Before
public void setUp() throws IOException {
  conf = new Configuration();
  clientProtocol = mock(ClientProtocol.class);
  Cluster cluster = mock(Cluster.class);
  when(cluster.getConf()).thenReturn(conf);
  when(cluster.getClient()).thenReturn(clientProtocol);
  JobStatus jobStatus = new JobStatus(new JobID("job_000", 1), 0f, 0f, 0f, 0f, 
      State.RUNNING, JobPriority.HIGH, "tmp-user", "tmp-jobname", 
      "tmp-jobfile", "tmp-url");
  job = Job.getInstance(cluster, jobStatus, conf);
  job = spy(job);
}
Example 4
Source File: TestJobMonitorAndPrint.java    From big-c with Apache License 2.0 5 votes vote down vote up 
@Before
public void setUp() throws IOException {
  conf = new Configuration();
  clientProtocol = mock(ClientProtocol.class);
  Cluster cluster = mock(Cluster.class);
  when(cluster.getConf()).thenReturn(conf);
  when(cluster.getClient()).thenReturn(clientProtocol);
  JobStatus jobStatus = new JobStatus(new JobID("job_000", 1), 0f, 0f, 0f, 0f, 
      State.RUNNING, JobPriority.HIGH, "tmp-user", "tmp-jobname", 
      "tmp-jobfile", "tmp-url");
  job = Job.getInstance(cluster, jobStatus, conf);
  job = spy(job);
}
Example 5
Source File: TestJobMonitorAndPrint.java    From hadoop with Apache License 2.0 4 votes vote down vote up 
@Test
public void testJobMonitorAndPrint() throws Exception {
  JobStatus jobStatus_1 = new JobStatus(new JobID("job_000", 1), 1f, 0.1f,
      0.1f, 0f, State.RUNNING, JobPriority.HIGH, "tmp-user", "tmp-jobname",
      "tmp-queue", "tmp-jobfile", "tmp-url", true);
  JobStatus jobStatus_2 = new JobStatus(new JobID("job_000", 1), 1f, 1f,
      1f, 1f, State.SUCCEEDED, JobPriority.HIGH, "tmp-user", "tmp-jobname",
      "tmp-queue", "tmp-jobfile", "tmp-url", true);

  doAnswer(
      new Answer<TaskCompletionEvent[]>() {
        @Override
        public TaskCompletionEvent[] answer(InvocationOnMock invocation)
            throws Throwable {
          return new TaskCompletionEvent[0];
        }
      }
      ).when(job).getTaskCompletionEvents(anyInt(), anyInt());

  doReturn(new TaskReport[5]).when(job).getTaskReports(isA(TaskType.class));
  when(clientProtocol.getJobStatus(any(JobID.class))).thenReturn(jobStatus_1, jobStatus_2);
  // setup the logger to capture all logs
  Layout layout =
      Logger.getRootLogger().getAppender("stdout").getLayout();
  ByteArrayOutputStream os = new ByteArrayOutputStream();
  WriterAppender appender = new WriterAppender(layout, os);
  appender.setThreshold(Level.ALL);
  Logger qlogger = Logger.getLogger(Job.class);
  qlogger.addAppender(appender);

  job.monitorAndPrintJob();

  qlogger.removeAppender(appender);
  LineNumberReader r = new LineNumberReader(new StringReader(os.toString()));
  String line;
  boolean foundHundred = false;
  boolean foundComplete = false;
  boolean foundUber = false;
  String uberModeMatch = "uber mode : true";
  String progressMatch = "map 100% reduce 100%";
  String completionMatch = "completed successfully";
  while ((line = r.readLine()) != null) {
    if (line.contains(uberModeMatch)) {
      foundUber = true;
    }
    foundHundred = line.contains(progressMatch);      
    if (foundHundred)
      break;
  }
  line = r.readLine();
  foundComplete = line.contains(completionMatch);
  assertTrue(foundUber);
  assertTrue(foundHundred);
  assertTrue(foundComplete);

  System.out.println("The output of job.toString() is : \n" + job.toString());
  assertTrue(job.toString().contains("Number of maps: 5\n"));
  assertTrue(job.toString().contains("Number of reduces: 5\n"));
}
Example 6
Source File: TestJobMonitorAndPrint.java    From big-c with Apache License 2.0 4 votes vote down vote up 
@Test
public void testJobMonitorAndPrint() throws Exception {
  JobStatus jobStatus_1 = new JobStatus(new JobID("job_000", 1), 1f, 0.1f,
      0.1f, 0f, State.RUNNING, JobPriority.HIGH, "tmp-user", "tmp-jobname",
      "tmp-queue", "tmp-jobfile", "tmp-url", true);
  JobStatus jobStatus_2 = new JobStatus(new JobID("job_000", 1), 1f, 1f,
      1f, 1f, State.SUCCEEDED, JobPriority.HIGH, "tmp-user", "tmp-jobname",
      "tmp-queue", "tmp-jobfile", "tmp-url", true);

  doAnswer(
      new Answer<TaskCompletionEvent[]>() {
        @Override
        public TaskCompletionEvent[] answer(InvocationOnMock invocation)
            throws Throwable {
          return new TaskCompletionEvent[0];
        }
      }
      ).when(job).getTaskCompletionEvents(anyInt(), anyInt());

  doReturn(new TaskReport[5]).when(job).getTaskReports(isA(TaskType.class));
  when(clientProtocol.getJobStatus(any(JobID.class))).thenReturn(jobStatus_1, jobStatus_2);
  // setup the logger to capture all logs
  Layout layout =
      Logger.getRootLogger().getAppender("stdout").getLayout();
  ByteArrayOutputStream os = new ByteArrayOutputStream();
  WriterAppender appender = new WriterAppender(layout, os);
  appender.setThreshold(Level.ALL);
  Logger qlogger = Logger.getLogger(Job.class);
  qlogger.addAppender(appender);

  job.monitorAndPrintJob();

  qlogger.removeAppender(appender);
  LineNumberReader r = new LineNumberReader(new StringReader(os.toString()));
  String line;
  boolean foundHundred = false;
  boolean foundComplete = false;
  boolean foundUber = false;
  String uberModeMatch = "uber mode : true";
  String progressMatch = "map 100% reduce 100%";
  String completionMatch = "completed successfully";
  while ((line = r.readLine()) != null) {
    if (line.contains(uberModeMatch)) {
      foundUber = true;
    }
    foundHundred = line.contains(progressMatch);      
    if (foundHundred)
      break;
  }
  line = r.readLine();
  foundComplete = line.contains(completionMatch);
  assertTrue(foundUber);
  assertTrue(foundHundred);
  assertTrue(foundComplete);

  System.out.println("The output of job.toString() is : \n" + job.toString());
  assertTrue(job.toString().contains("Number of maps: 5\n"));
  assertTrue(job.toString().contains("Number of reduces: 5\n"));
}