org.apache.hadoop.tools.rumen.JobStory Java Examples

@Override
public synchronized JobStatus submitJob(JobID jobId) throws IOException {
  boolean loggingEnabled = LOG.isDebugEnabled();
  if (loggingEnabled) {
    LOG.debug("submitJob for jobname = " + jobId);
  }
  if (jobs.containsKey(jobId)) {
    // job already running, don't start twice
    if (loggingEnabled) {
      LOG.debug("Job '" + jobId.getId() + "' already present ");
    }
    return jobs.get(jobId).getStatus();
  }
  JobStory jobStory = SimulatorJobCache.get(jobId);
  if (jobStory == null) {
    throw new IllegalArgumentException("Job not found in SimulatorJobCache: "+jobId);
  }
  validateAndSetClock(jobStory.getSubmissionTime());
  
  SimulatorJobInProgress job = new SimulatorJobInProgress(jobId, this,
                                                          this.conf, 
                                                          jobStory);
  return addJob(jobId, job);
}
 

@Test (timeout=30000)
public void testCompareGridmixJob() throws Exception {
  Configuration conf = new Configuration();
  Path outRoot = new Path("target");
  JobStory jobDesc = mock(JobStory.class);
  when(jobDesc.getName()).thenReturn("JobName");
  when(jobDesc.getJobConf()).thenReturn(new JobConf(conf));
  UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
  GridmixJob j1 = new LoadJob(conf, 1000L, jobDesc, outRoot, ugi, 0);
  GridmixJob j2 = new LoadJob(conf, 1000L, jobDesc, outRoot, ugi, 0);
  GridmixJob j3 = new LoadJob(conf, 1000L, jobDesc, outRoot, ugi, 1);
  GridmixJob j4 = new LoadJob(conf, 1000L, jobDesc, outRoot, ugi, 1);

  assertTrue(j1.equals(j2));
  assertEquals(0, j1.compareTo(j2));
  // Only one parameter matters
  assertFalse(j1.equals(j3));
  // compare id and submissionMillis
  assertEquals(-1, j1.compareTo(j3));
  assertEquals(-1, j1.compareTo(j4));

}
 

@Test  (timeout=600000)
public void testMapTasksOnlySleepJobs() throws Exception {
  Configuration configuration = GridmixTestUtils.mrvl.getConfig();

  DebugJobProducer jobProducer = new DebugJobProducer(5, configuration);
  configuration.setBoolean(SleepJob.SLEEPJOB_MAPTASK_ONLY, true);

  UserGroupInformation ugi = UserGroupInformation.getLoginUser();
  JobStory story;
  int seq = 1;
  while ((story = jobProducer.getNextJob()) != null) {
    GridmixJob gridmixJob = JobCreator.SLEEPJOB.createGridmixJob(configuration, 0,
            story, new Path("ignored"), ugi, seq++);
    gridmixJob.buildSplits(null);
    Job job = gridmixJob.call();
    assertEquals(0, job.getNumReduceTasks());
  }
  jobProducer.close();
  assertEquals(6, seq);
}
 

private void testRandomLocation(int locations, int njobs,
                                UserGroupInformation ugi) throws Exception {
  Configuration configuration = new Configuration();

  DebugJobProducer jobProducer = new DebugJobProducer(njobs, configuration);
  Configuration jconf = GridmixTestUtils.mrvl.getConfig();
  jconf.setInt(JobCreator.SLEEPJOB_RANDOM_LOCATIONS, locations);

  JobStory story;
  int seq = 1;
  while ((story = jobProducer.getNextJob()) != null) {
    GridmixJob gridmixJob = JobCreator.SLEEPJOB.createGridmixJob(jconf, 0,
            story, new Path("ignored"), ugi, seq++);
    gridmixJob.buildSplits(null);
    List<InputSplit> splits = new SleepJob.SleepInputFormat()
            .getSplits(gridmixJob.getJob());
    for (InputSplit split : splits) {
      assertEquals(locations, split.getLocations().length);
    }
  }
  jobProducer.close();
}
 

@Test  (timeout=600000)
public void testMapTasksOnlySleepJobs() throws Exception {
  Configuration configuration = GridmixTestUtils.mrvl.getConfig();

  DebugJobProducer jobProducer = new DebugJobProducer(5, configuration);
  configuration.setBoolean(SleepJob.SLEEPJOB_MAPTASK_ONLY, true);

  UserGroupInformation ugi = UserGroupInformation.getLoginUser();
  JobStory story;
  int seq = 1;
  while ((story = jobProducer.getNextJob()) != null) {
    GridmixJob gridmixJob = JobCreator.SLEEPJOB.createGridmixJob(configuration, 0,
            story, new Path("ignored"), ugi, seq++);
    gridmixJob.buildSplits(null);
    Job job = gridmixJob.call();
    assertEquals(0, job.getNumReduceTasks());
  }
  jobProducer.close();
  assertEquals(6, seq);
}
 

private void testRandomLocation(int locations, int njobs,
                                UserGroupInformation ugi) throws Exception {
  Configuration configuration = new Configuration();

  DebugJobProducer jobProducer = new DebugJobProducer(njobs, configuration);
  Configuration jconf = GridmixTestUtils.mrvl.getConfig();
  jconf.setInt(JobCreator.SLEEPJOB_RANDOM_LOCATIONS, locations);

  JobStory story;
  int seq = 1;
  while ((story = jobProducer.getNextJob()) != null) {
    GridmixJob gridmixJob = JobCreator.SLEEPJOB.createGridmixJob(jconf, 0,
            story, new Path("ignored"), ugi, seq++);
    gridmixJob.buildSplits(null);
    List<InputSplit> splits = new SleepJob.SleepInputFormat()
            .getSplits(gridmixJob.getJob());
    for (InputSplit split : splits) {
      assertEquals(locations, split.getLocations().length);
    }
  }
  jobProducer.close();
}
 

/**
 * Generates a job stats.
 */
public static JobStats generateJobStats(Job job, JobStory jobdesc) {
  int seq = GridmixJob.getJobSeqId(job);
  // bail out if job description is missing for a job to be simulated
  if (seq >= 0 && jobdesc == null) {
    throw new IllegalArgumentException("JobStory not available for job " 
                                       + job.getJobID());
  }
  
  int maps = -1;
  int reds = -1;
  if (jobdesc != null) {
    // Note that the ZombieJob will return a >= 0 value
    maps = jobdesc.getNumberMaps();
    reds = jobdesc.getNumberReduces();
  }
  return new JobStats(maps, reds, job);
}
 

@Test (timeout=30000)
public void testCompareGridmixJob() throws Exception {
  Configuration conf = new Configuration();
  Path outRoot = new Path("target");
  JobStory jobDesc = mock(JobStory.class);
  when(jobDesc.getName()).thenReturn("JobName");
  when(jobDesc.getJobConf()).thenReturn(new JobConf(conf));
  UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
  GridmixJob j1 = new LoadJob(conf, 1000L, jobDesc, outRoot, ugi, 0);
  GridmixJob j2 = new LoadJob(conf, 1000L, jobDesc, outRoot, ugi, 0);
  GridmixJob j3 = new LoadJob(conf, 1000L, jobDesc, outRoot, ugi, 1);
  GridmixJob j4 = new LoadJob(conf, 1000L, jobDesc, outRoot, ugi, 1);

  assertTrue(j1.equals(j2));
  assertEquals(0, j1.compareTo(j2));
  // Only one parameter matters
  assertFalse(j1.equals(j3));
  // compare id and submissionMillis
  assertEquals(-1, j1.compareTo(j3));
  assertEquals(-1, j1.compareTo(j4));

}
 

/**
 * Filter some jobs being fed to the simulator. For now, we filter out killed
 * jobs to facilitate debugging.
 * 
 * @throws IOException
 */
private JobStory getNextJobFiltered() throws IOException {
  while (true) {
    ZombieJob job = producer.getNextJob();
    if (job == null) {
      return null;
    }
    if (job.getOutcome() == Pre21JobHistoryConstants.Values.KILLED) {
      continue;
    }
    if (job.getNumberMaps() == 0) {
      continue;
    }
    if (job.getNumLoggedMaps() == 0) {
      continue;
    }
    return job;
  }
}
 

/**
 * Generates a job stats.
 */
public static JobStats generateJobStats(Job job, JobStory jobdesc) {
  int seq = GridmixJob.getJobSeqId(job);
  // bail out if job description is missing for a job to be simulated
  if (seq >= 0 && jobdesc == null) {
    throw new IllegalArgumentException("JobStory not available for job " 
                                       + job.getJobID());
  }
  
  int maps = -1;
  int reds = -1;
  if (jobdesc != null) {
    // Note that the ZombieJob will return a >= 0 value
    maps = jobdesc.getNumberMaps();
    reds = jobdesc.getNumberReduces();
  }
  return new JobStats(maps, reds, job);
}
 

@Override
public JobStory getNextJob() throws IOException {
  if (numJobs.getAndDecrement() > 0) {
    final MockJob ret = new MockJob(conf);
    submitted.add(ret);
    return ret;
  }
  return null;
}
 

public DebugJobProducer(int numJobs, Configuration conf) {
  super();
  MockJob.reset();
  this.conf = conf;
  this.numJobs = new AtomicInteger(numJobs);
  this.submitted = new ArrayList<JobStory>();
}
 

private JobStory getNextJobFromTrace() throws IOException {
  JobStory story = jobProducer.getNextJob();
  if (story != null) {
    ++numJobsInTrace;
  }
  return story;
}
 

/**
 * Verifies that the given {@code JobStory} corresponds to the checked-in
 * WordCount {@code JobStory}. The verification is effected via JUnit
 * assertions.
 *
 * @param js the candidate JobStory.
 */
private void verifyWordCountJobStory(JobStory js) {
  assertNotNull("Null JobStory", js);
  String expectedJobStory = "WordCount:johndoe:default:1285322645148:3:1";
  String actualJobStory = js.getName() + ":" + js.getUser() + ":"
          + js.getQueueName() + ":" + js.getSubmissionTime() + ":"
          + js.getNumberMaps() + ":" + js.getNumberReduces();
  assertEquals("Unexpected JobStory", expectedJobStory, actualJobStory);
}
 

private JobStory getNextJobFromTrace() throws IOException {
  JobStory story = jobProducer.getNextJob();
  if (story != null) {
    ++numJobsInTrace;
  }
  return story;
}
 

public SleepJob(Configuration conf, long submissionMillis, JobStory jobdesc,
    Path outRoot, UserGroupInformation ugi, int seq, int numLocations,
    String[] hosts) throws IOException {
  super(conf, submissionMillis, jobdesc, outRoot, ugi, seq);
  this.fakeLocations = numLocations;
  this.hosts = hosts.clone();
  this.selector = (fakeLocations > 0)? new Selector(hosts.length, (float) fakeLocations
      / hosts.length, rand.get()) : null;
  this.mapTasksOnly = conf.getBoolean(SLEEPJOB_MAPTASK_ONLY, false);
  mapMaxSleepTime = conf.getLong(GRIDMIX_SLEEP_MAX_MAP_TIME, Long.MAX_VALUE);
  reduceMaxSleepTime = conf.getLong(GRIDMIX_SLEEP_MAX_REDUCE_TIME,
      Long.MAX_VALUE);
}
 

/**
 * Create the list of unique distributed cache files needed for all the
 * simulated jobs and write the list to a special file.
 * @param jsp job story producer for the trace
 * @return exit code
 * @throws IOException
 */
private int buildDistCacheFilesList(JobStoryProducer jsp) throws IOException {
  // Read all the jobs from the trace file and build the list of unique
  // distributed cache files.
  JobStory jobStory;
  while ((jobStory = jsp.getNextJob()) != null) {
    if (jobStory.getOutcome() == Pre21JobHistoryConstants.Values.SUCCESS && 
       jobStory.getSubmissionTime() >= 0) {
      updateHDFSDistCacheFilesList(jobStory);
    }
  }
  jsp.close();

  return writeDistCacheFilesList();
}
 

/**
 * Create the list of unique distributed cache files needed for all the
 * simulated jobs and write the list to a special file.
 * @param jsp job story producer for the trace
 * @return exit code
 * @throws IOException
 */
private int buildDistCacheFilesList(JobStoryProducer jsp) throws IOException {
  // Read all the jobs from the trace file and build the list of unique
  // distributed cache files.
  JobStory jobStory;
  while ((jobStory = jsp.getNextJob()) != null) {
    if (jobStory.getOutcome() == Pre21JobHistoryConstants.Values.SUCCESS && 
       jobStory.getSubmissionTime() >= 0) {
      updateHDFSDistCacheFilesList(jobStory);
    }
  }
  jsp.close();

  return writeDistCacheFilesList();
}
 

public SleepJob(Configuration conf, long submissionMillis, JobStory jobdesc,
    Path outRoot, UserGroupInformation ugi, int seq, int numLocations,
    String[] hosts) throws IOException {
  super(conf, submissionMillis, jobdesc, outRoot, ugi, seq);
  this.fakeLocations = numLocations;
  this.hosts = hosts.clone();
  this.selector = (fakeLocations > 0)? new Selector(hosts.length, (float) fakeLocations
      / hosts.length, rand.get()) : null;
  this.mapTasksOnly = conf.getBoolean(SLEEPJOB_MAPTASK_ONLY, false);
  mapMaxSleepTime = conf.getLong(GRIDMIX_SLEEP_MAX_MAP_TIME, Long.MAX_VALUE);
  reduceMaxSleepTime = conf.getLong(GRIDMIX_SLEEP_MAX_REDUCE_TIME,
      Long.MAX_VALUE);
}
 

public DebugJobProducer(int numJobs, Configuration conf) {
  super();
  MockJob.reset();
  this.conf = conf;
  this.numJobs = new AtomicInteger(numJobs);
  this.submitted = new ArrayList<JobStory>();
}
 

@Override
public JobStory getNextJob() throws IOException {
  if (numJobs.getAndDecrement() > 0) {
    final MockJob ret = new MockJob(conf);
    submitted.add(ret);
    return ret;
  }
  return null;
}
 

/**
 * Verifies that the given {@code JobStory} corresponds to the checked-in
 * WordCount {@code JobStory}. The verification is effected via JUnit
 * assertions.
 *
 * @param js the candidate JobStory.
 */
private void verifyWordCountJobStory(JobStory js) {
  assertNotNull("Null JobStory", js);
  String expectedJobStory = "WordCount:johndoe:default:1285322645148:3:1";
  String actualJobStory = js.getName() + ":" + js.getUser() + ":"
          + js.getQueueName() + ":" + js.getSubmissionTime() + ":"
          + js.getNumberMaps() + ":" + js.getNumberReduces();
  assertEquals("Unexpected JobStory", expectedJobStory, actualJobStory);
}
 

@Override
public List<SimulatorEvent> init(long when) throws IOException {
  JobStory job = jobStoryProducer.getNextJob();
  if (job.getSubmissionTime() != when) {
    throw new IOException("Inconsistent submission time for the first job: "
        + when + " != " + job.getSubmissionTime()+".");
  }
  JobSubmissionEvent event = new JobSubmissionEvent(this, when, job);
  return Collections.<SimulatorEvent> singletonList(event);
}
 

@SuppressWarnings("deprecation")
private JobStatus submitJob(JobStory job)
    throws IOException, InterruptedException {
  // honor the JobID from JobStory first.
  JobID jobId = job.getJobID();
  if (jobId == null) {
    // If not available, obtain JobID from JobTracker.
    jobId = jobTracker.getNewJobId();
  }
  
  SimulatorJobCache.put(org.apache.hadoop.mapred.JobID.downgrade(jobId), job);
  return jobTracker.submitJob(org.apache.hadoop.mapred.JobID.downgrade(jobId));
}
 

@Override
public JobStory getNextJob() throws IOException {
  JobStory job = getNextJobFiltered();
  if (job == null)
    return null;
  if (firstJob) {
    firstJob = false;
    relativeTime = job.getSubmissionTime() - firstJobStartTime;
  }

  return new SimulatorJobStory(job, job.getSubmissionTime() - relativeTime);
}
 

@Override
public JobStory getNextJob() {
  if (index >= times.length) {
    return null;
  }
  return jobs.get(index++);
}
 

public MockSimulatorEngine(int nJobs,
    @SuppressWarnings("unused") int nTrackers) {
  super();
  fixedJobs = nJobs;
  jobs = new HashMap<JobID, JobStory>();
  submittedJobs = new HashSet<JobID>();
  completedJobs = new HashSet<JobID>();
}
 

public GridmixJob(Configuration conf, long submissionMillis,
    JobStory jobdesc, Path outRoot, int seq) throws IOException {
  ((StringBuilder)nameFormat.get().out()).setLength(JOBNAME.length());
  job = new Job(conf, nameFormat.get().format("%05d", seq).toString());
  submissionTimeNanos = TimeUnit.NANOSECONDS.convert(
      submissionMillis, TimeUnit.MILLISECONDS);
  this.jobdesc = jobdesc;
  this.seq = seq;
  outdir = new Path(outRoot, "" + seq);
}
 

private JobStory getNextJobFiltered() throws IOException {
  JobStory job;
  do {
    job = jobProducer.getNextJob();
  } while (job != null
      && (job.getOutcome() != Pre21JobHistoryConstants.Values.SUCCESS ||
          job.getSubmissionTime() < 0));
  return null == job ? null : new FilterJobStory(job) {
      @Override
      public TaskInfo getTaskInfo(TaskType taskType, int taskNumber) {
        return new MinTaskInfo(this.job.getTaskInfo(taskType, taskNumber));
      }
    };
}
 

@Override
public JobStory getNextJob() throws IOException {
  if (numJobs.getAndDecrement() > 0) {
    final MockJob ret = new MockJob(conf);
    submitted.add(ret);
    return ret;
  }
  return null;
}