org.apache.hadoop.tools.rumen.JobStoryProducer Java Examples

/**
 * Setup gridmix for emulation of distributed cache load. This includes
 * generation of distributed cache files, if needed.
 * @param conf gridmix configuration
 * @param traceIn trace file path(if it is '-', then trace comes from the
 *                stream stdin)
 * @param ioPath <ioPath>/input/ is the dir where input data (a) exists
 *               or (b) is generated. <ioPath>/distributedCache/ is the
 *               folder where distributed cache data (a) exists or (b) is to be
 *               generated by gridmix.
 * @param generate true if -generate option was specified
 * @return exit code
 * @throws IOException
 * @throws InterruptedException
 */
private int setupDistCacheEmulation(Configuration conf, String traceIn,
    Path ioPath, boolean generate) throws IOException, InterruptedException {
  distCacheEmulator.init(traceIn, factory.jobCreator, generate);
  int exitCode = 0;
  if (distCacheEmulator.shouldGenerateDistCacheData() ||
      distCacheEmulator.shouldEmulateDistCacheLoad()) {

    JobStoryProducer jsp = createJobStoryProducer(traceIn, conf);
    exitCode = distCacheEmulator.setupGenerateDistCacheData(jsp);
    if (exitCode == 0) {
      // If there are files to be generated, run a MapReduce job to generate
      // these distributed cache files of all the simulated jobs of this trace.
      writeDistCacheData(conf);
    }
  }
  return exitCode;
}
 

/**
 * Creating a new instance does not start the thread.
 *
 * @param submitter   Component to which deserialized jobs are passed
 * @param jobProducer Stream of job traces with which to construct a
 *                    {@link org.apache.hadoop.tools.rumen.ZombieJobProducer}
 * @param scratch     Directory into which to write output from simulated jobs
 * @param conf        Config passed to all jobs to be submitted
 * @param startFlag   Latch released from main to start pipeline
 * @throws java.io.IOException
 */
public StressJobFactory(
  JobSubmitter submitter, JobStoryProducer jobProducer, Path scratch,
  Configuration conf, CountDownLatch startFlag, UserResolver resolver)
  throws IOException {
  super(
    submitter, jobProducer, scratch, conf, startFlag, resolver);
  overloadMapTaskMapSlotRatio = conf.getFloat(
      CONF_OVERLOAD_MAPTASK_MAPSLOT_RATIO, OVERLOAD_MAPTASK_MAPSLOT_RATIO);
  overloadReduceTaskReduceSlotRatio = conf.getFloat(
      CONF_OVERLOAD_REDUCETASK_REDUCESLOT_RATIO, 
      OVERLOAD_REDUCETASK_REDUCESLOT_RATIO);
  maxMapSlotSharePerJob = conf.getFloat(
      CONF_MAX_MAPSLOT_SHARE_PER_JOB, MAX_MAPSLOT_SHARE_PER_JOB);
  maxReduceSlotSharePerJob = conf.getFloat(
      CONF_MAX_REDUCESLOT_SHARE_PER_JOB, MAX_REDUCESLOT_SHARE_PER_JOB);
  maxJobTrackerRatio = conf.getFloat(
      CONF_MAX_JOB_TRACKER_RATIO, MAX_JOB_TRACKER_RATIO);
}
 

/**
 * Constructor permitting JobStoryProducer to be mocked.
 * @param submitter Component to which deserialized jobs are passed
 * @param jobProducer Producer generating JobStory objects.
 * @param scratch Directory into which to write output from simulated jobs
 * @param conf Config passed to all jobs to be submitted
 * @param startFlag Latch released from main to start pipeline
 */
protected JobFactory(JobSubmitter submitter, JobStoryProducer jobProducer,
    Path scratch, Configuration conf, CountDownLatch startFlag,
    UserResolver userResolver) {
  sequence = new AtomicInteger(0);
  this.scratch = scratch;
  this.rateFactor = conf.getFloat(Gridmix.GRIDMIX_SUB_MUL, 1.0f);
  this.jobProducer = jobProducer;
  this.conf = new Configuration(conf);
  this.submitter = submitter;
  this.startFlag = startFlag;
  this.rThread = createReaderThread();
  if(LOG.isDebugEnabled()) {
    LOG.debug(" The submission thread name is " + rThread.getName());
  }
  this.userResolver = userResolver;
  this.jobCreator = JobCreator.getPolicy(conf, JobCreator.LOADJOB);
}
 

/**
 * Creating a new instance does not start the thread.
 *
 * @param submitter   Component to which deserialized jobs are passed
 * @param jobProducer Stream of job traces with which to construct a
 *                    {@link org.apache.hadoop.tools.rumen.ZombieJobProducer}
 * @param scratch     Directory into which to write output from simulated jobs
 * @param conf        Config passed to all jobs to be submitted
 * @param startFlag   Latch released from main to start pipeline
 * @throws java.io.IOException
 */
public StressJobFactory(
  JobSubmitter submitter, JobStoryProducer jobProducer, Path scratch,
  Configuration conf, CountDownLatch startFlag, UserResolver resolver)
  throws IOException {
  super(
    submitter, jobProducer, scratch, conf, startFlag, resolver);
  overloadMapTaskMapSlotRatio = conf.getFloat(
      CONF_OVERLOAD_MAPTASK_MAPSLOT_RATIO, OVERLOAD_MAPTASK_MAPSLOT_RATIO);
  overloadReduceTaskReduceSlotRatio = conf.getFloat(
      CONF_OVERLOAD_REDUCETASK_REDUCESLOT_RATIO, 
      OVERLOAD_REDUCETASK_REDUCESLOT_RATIO);
  maxMapSlotSharePerJob = conf.getFloat(
      CONF_MAX_MAPSLOT_SHARE_PER_JOB, MAX_MAPSLOT_SHARE_PER_JOB);
  maxReduceSlotSharePerJob = conf.getFloat(
      CONF_MAX_REDUCESLOT_SHARE_PER_JOB, MAX_REDUCESLOT_SHARE_PER_JOB);
  maxJobTrackerRatio = conf.getFloat(
      CONF_MAX_JOB_TRACKER_RATIO, MAX_JOB_TRACKER_RATIO);
}
 

/**
 * Constructor permitting JobStoryProducer to be mocked.
 * @param submitter Component to which deserialized jobs are passed
 * @param jobProducer Producer generating JobStory objects.
 * @param scratch Directory into which to write output from simulated jobs
 * @param conf Config passed to all jobs to be submitted
 * @param startFlag Latch released from main to start pipeline
 */
protected JobFactory(JobSubmitter submitter, JobStoryProducer jobProducer,
    Path scratch, Configuration conf, CountDownLatch startFlag,
    UserResolver userResolver) {
  sequence = new AtomicInteger(0);
  this.scratch = scratch;
  this.rateFactor = conf.getFloat(Gridmix.GRIDMIX_SUB_MUL, 1.0f);
  this.jobProducer = jobProducer;
  this.conf = new Configuration(conf);
  this.submitter = submitter;
  this.startFlag = startFlag;
  this.rThread = createReaderThread();
  if(LOG.isDebugEnabled()) {
    LOG.debug(" The submission thread name is " + rThread.getName());
  }
  this.userResolver = userResolver;
  this.jobCreator = JobCreator.getPolicy(conf, JobCreator.LOADJOB);
}
 

/**
 * Setup gridmix for emulation of distributed cache load. This includes
 * generation of distributed cache files, if needed.
 * @param conf gridmix configuration
 * @param traceIn trace file path(if it is '-', then trace comes from the
 *                stream stdin)
 * @param ioPath <ioPath>/input/ is the dir where input data (a) exists
 *               or (b) is generated. <ioPath>/distributedCache/ is the
 *               folder where distributed cache data (a) exists or (b) is to be
 *               generated by gridmix.
 * @param generate true if -generate option was specified
 * @return exit code
 * @throws IOException
 * @throws InterruptedException
 */
private int setupDistCacheEmulation(Configuration conf, String traceIn,
    Path ioPath, boolean generate) throws IOException, InterruptedException {
  distCacheEmulator.init(traceIn, factory.jobCreator, generate);
  int exitCode = 0;
  if (distCacheEmulator.shouldGenerateDistCacheData() ||
      distCacheEmulator.shouldEmulateDistCacheLoad()) {

    JobStoryProducer jsp = createJobStoryProducer(traceIn, conf);
    exitCode = distCacheEmulator.setupGenerateDistCacheData(jsp);
    if (exitCode == 0) {
      // If there are files to be generated, run a MapReduce job to generate
      // these distributed cache files of all the simulated jobs of this trace.
      writeDistCacheData(conf);
    }
  }
  return exitCode;
}
 

/**
 * Create the list of unique distributed cache files needed for all the
 * simulated jobs and write the list to a special file.
 * @param jsp job story producer for the trace
 * @return exit code
 * @throws IOException
 */
private int buildDistCacheFilesList(JobStoryProducer jsp) throws IOException {
  // Read all the jobs from the trace file and build the list of unique
  // distributed cache files.
  JobStory jobStory;
  while ((jobStory = jsp.getNextJob()) != null) {
    if (jobStory.getOutcome() == Pre21JobHistoryConstants.Values.SUCCESS && 
       jobStory.getSubmissionTime() >= 0) {
      updateHDFSDistCacheFilesList(jobStory);
    }
  }
  jsp.close();

  return writeDistCacheFilesList();
}
 

/**
 * Constructor permitting JobStoryProducer to be mocked.
 * @param submitter Component to which deserialized jobs are passed
 * @param jobProducer Producer generating JobStory objects.
 * @param scratch Directory into which to write output from simulated jobs
 * @param conf Config passed to all jobs to be submitted
 * @param startFlag Latch released from main to start pipeline
 */
protected JobFactory(JobSubmitter submitter, JobStoryProducer jobProducer,
    Path scratch, Configuration conf, CountDownLatch startFlag) {
  sequence = new AtomicInteger(0);
  this.scratch = scratch;
  this.rateFactor = conf.getFloat(Gridmix.GRIDMIX_SUB_MUL, 1.0f);
  this.jobProducer = jobProducer;
  this.conf = new Configuration(conf);
  this.submitter = submitter;
  this.startFlag = startFlag;
  this.rThread = new ReaderThread();
}
 

@Test (timeout=120000)
public void testSerialReaderThread() throws Exception {

  Configuration conf = new Configuration();
  File fin = new File("src" + File.separator + "test" + File.separator
          + "resources" + File.separator + "data" + File.separator
          + "wordcount2.json");
  // read couple jobs from wordcount2.json
  JobStoryProducer jobProducer = new ZombieJobProducer(new Path(
          fin.getAbsolutePath()), null, conf);
  CountDownLatch startFlag = new CountDownLatch(1);
  UserResolver resolver = new SubmitterUserResolver();
  FakeJobSubmitter submitter = new FakeJobSubmitter();
  File ws = new File("target" + File.separator + this.getClass().getName());
  if (!ws.exists()) {
    Assert.assertTrue(ws.mkdirs());
  }

  SerialJobFactory jobFactory = new SerialJobFactory(submitter, jobProducer,
          new Path(ws.getAbsolutePath()), conf, startFlag, resolver);

  Path ioPath = new Path(ws.getAbsolutePath());
  jobFactory.setDistCacheEmulator(new DistributedCacheEmulator(conf, ioPath));
  Thread test = jobFactory.createReaderThread();
  test.start();
  Thread.sleep(1000);
  // SerialReaderThread waits startFlag
  assertEquals(0, submitter.getJobs().size());
  // start!
  startFlag.countDown();
  while (test.isAlive()) {
    Thread.sleep(1000);
    jobFactory.update(null);
  }
  // submitter was called twice
  assertEquals(2, submitter.getJobs().size());
}
 

@Test (timeout=120000)
public void testSerialReaderThread() throws Exception {

  Configuration conf = new Configuration();
  File fin = new File("src" + File.separator + "test" + File.separator
          + "resources" + File.separator + "data" + File.separator
          + "wordcount2.json");
  // read couple jobs from wordcount2.json
  JobStoryProducer jobProducer = new ZombieJobProducer(new Path(
          fin.getAbsolutePath()), null, conf);
  CountDownLatch startFlag = new CountDownLatch(1);
  UserResolver resolver = new SubmitterUserResolver();
  FakeJobSubmitter submitter = new FakeJobSubmitter();
  File ws = new File("target" + File.separator + this.getClass().getName());
  if (!ws.exists()) {
    Assert.assertTrue(ws.mkdirs());
  }

  SerialJobFactory jobFactory = new SerialJobFactory(submitter, jobProducer,
          new Path(ws.getAbsolutePath()), conf, startFlag, resolver);

  Path ioPath = new Path(ws.getAbsolutePath());
  jobFactory.setDistCacheEmulator(new DistributedCacheEmulator(conf, ioPath));
  Thread test = jobFactory.createReaderThread();
  test.start();
  Thread.sleep(1000);
  // SerialReaderThread waits startFlag
  assertEquals(0, submitter.getJobs().size());
  // start!
  startFlag.countDown();
  while (test.isAlive()) {
    Thread.sleep(1000);
    jobFactory.update(null);
  }
  // submitter was called twice
  assertEquals(2, submitter.getJobs().size());
}
 

/**
 * Create the list of unique distributed cache files needed for all the
 * simulated jobs and write the list to a special file.
 * @param jsp job story producer for the trace
 * @return exit code
 * @throws IOException
 */
private int buildDistCacheFilesList(JobStoryProducer jsp) throws IOException {
  // Read all the jobs from the trace file and build the list of unique
  // distributed cache files.
  JobStory jobStory;
  while ((jobStory = jsp.getNextJob()) != null) {
    if (jobStory.getOutcome() == Pre21JobHistoryConstants.Values.SUCCESS && 
       jobStory.getSubmissionTime() >= 0) {
      updateHDFSDistCacheFilesList(jobStory);
    }
  }
  jsp.close();

  return writeDistCacheFilesList();
}
 

/**
 * Tests the reading of traces in GridMix3. These traces are generated by
 * Rumen and are in the JSON format. The traces can optionally be compressed
 * and uncompressed traces can also be passed to GridMix3 via its standard
 * input stream. The testing is effected via JUnit assertions.
 *
 * @throws Exception if there was an error.
 */
@Test (timeout=20000)
public void testTraceReader() throws Exception {
  Configuration conf = new Configuration();
  FileSystem lfs = FileSystem.getLocal(conf);
  Path rootInputDir = new Path(System.getProperty("src.test.data"));
  rootInputDir = rootInputDir.makeQualified(lfs.getUri(),
          lfs.getWorkingDirectory());
  Path rootTempDir = new Path(System.getProperty("test.build.data",
          System.getProperty("java.io.tmpdir")), "testTraceReader");
  rootTempDir = rootTempDir.makeQualified(lfs.getUri(),
          lfs.getWorkingDirectory());
  Path inputFile = new Path(rootInputDir, "wordcount.json.gz");
  Path tempFile = new Path(rootTempDir, "gridmix3-wc.json");

  InputStream origStdIn = System.in;
  InputStream tmpIs = null;
  try {
    DebugGridmix dgm = new DebugGridmix();
    JobStoryProducer jsp = dgm.createJobStoryProducer(inputFile.toString(),
            conf);

    LOG.info("Verifying JobStory from compressed trace...");
    verifyWordCountJobStory(jsp.getNextJob());

    expandGzippedTrace(lfs, inputFile, tempFile);
    jsp = dgm.createJobStoryProducer(tempFile.toString(), conf);
    LOG.info("Verifying JobStory from uncompressed trace...");
    verifyWordCountJobStory(jsp.getNextJob());

    tmpIs = lfs.open(tempFile);
    System.setIn(tmpIs);
    LOG.info("Verifying JobStory from trace in standard input...");
    jsp = dgm.createJobStoryProducer("-", conf);
    verifyWordCountJobStory(jsp.getNextJob());
  } finally {
    System.setIn(origStdIn);
    if (tmpIs != null) {
      tmpIs.close();
    }
    lfs.delete(rootTempDir, true);
  }
}
 

/**
 * Tests the reading of traces in GridMix3. These traces are generated by
 * Rumen and are in the JSON format. The traces can optionally be compressed
 * and uncompressed traces can also be passed to GridMix3 via its standard
 * input stream. The testing is effected via JUnit assertions.
 *
 * @throws Exception if there was an error.
 */
@Test (timeout=20000)
public void testTraceReader() throws Exception {
  Configuration conf = new Configuration();
  FileSystem lfs = FileSystem.getLocal(conf);
  Path rootInputDir = new Path(System.getProperty("src.test.data"));
  rootInputDir = rootInputDir.makeQualified(lfs.getUri(),
          lfs.getWorkingDirectory());
  Path rootTempDir = new Path(System.getProperty("test.build.data",
          System.getProperty("java.io.tmpdir")), "testTraceReader");
  rootTempDir = rootTempDir.makeQualified(lfs.getUri(),
          lfs.getWorkingDirectory());
  Path inputFile = new Path(rootInputDir, "wordcount.json.gz");
  Path tempFile = new Path(rootTempDir, "gridmix3-wc.json");

  InputStream origStdIn = System.in;
  InputStream tmpIs = null;
  try {
    DebugGridmix dgm = new DebugGridmix();
    JobStoryProducer jsp = dgm.createJobStoryProducer(inputFile.toString(),
            conf);

    LOG.info("Verifying JobStory from compressed trace...");
    verifyWordCountJobStory(jsp.getNextJob());

    expandGzippedTrace(lfs, inputFile, tempFile);
    jsp = dgm.createJobStoryProducer(tempFile.toString(), conf);
    LOG.info("Verifying JobStory from uncompressed trace...");
    verifyWordCountJobStory(jsp.getNextJob());

    tmpIs = lfs.open(tempFile);
    System.setIn(tmpIs);
    LOG.info("Verifying JobStory from trace in standard input...");
    jsp = dgm.createJobStoryProducer("-", conf);
    verifyWordCountJobStory(jsp.getNextJob());
  } finally {
    System.setIn(origStdIn);
    if (tmpIs != null) {
      tmpIs.close();
    }
    lfs.delete(rootTempDir, true);
  }
}
 

/**
 * Initiate components in the simulation.
 * @throws InterruptedException
 * @throws IOException if trace or topology files cannot be open
 */
@SuppressWarnings("deprecation")
void init() throws InterruptedException, IOException {
  long now = System.currentTimeMillis();

  JobConf jobConf = new JobConf(getConf());
  jobConf.setClass("topology.node.switch.mapping.impl",
      StaticMapping.class, DNSToSwitchMapping.class);
  jobConf.set("fs.default.name", "file:///");
  jobConf.set("mapred.job.tracker", "localhost:8012");
  jobConf.setInt("mapred.jobtracker.job.history.block.size", 512);
  jobConf.setInt("mapred.jobtracker.job.history.buffer.size", 512);
  jobConf.setLong("mapred.tasktracker.expiry.interval", 5000);
  jobConf.setInt("mapred.reduce.copy.backoff", 4);
  jobConf.setLong("mapred.job.reuse.jvm.num.tasks", -1);
  jobConf.setUser("mumak");
  jobConf.set("mapred.system.dir", 
      jobConf.get("hadoop.log.dir", "/tmp/hadoop-"+jobConf.getUser()) + "/mapred/system");
  jobConf.set("mapred.jobtracker.taskScheduler", JobQueueTaskScheduler.class.getName());
  
  FileSystem lfs = FileSystem.getLocal(getConf());
  Path logPath =
    new Path(System.getProperty("hadoop.log.dir")).makeQualified(lfs);
  jobConf.set("mapred.system.dir", logPath.toString());
  jobConf.set("hadoop.job.history.location", (new Path(logPath, "history")
      .toString()));
  
  jt = SimulatorJobTracker.startTracker(jobConf, now, this);
  jt.offerService();
  
  // max Map/Reduce tasks per node
  int maxMaps = getConf().getInt("mapred.tasktracker.map.tasks.maximum",
      DEFAULT_MAP_SLOTS_PER_NODE);
  int maxReduces = getConf().getInt(
      "mapred.tasktracker.reduce.tasks.maximum",
      DEFAULT_REDUCE_SLOTS_PER_NODE);

  MachineNode defaultNode = new MachineNode.Builder("default", 2)
      .setMapSlots(maxMaps).setReduceSlots(maxReduces).build();
  ZombieCluster cluster = new ZombieCluster(new Path(topologyFile), 
      defaultNode, jobConf);
  long firstJobStartTime = now + 60000;
  JobStoryProducer jobStoryProducer = new SimulatorJobStoryProducer(
      new Path(traceFile), cluster, firstJobStartTime, jobConf);
  
  jc = new SimulatorJobClient(jt, jobStoryProducer);
  queue.addAll(jc.init(firstJobStartTime));

  // create TTs based on topology.json     
  startTaskTrackers(cluster, now);
  
  terminateTime = getConf().getLong("mumak.terminate.time", Long.MAX_VALUE);
  if (terminateTime <= 0) {
    throw new IllegalArgumentException("Terminate time must be positive: "
        + terminateTime);
  }
}
 

/**
 * Create distributed cache directories.
 * Also create a file that contains the list of distributed cache files
 * that will be used as distributed cache files for all the simulated jobs.
 * @param jsp job story producer for the trace
 * @return exit code
 * @throws IOException
 */
int setupGenerateDistCacheData(JobStoryProducer jsp)
    throws IOException {

  createDistCacheDirectory();
  return buildDistCacheFilesList(jsp);
}
 

/**
 * Create distributed cache directories.
 * Also create a file that contains the list of distributed cache files
 * that will be used as distributed cache files for all the simulated jobs.
 * @param jsp job story producer for the trace
 * @return exit code
 * @throws IOException
 */
int setupGenerateDistCacheData(JobStoryProducer jsp)
    throws IOException {

  createDistCacheDirectory();
  return buildDistCacheFilesList(jsp);
}
 

/**
 * Create an appropriate {@code JobStoryProducer} object for the
 * given trace.
 * 
 * @param traceIn the path to the trace file. The special path
 * "-" denotes the standard input stream.
 *
 * @param conf the configuration to be used.
 *
 * @throws IOException if there was an error.
 */
protected JobStoryProducer createJobStoryProducer(String traceIn,
    Configuration conf) throws IOException {
  if ("-".equals(traceIn)) {
    return new ZombieJobProducer(System.in, null);
  }
  return new ZombieJobProducer(new Path(traceIn), null, conf);
}
 

/**
 * Creating a new instance does not start the thread.
 *
 * @param submitter   Component to which deserialized jobs are passed
 * @param jobProducer Job story producer
 *                    {@link org.apache.hadoop.tools.rumen.ZombieJobProducer}
 * @param scratch     Directory into which to write output from simulated jobs
 * @param conf        Config passed to all jobs to be submitted
 * @param startFlag   Latch released from main to start pipeline
 * @param resolver
 * @throws java.io.IOException
 */
public ReplayJobFactory(
  JobSubmitter submitter, JobStoryProducer jobProducer, Path scratch,
  Configuration conf, CountDownLatch startFlag, UserResolver resolver)
  throws IOException {
  super(submitter, jobProducer, scratch, conf, startFlag, resolver);
}
 

/**
 * Creating a new instance does not start the thread.
 *
 * @param submitter   Component to which deserialized jobs are passed
 * @param jobProducer Job story producer
 *                    {@link org.apache.hadoop.tools.rumen.ZombieJobProducer}
 * @param scratch     Directory into which to write output from simulated jobs
 * @param conf        Config passed to all jobs to be submitted
 * @param startFlag   Latch released from main to start pipeline
 * @throws java.io.IOException
 */
public SerialJobFactory(
  JobSubmitter submitter, JobStoryProducer jobProducer, Path scratch,
  Configuration conf, CountDownLatch startFlag, UserResolver resolver)
  throws IOException {
  super(submitter, jobProducer, scratch, conf, startFlag, resolver);
}
 

/**
 * Creating a new instance does not start the thread.
 *
 * @param submitter   Component to which deserialized jobs are passed
 * @param jobProducer Job story producer
 *                    {@link org.apache.hadoop.tools.rumen.ZombieJobProducer}
 * @param scratch     Directory into which to write output from simulated jobs
 * @param conf        Config passed to all jobs to be submitted
 * @param startFlag   Latch released from main to start pipeline
 * @param resolver
 * @throws java.io.IOException
 */
public ReplayJobFactory(
  JobSubmitter submitter, JobStoryProducer jobProducer, Path scratch,
  Configuration conf, CountDownLatch startFlag, UserResolver resolver)
  throws IOException {
  super(submitter, jobProducer, scratch, conf, startFlag, resolver);
}
 

/**
 * Creating a new instance does not start the thread.
 *
 * @param submitter   Component to which deserialized jobs are passed
 * @param jobProducer Job story producer
 *                    {@link org.apache.hadoop.tools.rumen.ZombieJobProducer}
 * @param scratch     Directory into which to write output from simulated jobs
 * @param conf        Config passed to all jobs to be submitted
 * @param startFlag   Latch released from main to start pipeline
 * @throws java.io.IOException
 */
public SerialJobFactory(
  JobSubmitter submitter, JobStoryProducer jobProducer, Path scratch,
  Configuration conf, CountDownLatch startFlag, UserResolver resolver)
  throws IOException {
  super(submitter, jobProducer, scratch, conf, startFlag, resolver);
}
 

/**
 * Create an appropriate {@code JobStoryProducer} object for the
 * given trace.
 * 
 * @param traceIn the path to the trace file. The special path
 * "-" denotes the standard input stream.
 *
 * @param conf the configuration to be used.
 *
 * @throws IOException if there was an error.
 */
protected JobStoryProducer createJobStoryProducer(String traceIn,
    Configuration conf) throws IOException {
  if ("-".equals(traceIn)) {
    return new ZombieJobProducer(System.in, null);
  }
  return new ZombieJobProducer(new Path(traceIn), null, conf);
}
 

/**
 * Constructor.
 * 
 * @param jobTracker
 *          The job tracker where we submit job to. Note that the {@link
 *          SimulatorJobClient} interacts with the JobTracker through the
 *          {@link ClientProtocol}.
 * @param jobStoryProducer
 */
public SimulatorJobClient(JobTracker jobTracker, JobStoryProducer jobStoryProducer) {
  this.jobTracker = jobTracker;
  this.jobStoryProducer = jobStoryProducer;
}