org.apache.hadoop.tools.rumen.Pre21JobHistoryConstants.Values Java Examples

@SuppressWarnings("hiding") 
private TaskAttemptInfo scaleInfo(LoggedTask loggedTask,
    LoggedTaskAttempt loggedAttempt, int locality, int loggedLocality,
    double rackLocalOverNodeLocal, double rackRemoteOverNodeLocal) {
  TaskInfo taskInfo = getTaskInfo(loggedTask);
  double[] factors = new double[] { 1.0, rackLocalOverNodeLocal,
      rackRemoteOverNodeLocal };
  double scaleFactor = factors[locality] / factors[loggedLocality];
  State state = convertState(loggedAttempt.getResult());
  if (loggedTask.getTaskType() == Values.MAP) {
    long taskTime = 0;
    if (loggedAttempt.getStartTime() == 0) {
      taskTime = makeUpMapRuntime(state, locality);
    } else {
      taskTime = loggedAttempt.getFinishTime() - loggedAttempt.getStartTime();
    }
    taskTime = sanitizeTaskRuntime(taskTime, loggedAttempt.getAttemptID());
    taskTime *= scaleFactor;
    return new MapTaskAttemptInfo
      (state, taskInfo, taskTime, loggedAttempt.allSplitVectors());
  } else {
    throw new IllegalArgumentException("taskType can only be MAP: "
        + loggedTask.getTaskType());
  }
}
 

private static Values getPre21Value(String name) {
  if (name.equalsIgnoreCase("JOB_CLEANUP")) {
    return Values.CLEANUP;
  }
  if (name.equalsIgnoreCase("JOB_SETUP")) {
    return Values.SETUP;
  }

  // Note that pre-21, the task state of a successful task was logged as 
  // SUCCESS while from 21 onwards, its logged as SUCCEEDED.
  if (name.equalsIgnoreCase(TaskStatus.State.SUCCEEDED.toString())) {
    return Values.SUCCESS;
  }
  
  return Values.valueOf(StringUtils.toUpperCase(name));
}
 

private static Values getPre21Value(String name) {
  if (name.equalsIgnoreCase("JOB_CLEANUP")) {
    return Values.CLEANUP;
  }
  if (name.equalsIgnoreCase("JOB_SETUP")) {
    return Values.SETUP;
  }

  // Note that pre-21, the task state of a successful task was logged as 
  // SUCCESS while from 21 onwards, its logged as SUCCEEDED.
  if (name.equalsIgnoreCase(TaskStatus.State.SUCCEEDED.toString())) {
    return Values.SUCCESS;
  }
  
  return Values.valueOf(StringUtils.toUpperCase(name));
}
 

@SuppressWarnings("hiding") 
private TaskAttemptInfo scaleInfo(LoggedTask loggedTask,
    LoggedTaskAttempt loggedAttempt, int locality, int loggedLocality,
    double rackLocalOverNodeLocal, double rackRemoteOverNodeLocal) {
  TaskInfo taskInfo = getTaskInfo(loggedTask);
  double[] factors = new double[] { 1.0, rackLocalOverNodeLocal,
      rackRemoteOverNodeLocal };
  double scaleFactor = factors[locality] / factors[loggedLocality];
  State state = convertState(loggedAttempt.getResult());
  if (loggedTask.getTaskType() == Values.MAP) {
    long taskTime = 0;
    if (loggedAttempt.getStartTime() == 0) {
      taskTime = makeUpMapRuntime(state, locality);
    } else {
      taskTime = loggedAttempt.getFinishTime() - loggedAttempt.getStartTime();
    }
    taskTime = sanitizeTaskRuntime(taskTime, loggedAttempt.getAttemptID());
    taskTime *= scaleFactor;
    return new MapTaskAttemptInfo
      (state, taskInfo, taskTime, loggedAttempt.allSplitVectors());
  } else {
    throw new IllegalArgumentException("taskType can only be MAP: "
        + loggedTask.getTaskType());
  }
}
 

@SuppressWarnings("hiding") 
private TaskAttemptInfo scaleInfo(LoggedTask loggedTask,
    LoggedTaskAttempt loggedAttempt, int locality, int loggedLocality,
    double rackLocalOverNodeLocal, double rackRemoteOverNodeLocal) {
  TaskInfo taskInfo = getTaskInfo(loggedTask);
  double[] factors = new double[] { 1.0, rackLocalOverNodeLocal,
      rackRemoteOverNodeLocal };
  double scaleFactor = factors[locality] / factors[loggedLocality];
  State state = convertState(loggedAttempt.getResult());
  if (loggedTask.getTaskType() == Values.MAP) {
    long taskTime = 0;
    if (loggedAttempt.getStartTime() == 0) {
      taskTime = makeUpMapRuntime(state, locality);
    } else {
      taskTime = loggedAttempt.getFinishTime() - loggedAttempt.getStartTime();
    }
    taskTime = sanitizeTaskRuntime(taskTime, loggedAttempt.getAttemptID());
    taskTime *= scaleFactor;
    return new MapTaskAttemptInfo(state, taskInfo, taskTime);
  } else {
    throw new IllegalArgumentException("taskType can only be MAP: "
        + loggedTask.getTaskType());
  }
}
 

private static State convertState(Values status) {
  if (status == Values.SUCCESS) {
    return State.SUCCEEDED;
  } else if (status == Values.FAILED) {
    return State.FAILED;
  } else if (status == Values.KILLED) {
    return State.KILLED;
  } else {
    throw new IllegalArgumentException("unknown status " + status);
  }
}
 

private void processJobFinishedEvent(JobFinishedEvent event) {
  result.setFinishTime(event.getFinishTime());
  result.setJobID(jobID);
  result.setOutcome(Values.SUCCESS);

  JobFinished job = (JobFinished)event.getDatum();
  Map<String, Long> countersMap =
      JobHistoryUtils.extractCounters(job.totalCounters);
  result.putTotalCounters(countersMap);
  countersMap = JobHistoryUtils.extractCounters(job.mapCounters);
  result.putMapCounters(countersMap);
  countersMap = JobHistoryUtils.extractCounters(job.reduceCounters);
  result.putReduceCounters(countersMap);
}
 

private void processJobUnsuccessfulCompletionEvent(
    JobUnsuccessfulCompletionEvent event) {
  result.setOutcome(Pre21JobHistoryConstants.Values
      .valueOf(event.getStatus()));
  result.setFinishTime(event.getFinishTime());
  // No counters in JobUnsuccessfulCompletionEvent
}
 

private static State convertState(Values status) {
  if (status == Values.SUCCESS) {
    return State.SUCCEEDED;
  } else if (status == Values.FAILED) {
    return State.FAILED;
  } else if (status == Values.KILLED) {
    return State.KILLED;
  } else {
    throw new IllegalArgumentException("unknown status " + status);
  }
}
 

private static State convertState(Values status) {
  if (status == Values.SUCCESS) {
    return State.SUCCEEDED;
  } else if (status == Values.FAILED) {
    return State.FAILED;
  } else if (status == Values.KILLED) {
    return State.KILLED;
  } else {
    throw new IllegalArgumentException("unknown status " + status);
  }
}
 

private void processJobFinishedEvent(JobFinishedEvent event) {
  result.setFinishTime(event.getFinishTime());
  result.setJobID(jobID);
  result.setOutcome(Values.SUCCESS);

  JobFinished job = (JobFinished)event.getDatum();
  Map<String, Long> countersMap =
      JobHistoryUtils.extractCounters(job.totalCounters);
  result.putTotalCounters(countersMap);
  countersMap = JobHistoryUtils.extractCounters(job.mapCounters);
  result.putMapCounters(countersMap);
  countersMap = JobHistoryUtils.extractCounters(job.reduceCounters);
  result.putReduceCounters(countersMap);
}
 

private void processJobUnsuccessfulCompletionEvent(
    JobUnsuccessfulCompletionEvent event) {
  result.setOutcome(Pre21JobHistoryConstants.Values
      .valueOf(event.getStatus()));
  result.setFinishTime(event.getFinishTime());
  // No counters in JobUnsuccessfulCompletionEvent
}
 

private JobStory getNextJobFiltered() throws IOException {
  JobStory job;
  do {
    job = jobProducer.getNextJob();
  } while (job != null
      && (job.getOutcome() != Pre21JobHistoryConstants.Values.SUCCESS ||
          job.getSubmissionTime() < 0));
  return null == job ? null : new FilterJobStory(job) {
      @Override
      public TaskInfo getTaskInfo(TaskType taskType, int taskNumber) {
        return new MinTaskInfo(this.job.getTaskInfo(taskType, taskNumber));
      }
    };
}
 

@Override
public Values getOutcome() {
  return job.getOutcome();
}
 

private TaskInfo getTaskInfo(LoggedTask loggedTask) {
  List<LoggedTaskAttempt> attempts = loggedTask.getAttempts();

  long inputBytes = -1;
  long inputRecords = -1;
  long outputBytes = -1;
  long outputRecords = -1;
  long heapMegabytes = -1;

  Values type = loggedTask.getTaskType();
  if ((type != Values.MAP) && (type != Values.REDUCE)) {
    throw new IllegalArgumentException(
        "getTaskInfo only supports MAP or REDUCE tasks: " + type.toString()
            + " for task = " + loggedTask.getTaskID());
  }

  for (LoggedTaskAttempt attempt : attempts) {
    attempt = sanitizeLoggedTaskAttempt(attempt);
    // ignore bad attempts or unsuccessful attempts.
    if ((attempt == null) || (attempt.getResult() != Values.SUCCESS)) {
      continue;
    }

    if (type == Values.MAP) {
      inputBytes = attempt.getHdfsBytesRead();
      inputRecords = attempt.getMapInputRecords();
      outputBytes =
          (job.getTotalReduces() > 0) ? attempt.getMapOutputBytes() : attempt
              .getHdfsBytesWritten();
      outputRecords = attempt.getMapOutputRecords();
      heapMegabytes =
          (job.getJobMapMB() > 0) ? job.getJobMapMB() : job
              .getHeapMegabytes();
    } else {
      inputBytes = attempt.getReduceShuffleBytes();
      inputRecords = attempt.getReduceInputRecords();
      outputBytes = attempt.getHdfsBytesWritten();
      outputRecords = attempt.getReduceOutputRecords();
      heapMegabytes =
          (job.getJobReduceMB() > 0) ? job.getJobReduceMB() : job
              .getHeapMegabytes();
    }
    break;
  }

  TaskInfo taskInfo =
      new TaskInfo(inputBytes, (int) inputRecords, outputBytes,
          (int) outputRecords, (int) heapMegabytes);
  return taskInfo;
}
 

@Override
public Values getOutcome() {
  return Values.SUCCESS;
 }
 

public Values getOutcome() {
  return job.getOutcome();
}
 

protected JobStory getNextJobFiltered() throws IOException {
  JobStory job = getNextJobFromTrace();
  // filter out the following jobs
  //    - unsuccessful jobs
  //    - jobs with missing submit-time
  //    - reduce only jobs
  // These jobs are not yet supported in Gridmix
  while (job != null &&
    (job.getOutcome() != Pre21JobHistoryConstants.Values.SUCCESS ||
      job.getSubmissionTime() < 0 || job.getNumberMaps() == 0)) {
    if (LOG.isDebugEnabled()) {
      List<String> reason = new ArrayList<String>();
      if (job.getOutcome() != Pre21JobHistoryConstants.Values.SUCCESS) {
        reason.add("STATE (" + job.getOutcome().name() + ")");
      }
      if (job.getSubmissionTime() < 0) {
        reason.add("SUBMISSION-TIME (" + job.getSubmissionTime() + ")");
      }
      if (job.getNumberMaps() == 0) {
        reason.add("ZERO-MAPS-JOB");
      }
      
      // TODO This should never happen. Probably we missed something!
      if (reason.size() == 0) {
        reason.add("N/A");
      }
      
      LOG.debug("Ignoring job " + job.getJobID() + " from the input trace."
                + " Reason: " + StringUtils.join(reason, ","));
    }
    job = getNextJobFromTrace();
  }
  return null == job ? null : new FilterJobStory(job) {
    @Override
    public TaskInfo getTaskInfo(TaskType taskType, int taskNumber) {
      TaskInfo info = this.job.getTaskInfo(taskType, taskNumber);
      if (info != null) {
        info = new MinTaskInfo(info);
      } else {
        info = new MinTaskInfo(new TaskInfo(0, 0, 0, 0, 0));
      }
      return info;
    }
  };
}
 

public Values getOutcome() {
  return job.getOutcome();
}
 

@Override
public Values getOutcome() {
  return Values.SUCCESS;
}
 

private void processJobStatusChangedEvent(JobStatusChangedEvent event) {
  result.setOutcome(Pre21JobHistoryConstants.Values
      .valueOf(event.getStatus()));
}
 

private TaskInfo getTaskInfo(LoggedTask loggedTask) {
  if (loggedTask == null) {
    return new TaskInfo(0, 0, 0, 0, 0);
  }
  List<LoggedTaskAttempt> attempts = loggedTask.getAttempts();

  long inputBytes = -1;
  long inputRecords = -1;
  long outputBytes = -1;
  long outputRecords = -1;
  long heapMegabytes = -1;
  ResourceUsageMetrics metrics = new ResourceUsageMetrics();

  Values type = loggedTask.getTaskType();
  if ((type != Values.MAP) && (type != Values.REDUCE)) {
    throw new IllegalArgumentException(
        "getTaskInfo only supports MAP or REDUCE tasks: " + type.toString()
            + " for task = " + loggedTask.getTaskID());
  }

  for (LoggedTaskAttempt attempt : attempts) {
    attempt = sanitizeLoggedTaskAttempt(attempt);
    // ignore bad attempts or unsuccessful attempts.
    if ((attempt == null) || (attempt.getResult() != Values.SUCCESS)) {
      continue;
    }

    if (type == Values.MAP) {
      inputBytes = attempt.getHdfsBytesRead();
      inputRecords = attempt.getMapInputRecords();
      outputBytes =
          (job.getTotalReduces() > 0) ? attempt.getMapOutputBytes() : attempt
              .getHdfsBytesWritten();
      outputRecords = attempt.getMapOutputRecords();
      heapMegabytes =
          (job.getJobMapMB() > 0) ? job.getJobMapMB() : job
              .getHeapMegabytes();
    } else {
      inputBytes = attempt.getReduceShuffleBytes();
      inputRecords = attempt.getReduceInputRecords();
      outputBytes = attempt.getHdfsBytesWritten();
      outputRecords = attempt.getReduceOutputRecords();
      heapMegabytes =
          (job.getJobReduceMB() > 0) ? job.getJobReduceMB() : job
              .getHeapMegabytes();
    }
    // set the resource usage metrics
    metrics = attempt.getResourceUsageMetrics();
    break;
  }

  TaskInfo taskInfo =
      new TaskInfo(inputBytes, (int) inputRecords, outputBytes,
          (int) outputRecords, (int) heapMegabytes,
          metrics);
  return taskInfo;
}
 

@Override
public Values getOutcome() {
  return job.getOutcome();
}
 

@Override
public Values getOutcome() {
  return Values.SUCCESS;
 }
 

protected JobStory getNextJobFiltered() throws IOException {
  JobStory job = getNextJobFromTrace();
  // filter out the following jobs
  //    - unsuccessful jobs
  //    - jobs with missing submit-time
  //    - reduce only jobs
  // These jobs are not yet supported in Gridmix
  while (job != null &&
    (job.getOutcome() != Pre21JobHistoryConstants.Values.SUCCESS ||
      job.getSubmissionTime() < 0 || job.getNumberMaps() == 0)) {
    if (LOG.isDebugEnabled()) {
      List<String> reason = new ArrayList<String>();
      if (job.getOutcome() != Pre21JobHistoryConstants.Values.SUCCESS) {
        reason.add("STATE (" + job.getOutcome().name() + ")");
      }
      if (job.getSubmissionTime() < 0) {
        reason.add("SUBMISSION-TIME (" + job.getSubmissionTime() + ")");
      }
      if (job.getNumberMaps() == 0) {
        reason.add("ZERO-MAPS-JOB");
      }
      
      // TODO This should never happen. Probably we missed something!
      if (reason.size() == 0) {
        reason.add("N/A");
      }
      
      LOG.debug("Ignoring job " + job.getJobID() + " from the input trace."
                + " Reason: " + StringUtils.join(reason, ","));
    }
    job = getNextJobFromTrace();
  }
  return null == job ? null : new FilterJobStory(job) {
    @Override
    public TaskInfo getTaskInfo(TaskType taskType, int taskNumber) {
      TaskInfo info = this.job.getTaskInfo(taskType, taskNumber);
      if (info != null) {
        info = new MinTaskInfo(info);
      } else {
        info = new MinTaskInfo(new TaskInfo(0, 0, 0, 0, 0));
      }
      return info;
    }
  };
}
 

public Values getOutcome() {
  return job.getOutcome();
}
 

private void processJobStatusChangedEvent(JobStatusChangedEvent event) {
  result.setOutcome(Pre21JobHistoryConstants.Values
      .valueOf(event.getStatus()));
}
 

private TaskInfo getTaskInfo(LoggedTask loggedTask) {
  if (loggedTask == null) {
    return new TaskInfo(0, 0, 0, 0, 0);
  }
  List<LoggedTaskAttempt> attempts = loggedTask.getAttempts();

  long inputBytes = -1;
  long inputRecords = -1;
  long outputBytes = -1;
  long outputRecords = -1;
  long heapMegabytes = -1;
  ResourceUsageMetrics metrics = new ResourceUsageMetrics();

  Values type = loggedTask.getTaskType();
  if ((type != Values.MAP) && (type != Values.REDUCE)) {
    throw new IllegalArgumentException(
        "getTaskInfo only supports MAP or REDUCE tasks: " + type.toString()
            + " for task = " + loggedTask.getTaskID());
  }

  for (LoggedTaskAttempt attempt : attempts) {
    attempt = sanitizeLoggedTaskAttempt(attempt);
    // ignore bad attempts or unsuccessful attempts.
    if ((attempt == null) || (attempt.getResult() != Values.SUCCESS)) {
      continue;
    }

    if (type == Values.MAP) {
      inputBytes = attempt.getHdfsBytesRead();
      inputRecords = attempt.getMapInputRecords();
      outputBytes =
          (job.getTotalReduces() > 0) ? attempt.getMapOutputBytes() : attempt
              .getHdfsBytesWritten();
      outputRecords = attempt.getMapOutputRecords();
      heapMegabytes =
          (job.getJobMapMB() > 0) ? job.getJobMapMB() : job
              .getHeapMegabytes();
    } else {
      inputBytes = attempt.getReduceShuffleBytes();
      inputRecords = attempt.getReduceInputRecords();
      outputBytes = attempt.getHdfsBytesWritten();
      outputRecords = attempt.getReduceOutputRecords();
      heapMegabytes =
          (job.getJobReduceMB() > 0) ? job.getJobReduceMB() : job
              .getHeapMegabytes();
    }
    // set the resource usage metrics
    metrics = attempt.getResourceUsageMetrics();
    break;
  }

  TaskInfo taskInfo =
      new TaskInfo(inputBytes, (int) inputRecords, outputBytes,
          (int) outputRecords, (int) heapMegabytes,
          metrics);
  return taskInfo;
}
 

@Override
public Values getOutcome() {
  return job.getOutcome();
}
 

/**
 * Get the outcome of the job execution.
 * @return The outcome of the job execution.
 */
public Values getOutcome();