Java Code Examples for org.apache.hadoop.mapreduce.TaskAttemptID#getId()

public void map(Object key, Text value, Context context)
        throws IOException, InterruptedException {
  // Make one mapper slower for speculative execution
  TaskAttemptID taid = context.getTaskAttemptID();
  long sleepTime = 100;
  Configuration conf = context.getConfiguration();
  boolean test_speculate_map =
          conf.getBoolean(MRJobConfig.MAP_SPECULATIVE, false);

  // IF TESTING MAPPER SPECULATIVE EXECUTION:
  //   Make the "*_m_000000_0" attempt take much longer than the others.
  //   When speculative execution is enabled, this should cause the attempt
  //   to be killed and restarted. At that point, the attempt ID will be
  //   "*_m_000000_1", so sleepTime will still remain 100ms.
  if ( (taid.getTaskType() == TaskType.MAP) && test_speculate_map
        && (taid.getTaskID().getId() == 0) && (taid.getId() == 0)) {
    sleepTime = 10000;
  }
  try{
    Thread.sleep(sleepTime);
  } catch(InterruptedException ie) {
    // Ignore
  }
  context.write(value, new IntWritable(1));
}
 

public void reduce(Text key, Iterable<IntWritable> values, 
                       Context context) throws IOException, InterruptedException {
  // Make one reducer slower for speculative execution
  TaskAttemptID taid = context.getTaskAttemptID();
  long sleepTime = 100;
  Configuration conf = context.getConfiguration();
  boolean test_speculate_reduce =
            conf.getBoolean(MRJobConfig.REDUCE_SPECULATIVE, false);

  // IF TESTING REDUCE SPECULATIVE EXECUTION:
  //   Make the "*_r_000000_0" attempt take much longer than the others.
  //   When speculative execution is enabled, this should cause the attempt
  //   to be killed and restarted. At that point, the attempt ID will be
  //   "*_r_000000_1", so sleepTime will still remain 100ms.
  if ( (taid.getTaskType() == TaskType.REDUCE) && test_speculate_reduce
        && (taid.getTaskID().getId() == 0) && (taid.getId() == 0)) {
    sleepTime = 10000;
  }
  try{
    Thread.sleep(sleepTime);
  } catch(InterruptedException ie) {
    // Ignore
  }
  context.write(key,new IntWritable(0));
}
 

private static Path writeOutputFile(TaskAttemptID id, Path dest,
                                    String content, long copies)
    throws IOException {
  String fileName = ((id.getTaskType() == TaskType.REDUCE) ? "r_" : "m_") +
      id.getTaskID().getId() + "_" + id.getId() + "_" +
      UUID.randomUUID().toString();
  Path outPath = new Path(dest, fileName);
  FileSystem fs = outPath.getFileSystem(getConfiguration());

  try (OutputStream out = fs.create(outPath)) {
    byte[] bytes = content.getBytes(StandardCharsets.UTF_8);
    for (int i = 0; i < copies; i += 1) {
      out.write(bytes);
    }
  }

  return outPath;
}
 

public void map(Object key, Text value, Context context)
        throws IOException, InterruptedException {
  // Make one mapper slower for speculative execution
  TaskAttemptID taid = context.getTaskAttemptID();
  long sleepTime = 100;
  Configuration conf = context.getConfiguration();
  boolean test_speculate_map =
          conf.getBoolean(MRJobConfig.MAP_SPECULATIVE, false);

  // IF TESTING MAPPER SPECULATIVE EXECUTION:
  //   Make the "*_m_000000_0" attempt take much longer than the others.
  //   When speculative execution is enabled, this should cause the attempt
  //   to be killed and restarted. At that point, the attempt ID will be
  //   "*_m_000000_1", so sleepTime will still remain 100ms.
  if ( (taid.getTaskType() == TaskType.MAP) && test_speculate_map
        && (taid.getTaskID().getId() == 0) && (taid.getId() == 0)) {
    sleepTime = 10000;
  }
  try{
    Thread.sleep(sleepTime);
  } catch(InterruptedException ie) {
    // Ignore
  }
  context.write(value, new IntWritable(1));
}
 

public void reduce(Text key, Iterable<IntWritable> values, 
                       Context context) throws IOException, InterruptedException {
  // Make one reducer slower for speculative execution
  TaskAttemptID taid = context.getTaskAttemptID();
  long sleepTime = 100;
  Configuration conf = context.getConfiguration();
  boolean test_speculate_reduce =
            conf.getBoolean(MRJobConfig.REDUCE_SPECULATIVE, false);

  // IF TESTING REDUCE SPECULATIVE EXECUTION:
  //   Make the "*_r_000000_0" attempt take much longer than the others.
  //   When speculative execution is enabled, this should cause the attempt
  //   to be killed and restarted. At that point, the attempt ID will be
  //   "*_r_000000_1", so sleepTime will still remain 100ms.
  if ( (taid.getTaskType() == TaskType.REDUCE) && test_speculate_reduce
        && (taid.getTaskID().getId() == 0) && (taid.getId() == 0)) {
    sleepTime = 10000;
  }
  try{
    Thread.sleep(sleepTime);
  } catch(InterruptedException ie) {
    // Ignore
  }
  context.write(key,new IntWritable(0));
}
 

public static String getLocalLogDir(TaskAttemptID attemptId) {
    int tid = attemptId.getTaskID().getId();
    int aid = attemptId.getId();
    String jid = attemptId.getJobID().toString();
    StringBuilder sb = new StringBuilder(jid).append('-');
    sb.append(tid).append('-').append(aid);
    String localLogDir = sb.toString();
    return localLogDir;
}
 

@Override
public void cleanUpPartialOutputForTask(TaskAttemptContext context)
    throws IOException {

  // we double check this is never invoked from a non-preemptable subclass.
  // This should never happen, since the invoking codes is checking it too,
  // but it is safer to double check. Errors handling this would produce
  // inconsistent output.

  if (!this.getClass().isAnnotationPresent(Checkpointable.class)) {
    throw new IllegalStateException("Invoking cleanUpPartialOutputForTask() " +
        "from non @Preemptable class");
  }
  FileSystem fs =
    fsFor(getTaskAttemptPath(context), context.getConfiguration());

  LOG.info("cleanUpPartialOutputForTask: removing everything belonging to " +
      context.getTaskAttemptID().getTaskID() + " in: " +
      getCommittedTaskPath(context).getParent());

  final TaskAttemptID taid = context.getTaskAttemptID();
  final TaskID tid = taid.getTaskID();
  Path pCommit = getCommittedTaskPath(context).getParent();
  // remove any committed output
  for (int i = 0; i < taid.getId(); ++i) {
    TaskAttemptID oldId = new TaskAttemptID(tid, i);
    Path pTask = new Path(pCommit, oldId.toString());
    if (fs.exists(pTask) && !fs.delete(pTask, true)) {
      throw new IOException("Failed to delete " + pTask);
    }
  }
}
 

/**
 * Mask the job ID part in a {@link TaskAttemptID}.
 * 
 * @param attemptId
 *          raw {@link TaskAttemptID} read from trace
 * @return masked {@link TaskAttemptID} with empty {@link JobID}.
 */
private TaskAttemptID maskAttemptID(TaskAttemptID attemptId) {
  JobID jobId = new JobID();
  TaskType taskType = attemptId.getTaskType();
  TaskID taskId = attemptId.getTaskID();
  return new TaskAttemptID(jobId.getJtIdentifier(), jobId.getId(), taskType,
      taskId.getId(), attemptId.getId());
}
 

@Override
public void cleanUpPartialOutputForTask(TaskAttemptContext context)
    throws IOException {

  // we double check this is never invoked from a non-preemptable subclass.
  // This should never happen, since the invoking codes is checking it too,
  // but it is safer to double check. Errors handling this would produce
  // inconsistent output.

  if (!this.getClass().isAnnotationPresent(Checkpointable.class)) {
    throw new IllegalStateException("Invoking cleanUpPartialOutputForTask() " +
        "from non @Preemptable class");
  }
  FileSystem fs =
    fsFor(getTaskAttemptPath(context), context.getConfiguration());

  LOG.info("cleanUpPartialOutputForTask: removing everything belonging to " +
      context.getTaskAttemptID().getTaskID() + " in: " +
      getCommittedTaskPath(context).getParent());

  final TaskAttemptID taid = context.getTaskAttemptID();
  final TaskID tid = taid.getTaskID();
  Path pCommit = getCommittedTaskPath(context).getParent();
  // remove any committed output
  for (int i = 0; i < taid.getId(); ++i) {
    TaskAttemptID oldId = new TaskAttemptID(tid, i);
    Path pTask = new Path(pCommit, oldId.toString());
    if (fs.exists(pTask) && !fs.delete(pTask, true)) {
      throw new IOException("Failed to delete " + pTask);
    }
  }
}
 

/**
 * Mask the job ID part in a {@link TaskAttemptID}.
 * 
 * @param attemptId
 *          raw {@link TaskAttemptID} read from trace
 * @return masked {@link TaskAttemptID} with empty {@link JobID}.
 */
private TaskAttemptID maskAttemptID(TaskAttemptID attemptId) {
  JobID jobId = new JobID();
  TaskType taskType = attemptId.getTaskType();
  TaskID taskId = attemptId.getTaskID();
  return new TaskAttemptID(jobId.getJtIdentifier(), jobId.getId(), taskType,
      taskId.getId(), attemptId.getId());
}
 

protected void setup(Context context)
  throws IOException, InterruptedException {
  Configuration conf = context.getConfiguration();
  this.mapSleepCount =
    conf.getInt(MAP_SLEEP_COUNT, mapSleepCount);
  this.mapSleepDuration = mapSleepCount == 0 ? 0 :
    conf.getLong(MAP_SLEEP_TIME , 100) / mapSleepCount;
  vertexName = conf.get(
      org.apache.tez.mapreduce.hadoop.MRJobConfig.VERTEX_NAME);

  TaskAttemptID taId = context.getTaskAttemptID();

  String[] taskIds = conf.getStrings(MAP_ERROR_TASK_IDS);
  if (taId.getId()+1 >= context.getMaxMapAttempts()) {
    finalAttempt = true;
  }
  boolean found = false;
  if (taskIds != null) {
    if (taskIds.length == 1 && taskIds[0].equals("*")) {
      found = true;
    }
    if (!found) {
      for (String taskId : taskIds) {
        if (Integer.parseInt(taskId) ==
            taId.getTaskID().getId()) {
          found = true;
          break;
        }
      }
    }
  }
  if (found) {
    if (!finalAttempt) {
      throwError = conf.getBoolean(MAP_THROW_ERROR, false);
    }
    throwFatal = conf.getBoolean(MAP_FATAL_ERROR, false);
  }
}
 

protected void setup(Context context)
  throws IOException, InterruptedException {
  Configuration conf = context.getConfiguration();
  this.mapSleepCount =
    conf.getInt(MAP_SLEEP_COUNT, mapSleepCount);
  this.mapSleepDuration = mapSleepCount == 0 ? 0 :
    conf.getLong(MAP_SLEEP_TIME , 100) / mapSleepCount;
  vertexName = conf.get(
      org.apache.tez.mapreduce.hadoop.MRJobConfig.VERTEX_NAME);

  TaskAttemptID taId = context.getTaskAttemptID();

  ObjectRegistry objectRegistry = ObjectRegistryFactory.getObjectRegistry();
  String fooBarVal = (String) objectRegistry.get("FooBar");
  if (null == fooBarVal) {
    LOG.info("Adding FooBar key to Object cache");
    objectRegistry.add(ObjectLifeCycle.DAG,
        "FooBar", "BarFooFromTask" + taId.getTaskID().toString());
  } else {
    LOG.info("Got FooBar val from Object cache"
        + ", currentTaskId=" + taId.getTaskID().toString()
        + ", val=" + fooBarVal);
  }

  String[] taskIds = conf.getStrings(MAP_ERROR_TASK_IDS);
  if (taId.getId()+1 >= context.getMaxMapAttempts()) {
    finalAttempt = true;
  }
  boolean found = false;
  if (taskIds != null) {
    if (taskIds.length == 1 && taskIds[0].equals("*")) {
      found = true;
    }
    if (!found) {
      for (String taskId : taskIds) {
        if (Integer.valueOf(taskId).intValue() ==
            taId.getTaskID().getId()) {
          found = true;
          break;
        }
      }
    }
  }
  if (found) {
    if (!finalAttempt) {
      throwError = conf.getBoolean(MAP_THROW_ERROR, false);
    }
    throwFatal = conf.getBoolean(MAP_FATAL_ERROR, false);
  }
}
 

/**
 * Mask the job ID part in a {@link TaskAttemptID}.
 * 
 * @param attemptId
 *          raw {@link TaskAttemptID} read from trace
 * @return masked {@link TaskAttemptID} with empty {@link JobID}.
 */
private TaskAttemptID maskAttemptID(TaskAttemptID attemptId) {
  JobID jobId = new JobID();
  TaskID taskId = attemptId.getTaskID();
  return new TaskAttemptID(jobId.getJtIdentifier(), jobId.getId(),
      attemptId.isMap(), taskId.getId(), attemptId.getId());
}