Java Code Examples for org.apache.hadoop.mapreduce.TaskID#getId()

/**
 * Generate a unique filename, based on the task id, name, and extension
 * @param context the task that is calling this
 * @param name the base filename
 * @param extension the filename extension
 * @return a string like $name-[mrsct]-$id$extension
 */
public synchronized static String getUniqueFile(TaskAttemptContext context,
                                                String name,
                                                String extension) {
  TaskID taskId = context.getTaskAttemptID().getTaskID();
  int partition = taskId.getId();
  StringBuilder result = new StringBuilder();
  result.append(name);
  result.append('-');
  result.append(
      TaskID.getRepresentingCharacter(taskId.getTaskType()));
  result.append('-');
  result.append(NUMBER_FORMAT.format(partition));
  result.append(extension);
  return result.toString();
}
 

/**
 * Generate a unique filename, based on the task id, name, and extension
 * @param context the task that is calling this
 * @param name the base filename
 * @param extension the filename extension
 * @return a string like $name-[mrsct]-$id$extension
 */
public synchronized static String getUniqueFile(TaskAttemptContext context,
                                                String name,
                                                String extension) {
  TaskID taskId = context.getTaskAttemptID().getTaskID();
  int partition = taskId.getId();
  StringBuilder result = new StringBuilder();
  result.append(name);
  result.append('-');
  result.append(
      TaskID.getRepresentingCharacter(taskId.getTaskType()));
  result.append('-');
  result.append(NUMBER_FORMAT.format(partition));
  result.append(extension);
  return result.toString();
}
 

public void write(K key, V value) throws IOException, InterruptedException {
    TaskID taskID = job.getTaskAttemptID().getTaskID();
    int partition = taskID.getId();
    String baseName = getFileBaseName(key, NUMBER_FORMAT.format(partition));
    RecordWriter<K, V> rw = this.recordWriter.get(baseName);
    if (rw == null) {
        rw = getBaseRecordWriter(job, baseName);
        this.recordWriter.put(baseName, rw);
    }
    rw.write(null, value);
}
 

/**
 * Generate a unique filename, based on the task id, name, and extension
 * @param context the task that is calling this
 * @param name the base filename
 * @param extension the filename extension
 * @return a string like $name-[jobType]-$id$extension
 */
protected synchronized String getCustomFileName(TaskAttemptContext context,
        String name,
        String extension) {
    TaskID taskId = context.getTaskAttemptID().getTaskID();
    int partition = taskId.getId();
    return name + '-' + NUMBER_FORMAT.format(partition) + extension;
}
 

/**
 * Generate a unique filename, based on the task id, name, and extension
 * @param context the task that is calling this
 * @param name the base filename
 * @param extension the filename extension
 * @return a string like $name-[mr]-$id$extension
 */
public synchronized static String getUniqueFile(TaskAttemptContext context,
                                                String name,
                                                String extension) {
  TaskID taskId = context.getTaskAttemptID().getTaskID();
  int partition = taskId.getId();
  StringBuilder result = new StringBuilder();
  result.append(name);
  result.append('-');
  result.append(taskId.isMap() ? 'm' : 'r');
  result.append('-');
  result.append(NUMBER_FORMAT.format(partition));
  result.append(extension);
  return result.toString();
}
 

@Override
public synchronized String getCustomFileName(TaskAttemptContext context, String name, String extension) {
    TaskID taskId = context.getTaskAttemptID().getTaskID();
    int partition = taskId.getId();
    String sstableVersion = context.getConfiguration().get(Aegisthus.Feature.CONF_SSTABLE_VERSION);
    return context.getConfiguration().get(Aegisthus.Feature.CONF_DATASET, "keyspace-dataset")
            + "-" + sstableVersion
            + "-" + NUMBER_FORMAT.format(partition)
            + "-Data.db";
}
 

/**
 * Generate a unique filename, based on the task id, name, and extension
 * @param context the task that is calling this
 * @param name the base filename
 * @param extension the filename extension
 * @return a string like $name-[mr]-$id$extension
 */
public synchronized static String getUniqueFile(TaskAttemptContext context,
                                                String name,
                                                String extension) {
  TaskID taskId = context.getTaskAttemptID().getTaskID();
  int partition = taskId.getId();
  StringBuilder result = new StringBuilder();
  result.append(name);
  result.append('-');
  result.append(taskId.isMap() ? 'm' : 'r');
  result.append('-');
  result.append(NUMBER_FORMAT.format(partition));
  result.append(extension);
  return result.toString();
}
 

@ProcessElement
public void processElement(
    @Element KV<KeyT, ValueT> element,
    OutputReceiver<KV<Integer, KV<KeyT, ValueT>>> receiver,
    ProcessContext c) {

  Configuration config = c.sideInput(configView);

  TaskID taskID = createTaskIDForKV(element, config);
  int taskId = taskID.getId();
  receiver.output(KV.of(taskId, element));
}
 

protected String getUniqueName(String name, String extension) {
  int partition;
  
  NumberFormat numberFormat = NumberFormat.getInstance();
  numberFormat.setMinimumIntegerDigits(5);
  numberFormat.setGroupingUsed(false);
  
  if (null != getTaskAttemptContext()) {
    TaskID taskId = getTaskAttemptContext().getTaskAttemptID().getTaskID();
    partition = taskId.getId();
  } else {
    partition = getConfiguration().getInt(JobContext.TASK_PARTITION, -1);
  } 
  if (partition == -1) {
    throw new IllegalArgumentException("This method can only be called from an application");
  }
  
  String taskType = getConfiguration().getBoolean(JobContext.TASK_ISMAP, JobContext.DEFAULT_TASK_ISMAP) ? "m" : "r";
  
  StringBuilder result = new StringBuilder();
  result.append(name);
  result.append('-');
  result.append(taskType);
  result.append('-');
  result.append(numberFormat.format(partition));
  result.append(extension);
  return result.toString();
  
}
 

/**
 * Mask the job ID part in a {@link TaskAttemptID}.
 * 
 * @param attemptId
 *          raw {@link TaskAttemptID} read from trace
 * @return masked {@link TaskAttemptID} with empty {@link JobID}.
 */
private TaskAttemptID maskAttemptID(TaskAttemptID attemptId) {
  JobID jobId = new JobID();
  TaskType taskType = attemptId.getTaskType();
  TaskID taskId = attemptId.getTaskID();
  return new TaskAttemptID(jobId.getJtIdentifier(), jobId.getId(), taskType,
      taskId.getId(), attemptId.getId());
}
 

public synchronized void tipFailed(TaskID taskId) {
  if (!finishedMaps[taskId.getId()]) {
    finishedMaps[taskId.getId()] = true;
    if (--remainingMaps == 0) {
      notifyAll();
    }
    updateStatus();
  }
}
 

/**
 * Mask the job ID part in a {@link TaskAttemptID}.
 * 
 * @param attemptId
 *          raw {@link TaskAttemptID} read from trace
 * @return masked {@link TaskAttemptID} with empty {@link JobID}.
 */
private TaskAttemptID maskAttemptID(TaskAttemptID attemptId) {
  JobID jobId = new JobID();
  TaskType taskType = attemptId.getTaskType();
  TaskID taskId = attemptId.getTaskID();
  return new TaskAttemptID(jobId.getJtIdentifier(), jobId.getId(), taskType,
      taskId.getId(), attemptId.getId());
}
 

public synchronized void tipFailed(TaskID taskId) {
  if (!finishedMaps[taskId.getId()]) {
    finishedMaps[taskId.getId()] = true;
    if (--remainingMaps == 0) {
      notifyAll();
    }
    updateStatus();
  }
}
 

public void write(K key, V value) throws IOException, InterruptedException {
    TaskID taskID = job.getTaskAttemptID().getTaskID();
    int partition = taskID.getId();
    String baseName = getFileBaseName(key, NUMBER_FORMAT.format(partition));
    RecordWriter<K, V> rw = this.recordWriter.get(baseName);
    if (rw == null) {
        rw = getBaseRecordWriter(job, baseName);
        this.recordWriter.put(baseName, rw);
    }
    rw.write(null, value);
}
 

public void write(K key, V value) throws IOException, InterruptedException {
    TaskID taskID = job.getTaskAttemptID().getTaskID();
    int partition = taskID.getId();
    String baseName = getFileBaseName(key, NUMBER_FORMAT.format(partition));
    RecordWriter<K, V> rw = this.recordWriter.get(baseName);
    if (rw == null) {
        rw = getBaseRecordWriter(job, baseName);
        this.recordWriter.put(baseName, rw);
    }
    rw.write(null, value);
}
 

public void write(K key, V value) throws IOException, InterruptedException {
    TaskID taskID = job.getTaskAttemptID().getTaskID();
    int partition = taskID.getId();
    String baseName = getFileBaseName(key, NUMBER_FORMAT.format(partition));
    RecordWriter<K, V> rw = this.recordWriter.get(baseName);
    if (rw == null) {
        rw = getBaseRecordWriter(job, baseName);
        this.recordWriter.put(baseName, rw);
    }
    rw.write(null, value);
}
 

/**
 * Mask the job ID part in a {@link TaskAttemptID}.
 * 
 * @param attemptId
 *          raw {@link TaskAttemptID} read from trace
 * @return masked {@link TaskAttemptID} with empty {@link JobID}.
 */
private TaskAttemptID maskAttemptID(TaskAttemptID attemptId) {
  JobID jobId = new JobID();
  TaskID taskId = attemptId.getTaskID();
  return new TaskAttemptID(jobId.getJtIdentifier(), jobId.getId(),
      attemptId.isMap(), taskId.getId(), attemptId.getId());
}
 

/**
 * Mask the job ID part in a {@link TaskID}.
 * 
 * @param taskId
 *          raw {@link TaskID} read from trace
 * @return masked {@link TaskID} with empty {@link JobID}.
 */
private TaskID maskTaskID(TaskID taskId) {
  JobID jobId = new JobID();
  TaskType taskType = taskId.getTaskType();
  return new TaskID(jobId, taskType, taskId.getId());
}
 

/**
 * Mask the job ID part in a {@link TaskID}.
 * 
 * @param taskId
 *          raw {@link TaskID} read from trace
 * @return masked {@link TaskID} with empty {@link JobID}.
 */
private TaskID maskTaskID(TaskID taskId) {
  JobID jobId = new JobID();
  TaskType taskType = taskId.getTaskType();
  return new TaskID(jobId, taskType, taskId.getId());
}
 

/**
 * Mask the job ID part in a {@link TaskID}.
 * 
 * @param taskId
 *          raw {@link TaskID} read from trace
 * @return masked {@link TaskID} with empty {@link JobID}.
 */
private TaskID maskTaskID(TaskID taskId) {
  JobID jobId = new JobID();
  return new TaskID(jobId, taskId.isMap(), taskId.getId());
}