Java Code Examples for org.apache.hadoop.mapreduce.TaskAttemptID#forName()

@Before
@SuppressWarnings("unchecked") // mocked generics
public void setup() {
  LOG.info(">>>> " + name.getMethodName());
  job = new JobConf();
  job.setBoolean(MRJobConfig.SHUFFLE_FETCH_RETRY_ENABLED, false);
  jobWithRetry = new JobConf();
  jobWithRetry.setBoolean(MRJobConfig.SHUFFLE_FETCH_RETRY_ENABLED, true);
  id = TaskAttemptID.forName("attempt_0_1_r_1_1");
  ss = mock(ShuffleSchedulerImpl.class);
  mm = mock(MergeManagerImpl.class);
  r = mock(Reporter.class);
  metrics = mock(ShuffleClientMetrics.class);
  except = mock(ExceptionReporter.class);
  key = JobTokenSecretManager.createSecretKey(new byte[]{0,0,0,0});
  connection = mock(HttpURLConnection.class);

  allErrs = mock(Counters.Counter.class);
  when(r.getCounter(anyString(), anyString())).thenReturn(allErrs);

  ArrayList<TaskAttemptID> maps = new ArrayList<TaskAttemptID>(1);
  maps.add(map1ID);
  maps.add(map2ID);
  when(ss.getMapsForHost(host)).thenReturn(maps);
}
 

/**
 * Accepts any object name.
 * If object name of the form
 * a/b/c/gil.data/part-r-00000-48ae3461-203f-4dd3-b141-a45426e2d26c
 *    -attempt_20160317132a_wrong_0000_m_000000_1.csv
 * Then a/b/c/gil.data is returned.
 * Code testing that attempt_20160317132a_wrong_0000_m_000000_1 is valid
 * task id identifier
 *
 * @param objectName
 * @return unified object name
 */
private String extractUnifiedObjectName(String objectName) {
  Path p = new Path(objectName);
  int attemptIndex = objectName.indexOf(HADOOP_ATTEMPT);
  int dotIndex = objectName.lastIndexOf('.');
  if (attemptIndex >= 0 && dotIndex > attemptIndex) {
    String attempt = objectName.substring(attemptIndex, dotIndex);
    try {
      TaskAttemptID.forName(attempt);
      return p.getParent().toString();
    } catch (IllegalArgumentException e) {
      return objectName;
    }
  } else if (objectName.indexOf(HADOOP_SUCCESS) > 0) {
    return p.getParent().toString();
  }
  return objectName;
}
 

public void setDatum(Object odatum) {
  this.datum =
      (TaskAttemptUnsuccessfulCompletion)odatum;
  this.attemptId =
      TaskAttemptID.forName(datum.attemptId.toString());
  this.taskType =
      TaskType.valueOf(datum.taskType.toString());
  this.finishTime = datum.finishTime;
  this.hostname = datum.hostname.toString();
  this.rackName = datum.rackname.toString();
  this.port = datum.port;
  this.status = datum.status.toString();
  this.error = datum.error.toString();
  this.counters =
      EventReader.fromAvro(datum.counters);
  this.clockSplits =
      AvroArrayUtils.fromAvro(datum.clockSplits);
  this.cpuUsages =
      AvroArrayUtils.fromAvro(datum.cpuUsages);
  this.vMemKbytes =
      AvroArrayUtils.fromAvro(datum.vMemKbytes);
  this.physMemKbytes =
      AvroArrayUtils.fromAvro(datum.physMemKbytes);
}
 

public void setDatum(Object odatum) {
  this.datum =
      (TaskAttemptUnsuccessfulCompletion)odatum;
  this.attemptId =
      TaskAttemptID.forName(datum.attemptId.toString());
  this.taskType =
      TaskType.valueOf(datum.taskType.toString());
  this.finishTime = datum.finishTime;
  this.hostname = datum.hostname.toString();
  this.rackName = datum.rackname.toString();
  this.port = datum.port;
  this.status = datum.status.toString();
  this.error = datum.error.toString();
  this.counters =
      EventReader.fromAvro(datum.counters);
  this.clockSplits =
      AvroArrayUtils.fromAvro(datum.clockSplits);
  this.cpuUsages =
      AvroArrayUtils.fromAvro(datum.cpuUsages);
  this.gpuUsages =
      AvroArrayUtils.fromAvro(datum.gpuUsages);
  this.vMemKbytes =
      AvroArrayUtils.fromAvro(datum.vMemKbytes);
  this.physMemKbytes =
      AvroArrayUtils.fromAvro(datum.physMemKbytes);
}
 

/**
 * Accepts any object name. If object name is of the form
 * a/b/c/m.data/part-r-00000-48ae3461-203f-4dd3-b141-a45426e2d26c
 * .csv-attempt_20160317132wrong_0000_m_000000_1 Then
 * a/b/c/m.data/part-r-00000-48ae3461-203f-4dd3-b141-a45426e2d26c.csv is
 * returned. Perform test that attempt_20160317132wrong_0000_m_000000_1 is
 * valid task id identifier
 *
 * @param objectKey object key
 * @return unified object name
 */
public String nameWithoutTaskID(String objectKey) {
  int index = objectKey.indexOf("-" + HADOOP_ATTEMPT);
  if (index > 0) {
    String attempt = objectKey.substring(index + 1);
    try {
      if (attempt.indexOf(".") > 0) {
        attempt = attempt.substring(0, attempt.indexOf("."));
      }
      TaskAttemptID.forName(attempt);
      String res = objectKey.replace("-" + attempt, "");
      return res;
    } catch (IllegalArgumentException e) {
      return objectKey;
    }
  }
  return objectKey;
}
 

public void setDatum(Object oDatum) {
  this.datum = (MapAttemptFinished)oDatum;
  this.attemptId = TaskAttemptID.forName(datum.attemptId.toString());
  this.taskType = TaskType.valueOf(datum.taskType.toString());
  this.taskStatus = datum.taskStatus.toString();
  this.mapFinishTime = datum.mapFinishTime;
  this.finishTime = datum.finishTime;
  this.hostname = datum.hostname.toString();
  this.rackName = datum.rackname.toString();
  this.port = datum.port;
  this.state = datum.state.toString();
  this.counters = EventReader.fromAvro(datum.counters);
  this.clockSplits = AvroArrayUtils.fromAvro(datum.clockSplits);
  this.cpuUsages = AvroArrayUtils.fromAvro(datum.cpuUsages);
  this.gpuUsages = AvroArrayUtils.fromAvro(datum.gpuUsages);
  this.vMemKbytes = AvroArrayUtils.fromAvro(datum.vMemKbytes);
  this.physMemKbytes = AvroArrayUtils.fromAvro(datum.physMemKbytes);
}
 

public void setDatum(Object oDatum) {
  this.datum = (ReduceAttemptFinished)oDatum;
  this.attemptId = TaskAttemptID.forName(datum.attemptId.toString());
  this.taskType = TaskType.valueOf(datum.taskType.toString());
  this.taskStatus = datum.taskStatus.toString();
  this.shuffleFinishTime = datum.shuffleFinishTime;
  this.sortFinishTime = datum.sortFinishTime;
  this.finishTime = datum.finishTime;
  this.hostname = datum.hostname.toString();
  this.rackName = datum.rackname.toString();
  this.port = datum.port;
  this.state = datum.state.toString();
  this.counters = EventReader.fromAvro(datum.counters);
  this.clockSplits = AvroArrayUtils.fromAvro(datum.clockSplits);
  this.cpuUsages = AvroArrayUtils.fromAvro(datum.cpuUsages);
  this.vMemKbytes = AvroArrayUtils.fromAvro(datum.vMemKbytes);
  this.physMemKbytes = AvroArrayUtils.fromAvro(datum.physMemKbytes);
}
 

public RecordReader<Long, Row> getRecordReader(InputSplit split, JobConf jobConf, final Reporter reporter)
        throws IOException
{
    TaskAttemptContext tac = new TaskAttemptContext(jobConf, TaskAttemptID.forName(jobConf.get(MAPRED_TASK_ID)))
    {
        @Override
        public void progress()
        {
            reporter.progress();
        }
    };

    CqlRecordReader recordReader = new CqlRecordReader();
    recordReader.initialize((org.apache.hadoop.mapreduce.InputSplit)split, tac);
    return recordReader;
}
 

public void setDatum(Object oDatum) {
  this.datum = (MapAttemptFinished)oDatum;
  this.attemptId = TaskAttemptID.forName(datum.attemptId.toString());
  this.taskType = TaskType.valueOf(datum.taskType.toString());
  this.taskStatus = datum.taskStatus.toString();
  this.mapFinishTime = datum.mapFinishTime;
  this.finishTime = datum.finishTime;
  this.hostname = datum.hostname.toString();
  this.rackName = datum.rackname.toString();
  this.port = datum.port;
  this.state = datum.state.toString();
  this.counters = EventReader.fromAvro(datum.counters);
  this.clockSplits = AvroArrayUtils.fromAvro(datum.clockSplits);
  this.cpuUsages = AvroArrayUtils.fromAvro(datum.cpuUsages);
  this.vMemKbytes = AvroArrayUtils.fromAvro(datum.vMemKbytes);
  this.physMemKbytes = AvroArrayUtils.fromAvro(datum.physMemKbytes);
}
 

@VisibleForTesting
List<Map.Entry> previewTextBatch(FileStatus fileStatus, int batchSize)
    throws IOException, InterruptedException {
  int previewCount = previewBuffer.size();
  TextInputFormat textInputFormat = new TextInputFormat();
  long fileLength = fileStatus.getLen();
  List<Map.Entry> batch = new ArrayList<>();
  Path filePath = fileStatus.getPath();
  // MR allows file length to be 0 for text data (not for avro)
  if (fileLength == 0) {
    LOG.info("File length is 0 for {}", filePath);
    return batch;
  }
  // Hadoop does unsafe casting from long to int, so split length should not be greater than int
  // max value
  long splitLength = (fileLength < Integer.MAX_VALUE) ? fileLength: Integer.MAX_VALUE;
  InputSplit fileSplit = new FileSplit(filePath, 0, splitLength, null);
  TaskAttemptContext taskAttemptContext = new TaskAttemptContextImpl(hadoopConf,
      TaskAttemptID.forName("attempt_1439420318532_0011_m_000000_0"));

  try (RecordReader<LongWritable, Text> recordReader = textInputFormat.createRecordReader(fileSplit, taskAttemptContext)) {
    recordReader.initialize(fileSplit, taskAttemptContext);
    boolean hasNext = recordReader.nextKeyValue();

    while (hasNext && batch.size() < batchSize && previewCount < batchSize) {
      batch.add(new Pair(filePath.toUri().getPath() + "::" + recordReader.getCurrentKey(),
          String.valueOf(recordReader.getCurrentValue())));
      hasNext = recordReader.nextKeyValue(); // not like iterator.hasNext, actually advances
      previewCount++;
    }
  }
  return batch;
}
 

public void setDatum(Object oDatum) {
  this.datum = (TaskFinished)oDatum;
  this.taskid = TaskID.forName(datum.taskid.toString());
  if (datum.successfulAttemptId != null) {
    this.successfulAttemptId = TaskAttemptID
        .forName(datum.successfulAttemptId.toString());
  }
  this.finishTime = datum.finishTime;
  this.taskType = TaskType.valueOf(datum.taskType.toString());
  this.status = datum.status.toString();
  this.counters = EventReader.fromAvro(datum.counters);
}
 

public void setDatum(Object oDatum) {
  this.datum = (TaskAttemptFinished)oDatum;
  this.attemptId = TaskAttemptID.forName(datum.attemptId.toString());
  this.taskType = TaskType.valueOf(datum.taskType.toString());
  this.taskStatus = datum.taskStatus.toString();
  this.finishTime = datum.finishTime;
  this.rackName = datum.rackname.toString();
  this.hostname = datum.hostname.toString();
  this.state = datum.state.toString();
  this.counters = EventReader.fromAvro(datum.counters);
}
 

HistoryEvent maybeEmitEvent(ParsedLine line, String taskAttemptIDName,
    HistoryEventEmitter thatg) {
  if (taskAttemptIDName == null) {
    return null;
  }

  TaskAttemptID taskAttemptID = TaskAttemptID.forName(taskAttemptIDName);

  String finishTime = line.get("FINISH_TIME");
  String status = line.get("TASK_STATUS");

  if (finishTime != null && status != null
      && !status.equalsIgnoreCase("success")) {
    String hostName = line.get("HOSTNAME");
    String error = line.get("ERROR");

    TaskAttempt20LineEventEmitter that =
        (TaskAttempt20LineEventEmitter) thatg;

    ParsedHost pHost = ParsedHost.parse(hostName);
    String rackName = null;
    
    // Earlier versions of MR logged on hostnames (without rackname) for
    // unsuccessful attempts
    if (pHost != null) {
      rackName = pHost.getRackName();
      hostName = pHost.getNodeName();
    }
    return new TaskAttemptUnsuccessfulCompletionEvent
      (taskAttemptID,
       that.originalTaskType, status, Long.parseLong(finishTime),
       hostName, -1, rackName, error, null);
  }

  return null;
}
 

static private TaskAttemptID createTaskAttemptID(boolean isMap, 
                                                 int taskNumber) {
  String attempt = taskAttemptIdPrefix + (isMap ? "m" : "r") + 
                   "_00000" + taskNumber + "_0";
  TaskAttemptID taskId = null;
  try {
    taskId = TaskAttemptID.forName(attempt);
  } catch (IllegalArgumentException iae) {
    Assert.fail("Invalid task attempt id string " + iae);
  }
  return taskId;
}
 

HistoryEvent maybeEmitEvent(ParsedLine line, String taskAttemptIDName,
    HistoryEventEmitter thatg) {
  if (taskAttemptIDName == null) {
    return null;
  }

  TaskAttemptID taskAttemptID = TaskAttemptID.forName(taskAttemptIDName);

  String startTime = line.get("START_TIME");
  String taskType = line.get("TASK_TYPE");
  String trackerName = line.get("TRACKER_NAME");
  String httpPort = line.get("HTTP_PORT");
  String locality = line.get("LOCALITY");
  if (locality == null) {
    locality = "";
  }
  String avataar = line.get("AVATAAR");
  if (avataar == null) {
    avataar = "";
  }

  if (startTime != null && taskType != null) {
    TaskAttempt20LineEventEmitter that =
        (TaskAttempt20LineEventEmitter) thatg;

    that.originalStartTime = Long.parseLong(startTime);
    that.originalTaskType =
        Version20LogInterfaceUtils.get20TaskType(taskType);

    int port =
        httpPort.equals("") ? DEFAULT_HTTP_PORT : Integer
            .parseInt(httpPort);

    return new TaskAttemptStartedEvent(taskAttemptID,
        that.originalTaskType, that.originalStartTime, trackerName, port, -1,
        locality, avataar);
  }

  return null;
}
 

HistoryEvent maybeEmitEvent(ParsedLine line, String taskAttemptIDName,
    HistoryEventEmitter thatg) {
  if (taskAttemptIDName == null) {
    return null;
  }

  TaskAttemptID taskAttemptID = TaskAttemptID.forName(taskAttemptIDName);

  String startTime = line.get("START_TIME");
  String taskType = line.get("TASK_TYPE");
  String trackerName = line.get("TRACKER_NAME");
  String httpPort = line.get("HTTP_PORT");
  String locality = line.get("LOCALITY");
  if (locality == null) {
    locality = "";
  }
  String avataar = line.get("AVATAAR");
  if (avataar == null) {
    avataar = "";
  }

  if (startTime != null && taskType != null) {
    TaskAttempt20LineEventEmitter that =
        (TaskAttempt20LineEventEmitter) thatg;

    that.originalStartTime = Long.parseLong(startTime);
    that.originalTaskType =
        Version20LogInterfaceUtils.get20TaskType(taskType);

    int port =
        httpPort.equals("") ? DEFAULT_HTTP_PORT : Integer
            .parseInt(httpPort);

    return new TaskAttemptStartedEvent(taskAttemptID,
        that.originalTaskType, that.originalStartTime, trackerName, port, -1,
        locality, avataar);
  }

  return null;
}
 

HistoryEvent maybeEmitEvent(ParsedLine line, String taskAttemptIDName,
    HistoryEventEmitter thatg) {
  if (taskAttemptIDName == null) {
    return null;
  }

  TaskAttemptID taskAttemptID = TaskAttemptID.forName(taskAttemptIDName);

  String finishTime = line.get("FINISH_TIME");
  String status = line.get("TASK_STATUS");

  if (finishTime != null && status != null
      && status.equalsIgnoreCase("success")) {
    String hostName = line.get("HOSTNAME");
    String counters = line.get("COUNTERS");
    String state = line.get("STATE_STRING");

    TaskAttempt20LineEventEmitter that =
        (TaskAttempt20LineEventEmitter) thatg;

    ParsedHost pHost = ParsedHost.parse(hostName);

    return new TaskAttemptFinishedEvent(taskAttemptID,
        that.originalTaskType, status, Long.parseLong(finishTime),
        pHost.getRackName(), pHost.getNodeName(), state, 
        maybeParseCounters(counters));
  }

  return null;
}
 

HistoryEvent maybeEmitEvent(ParsedLine line, String taskAttemptIDName,
    HistoryEventEmitter thatg) {
  if (taskAttemptIDName == null) {
    return null;
  }

  TaskAttemptID taskAttemptID = TaskAttemptID.forName(taskAttemptIDName);

  String finishTime = line.get("FINISH_TIME");
  String status = line.get("TASK_STATUS");

  if (finishTime != null && status != null
      && status.equalsIgnoreCase("success")) {
    String hostName = line.get("HOSTNAME");
    String counters = line.get("COUNTERS");
    String state = line.get("STATE_STRING");
    String shuffleFinish = line.get("SHUFFLE_FINISHED");
    String sortFinish = line.get("SORT_FINISHED");

    if (shuffleFinish != null && sortFinish != null
        && "success".equalsIgnoreCase(status)) {
      ReduceAttempt20LineHistoryEventEmitter that =
          (ReduceAttempt20LineHistoryEventEmitter) thatg;

      return new ReduceAttemptFinishedEvent
        (taskAttemptID,
         that.originalTaskType, status,
         Long.parseLong(shuffleFinish),
         Long.parseLong(sortFinish),
         Long.parseLong(finishTime),
         hostName, -1, null,
         state, maybeParseCounters(counters),
         null);
    }
  }

  return null;
}
 

void setAttemptID(String attemptID) {
  this.attemptID = TaskAttemptID.forName(attemptID);
}
 

/**
 * Create a new {@link TaskAttemptID} instance.
 *
 * @return a new {@link TaskAttemptID} instance
 */
public static TaskAttemptID newTaskAttemptID() {
  return TaskAttemptID.forName(ATTEMPT + SEPARATOR + Long.toString(System.currentTimeMillis()) + SEPARATOR + 0
      + SEPARATOR + 'm' + SEPARATOR + 0 + SEPARATOR + 0);
}