Java Code Examples for org.apache.hadoop.mapreduce.v2.api.records.TaskType#MAP

The following examples show how to use org.apache.hadoop.mapreduce.v2.api.records.TaskType#MAP . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: JobImpl.java    From big-c with Apache License 2.0 6 votes vote down vote up 
private void actOnUnusableNode(NodeId nodeId, NodeState nodeState) {
  // rerun previously successful map tasks
  List<TaskAttemptId> taskAttemptIdList = nodesToSucceededTaskAttempts.get(nodeId);
  if(taskAttemptIdList != null) {
    String mesg = "TaskAttempt killed because it ran on unusable node "
        + nodeId;
    for(TaskAttemptId id : taskAttemptIdList) {
      if(TaskType.MAP == id.getTaskId().getTaskType()) {
        // reschedule only map tasks because their outputs maybe unusable
        LOG.info(mesg + ". AttemptId:" + id);
        eventHandler.handle(new TaskAttemptKillEvent(id, mesg));
      }
    }
  }
  // currently running task attempts on unusable nodes are handled in
  // RMContainerAllocator
}
Example 2
Source File: JobImpl.java    From hadoop with Apache License 2.0 6 votes vote down vote up 
protected void addTask(Task task) {
  synchronized (tasksSyncHandle) {
    if (lazyTasksCopyNeeded) {
      Map<TaskId, Task> newTasks = new LinkedHashMap<TaskId, Task>();
      newTasks.putAll(tasks);
      tasks = newTasks;
      lazyTasksCopyNeeded = false;
    }
  }
  tasks.put(task.getID(), task);
  if (task.getType() == TaskType.MAP) {
    mapTasks.add(task.getID());
  } else if (task.getType() == TaskType.REDUCE) {
    reduceTasks.add(task.getID());
  }
  metrics.waitingTask(task);
}
Example 3
Source File: DefaultSpeculator.java    From big-c with Apache License 2.0 6 votes vote down vote up 
private AtomicInteger containerNeed(TaskId taskID) {
  JobId jobID = taskID.getJobId();
  TaskType taskType = taskID.getTaskType();

  ConcurrentMap<JobId, AtomicInteger> relevantMap
      = taskType == TaskType.MAP ? mapContainerNeeds : reduceContainerNeeds;

  AtomicInteger result = relevantMap.get(jobID);

  if (result == null) {
    relevantMap.putIfAbsent(jobID, new AtomicInteger(0));
    result = relevantMap.get(jobID);
  }

  return result;
}
Example 4
Source File: StartEndTimesBase.java    From big-c with Apache License 2.0 6 votes vote down vote up 
protected DataStatistics dataStatisticsForTask(TaskId taskID) {
  JobId jobID = taskID.getJobId();
  Job job = context.getJob(jobID);

  if (job == null) {
    return null;
  }

  Task task = job.getTask(taskID);

  if (task == null) {
    return null;
  }

  return task.getType() == TaskType.MAP
          ? mapperStatistics.get(job)
          : task.getType() == TaskType.REDUCE
              ? reducerStatistics.get(job)
              : null;
}
Example 5
Source File: MapTaskImpl.java    From big-c with Apache License 2.0 5 votes vote down vote up 
public MapTaskImpl(JobId jobId, int partition, EventHandler eventHandler,
    Path remoteJobConfFile, JobConf conf,
    TaskSplitMetaInfo taskSplitMetaInfo,
    TaskAttemptListener taskAttemptListener,
    Token<JobTokenIdentifier> jobToken,
    Credentials credentials, Clock clock,
    int appAttemptId, MRAppMetrics metrics, AppContext appContext) {
  super(jobId, TaskType.MAP, partition, eventHandler, remoteJobConfFile,
      conf, taskAttemptListener, jobToken, credentials, clock,
      appAttemptId, metrics, appContext);
  this.taskSplitMetaInfo = taskSplitMetaInfo;
}
Example 6
Source File: JobImpl.java    From hadoop with Apache License 2.0 5 votes vote down vote up 
private void taskSucceeded(JobImpl job, Task task) {
  if (task.getType() == TaskType.MAP) {
    job.succeededMapTaskCount++;
  } else {
    job.succeededReduceTaskCount++;
  }
  job.metrics.completedTask(task);
}
Example 7
Source File: TaskAttemptImpl.java    From big-c with Apache License 2.0 5 votes vote down vote up 
private int getMemoryRequired(Configuration conf, TaskType taskType) {
  int memory = 1024;
  if (taskType == TaskType.MAP)  {
    memory =
        conf.getInt(MRJobConfig.MAP_MEMORY_MB,
            MRJobConfig.DEFAULT_MAP_MEMORY_MB);
  } else if (taskType == TaskType.REDUCE) {
    memory =
        conf.getInt(MRJobConfig.REDUCE_MEMORY_MB,
            MRJobConfig.DEFAULT_REDUCE_MEMORY_MB);
  }
  
  return memory;
}
Example 8
Source File: MapTaskImpl.java    From hadoop with Apache License 2.0 5 votes vote down vote up 
public MapTaskImpl(JobId jobId, int partition, EventHandler eventHandler,
    Path remoteJobConfFile, JobConf conf,
    TaskSplitMetaInfo taskSplitMetaInfo,
    TaskAttemptListener taskAttemptListener,
    Token<JobTokenIdentifier> jobToken,
    Credentials credentials, Clock clock,
    int appAttemptId, MRAppMetrics metrics, AppContext appContext) {
  super(jobId, TaskType.MAP, partition, eventHandler, remoteJobConfFile,
      conf, taskAttemptListener, jobToken, credentials, clock,
      appAttemptId, metrics, appContext);
  this.taskSplitMetaInfo = taskSplitMetaInfo;
}
Example 9
Source File: JobImpl.java    From big-c with Apache License 2.0 5 votes vote down vote up 
private void taskFailed(JobImpl job, Task task) {
  if (task.getType() == TaskType.MAP) {
    job.failedMapTaskCount++;
  } else if (task.getType() == TaskType.REDUCE) {
    job.failedReduceTaskCount++;
  }
  job.addDiagnostic("Task failed " + task.getID());
  job.metrics.failedTask(task);
}
Example 10
Source File: TaskAttemptImpl.java    From big-c with Apache License 2.0 5 votes vote down vote up 
private static void updateMillisCounters(JobCounterUpdateEvent jce,
    TaskAttemptImpl taskAttempt) {
  TaskType taskType = taskAttempt.getID().getTaskId().getTaskType();
  long duration = (taskAttempt.getFinishTime() - taskAttempt.getLaunchTime());
  int mbRequired =
      taskAttempt.getMemoryRequired(taskAttempt.conf, taskType);
  int vcoresRequired = taskAttempt.getCpuRequired(taskAttempt.conf, taskType);

  int minSlotMemSize = taskAttempt.conf.getInt(
    YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB,
    YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB);

  int simSlotsRequired =
      minSlotMemSize == 0 ? 0 : (int) Math.ceil((float) mbRequired
          / minSlotMemSize);

  if (taskType == TaskType.MAP) {
    jce.addCounterUpdate(JobCounter.SLOTS_MILLIS_MAPS, simSlotsRequired * duration);
    jce.addCounterUpdate(JobCounter.MB_MILLIS_MAPS, duration * mbRequired);
    jce.addCounterUpdate(JobCounter.VCORES_MILLIS_MAPS, duration * vcoresRequired);
    jce.addCounterUpdate(JobCounter.MILLIS_MAPS, duration);
  } else {
    jce.addCounterUpdate(JobCounter.SLOTS_MILLIS_REDUCES, simSlotsRequired * duration);
    jce.addCounterUpdate(JobCounter.MB_MILLIS_REDUCES, duration * mbRequired);
    jce.addCounterUpdate(JobCounter.VCORES_MILLIS_REDUCES, duration * vcoresRequired);
    jce.addCounterUpdate(JobCounter.MILLIS_REDUCES, duration);
  }
}
Example 11
Source File: TaskAttemptImpl.java    From hadoop with Apache License 2.0 5 votes vote down vote up 
private int getMemoryRequired(Configuration conf, TaskType taskType) {
  int memory = 1024;
  if (taskType == TaskType.MAP)  {
    memory =
        conf.getInt(MRJobConfig.MAP_MEMORY_MB,
            MRJobConfig.DEFAULT_MAP_MEMORY_MB);
  } else if (taskType == TaskType.REDUCE) {
    memory =
        conf.getInt(MRJobConfig.REDUCE_MEMORY_MB,
            MRJobConfig.DEFAULT_REDUCE_MEMORY_MB);
  }
  
  return memory;
}
Example 12
Source File: TestRuntimeEstimators.java    From big-c with Apache License 2.0 5 votes vote down vote up 
@Override
public float getProgress() {
  if (overridingState == TaskAttemptState.NEW) {
    return 0.0F;
  }
  return myAttemptID.getTaskId().getTaskType() == TaskType.MAP ? getMapProgress() : getReduceProgress();
}
Example 13
Source File: JobImpl.java    From big-c with Apache License 2.0 5 votes vote down vote up 
private void taskSucceeded(JobImpl job, Task task) {
  if (task.getType() == TaskType.MAP) {
    job.succeededMapTaskCount++;
  } else {
    job.succeededReduceTaskCount++;
  }
  job.metrics.completedTask(task);
}
Example 14
Source File: TaskAttemptImpl.java    From big-c with Apache License 2.0 4 votes vote down vote up 
@SuppressWarnings({ "unchecked" })
private void logAttemptFinishedEvent(TaskAttemptStateInternal state) {
  //Log finished events only if an attempt started.
  if (getLaunchTime() == 0) return; 
  String containerHostName = this.container == null ? "UNKNOWN"
       : this.container.getNodeId().getHost();
  int containerNodePort =
      this.container == null ? -1 : this.container.getNodeId().getPort();
  if (attemptId.getTaskId().getTaskType() == TaskType.MAP) {
    MapAttemptFinishedEvent mfe =
       new MapAttemptFinishedEvent(TypeConverter.fromYarn(attemptId),
       TypeConverter.fromYarn(attemptId.getTaskId().getTaskType()),
       state.toString(),
       this.reportedStatus.mapFinishTime,
       finishTime,
       containerHostName,
       containerNodePort,
       this.nodeRackName == null ? "UNKNOWN" : this.nodeRackName,
       this.reportedStatus.stateString,
       getCounters(),
       getProgressSplitBlock().burst());
       eventHandler.handle(
         new JobHistoryEvent(attemptId.getTaskId().getJobId(), mfe));
  } else {
     ReduceAttemptFinishedEvent rfe =
       new ReduceAttemptFinishedEvent(TypeConverter.fromYarn(attemptId),
       TypeConverter.fromYarn(attemptId.getTaskId().getTaskType()),
       state.toString(),
       this.reportedStatus.shuffleFinishTime,
       this.reportedStatus.sortFinishTime,
       finishTime,
       containerHostName,
       containerNodePort,
       this.nodeRackName == null ? "UNKNOWN" : this.nodeRackName,
       this.reportedStatus.stateString,
       getCounters(),
       getProgressSplitBlock().burst());
       eventHandler.handle(
         new JobHistoryEvent(attemptId.getTaskId().getJobId(), rfe));
  }
}
Example 15
Source File: MRApps.java    From hadoop with Apache License 2.0 4 votes vote down vote up 
public static TaskType taskType(String symbol) {
  // JDK 7 supports switch on strings
  if (symbol.equals("m")) return TaskType.MAP;
  if (symbol.equals("r")) return TaskType.REDUCE;
  throw new YarnRuntimeException("Unknown task symbol: "+ symbol);
}
Example 16
Source File: TestTaskImpl.java    From hadoop with Apache License 2.0 4 votes vote down vote up 
@Test
public void testCountersWithSpeculation() {
  mockTask = new MockTaskImpl(jobId, partition, dispatcher.getEventHandler(),
      remoteJobConfFile, conf, taskAttemptListener, jobToken,
      credentials, clock, startCount, metrics, appContext, TaskType.MAP) {
        @Override
        protected int getMaxAttempts() {
          return 1;
        }
  };
  TaskId taskId = getNewTaskID();
  scheduleTaskAttempt(taskId);
  launchTaskAttempt(getLastAttempt().getAttemptId());
  updateLastAttemptState(TaskAttemptState.RUNNING);
  MockTaskAttemptImpl baseAttempt = getLastAttempt();

  // add a speculative attempt
  mockTask.handle(new TaskTAttemptEvent(getLastAttempt().getAttemptId(),
      TaskEventType.T_ADD_SPEC_ATTEMPT));
  launchTaskAttempt(getLastAttempt().getAttemptId());
  updateLastAttemptState(TaskAttemptState.RUNNING);
  MockTaskAttemptImpl specAttempt = getLastAttempt();
  assertEquals(2, taskAttempts.size());

  Counters specAttemptCounters = new Counters();
  Counter cpuCounter = specAttemptCounters.findCounter(
      TaskCounter.CPU_MILLISECONDS);
  cpuCounter.setValue(1000);
  specAttempt.setCounters(specAttemptCounters);

  // have the spec attempt succeed but second attempt at 1.0 progress as well
  commitTaskAttempt(specAttempt.getAttemptId());
  specAttempt.setProgress(1.0f);
  specAttempt.setState(TaskAttemptState.SUCCEEDED);
  mockTask.handle(new TaskTAttemptEvent(specAttempt.getAttemptId(),
      TaskEventType.T_ATTEMPT_SUCCEEDED));
  assertEquals(TaskState.SUCCEEDED, mockTask.getState());
  baseAttempt.setProgress(1.0f);

  Counters taskCounters = mockTask.getCounters();
  assertEquals("wrong counters for task", specAttemptCounters, taskCounters);
}
Example 17
Source File: TestTaskImpl.java    From big-c with Apache License 2.0 4 votes vote down vote up 
@Test
public void testCountersWithSpeculation() {
  mockTask = new MockTaskImpl(jobId, partition, dispatcher.getEventHandler(),
      remoteJobConfFile, conf, taskAttemptListener, jobToken,
      credentials, clock, startCount, metrics, appContext, TaskType.MAP) {
        @Override
        protected int getMaxAttempts() {
          return 1;
        }
  };
  TaskId taskId = getNewTaskID();
  scheduleTaskAttempt(taskId);
  launchTaskAttempt(getLastAttempt().getAttemptId());
  updateLastAttemptState(TaskAttemptState.RUNNING);
  MockTaskAttemptImpl baseAttempt = getLastAttempt();

  // add a speculative attempt
  mockTask.handle(new TaskTAttemptEvent(getLastAttempt().getAttemptId(),
      TaskEventType.T_ADD_SPEC_ATTEMPT));
  launchTaskAttempt(getLastAttempt().getAttemptId());
  updateLastAttemptState(TaskAttemptState.RUNNING);
  MockTaskAttemptImpl specAttempt = getLastAttempt();
  assertEquals(2, taskAttempts.size());

  Counters specAttemptCounters = new Counters();
  Counter cpuCounter = specAttemptCounters.findCounter(
      TaskCounter.CPU_MILLISECONDS);
  cpuCounter.setValue(1000);
  specAttempt.setCounters(specAttemptCounters);

  // have the spec attempt succeed but second attempt at 1.0 progress as well
  commitTaskAttempt(specAttempt.getAttemptId());
  specAttempt.setProgress(1.0f);
  specAttempt.setState(TaskAttemptState.SUCCEEDED);
  mockTask.handle(new TaskTAttemptEvent(specAttempt.getAttemptId(),
      TaskEventType.T_ATTEMPT_SUCCEEDED));
  assertEquals(TaskState.SUCCEEDED, mockTask.getState());
  baseAttempt.setProgress(1.0f);

  Counters taskCounters = mockTask.getCounters();
  assertEquals("wrong counters for task", specAttemptCounters, taskCounters);
}
Example 18
Source File: TestRuntimeEstimators.java    From hadoop with Apache License 2.0 4 votes vote down vote up 
@Override
public Map<TaskId, Task> getTasks(TaskType taskType) {
  return taskType == TaskType.MAP ? mapTasks : reduceTasks;
}
Example 19
Source File: MapTaskImpl.java    From big-c with Apache License 2.0 4 votes vote down vote up 
@Override
public TaskType getType() {
  return TaskType.MAP;
}
Example 20
Source File: TestRuntimeEstimators.java    From big-c with Apache License 2.0 4 votes vote down vote up 
int taskTypeSlots(TaskType type) {
  return type == TaskType.MAP ? MAP_SLOT_REQUIREMENT : REDUCE_SLOT_REQUIREMENT;
}