Java Code Examples for org.apache.hadoop.mapreduce.v2.api.records.TaskType#REDUCE

The following examples show how to use org.apache.hadoop.mapreduce.v2.api.records.TaskType#REDUCE . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: AMWebServices.java    From hadoop with Apache License 2.0 6 votes vote down vote up 
@GET
@Path("/jobs/{jobid}/tasks/{taskid}/attempts")
@Produces({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML })
public TaskAttemptsInfo getJobTaskAttempts(@Context HttpServletRequest hsr,
    @PathParam("jobid") String jid, @PathParam("taskid") String tid) {

  init();
  TaskAttemptsInfo attempts = new TaskAttemptsInfo();
  Job job = getJobFromJobIdString(jid, appCtx);
  checkAccess(job, hsr);
  Task task = getTaskFromTaskIdString(tid, job);

  for (TaskAttempt ta : task.getAttempts().values()) {
    if (ta != null) {
      if (task.getType() == TaskType.REDUCE) {
        attempts.add(new ReduceTaskAttemptInfo(ta, task.getType()));
      } else {
        attempts.add(new TaskAttemptInfo(ta, task.getType(), true));
      }
    }
  }
  return attempts;
}
Example 2
Source File: AMWebServices.java    From hadoop with Apache License 2.0 6 votes vote down vote up 
@GET
@Path("/jobs/{jobid}/tasks/{taskid}/attempts/{attemptid}")
@Produces({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML })
public TaskAttemptInfo getJobTaskAttemptId(@Context HttpServletRequest hsr,
    @PathParam("jobid") String jid, @PathParam("taskid") String tid,
    @PathParam("attemptid") String attId) {

  init();
  Job job = getJobFromJobIdString(jid, appCtx);
  checkAccess(job, hsr);
  Task task = getTaskFromTaskIdString(tid, job);
  TaskAttempt ta = getTaskAttemptFromTaskAttemptString(attId, task);
  if (task.getType() == TaskType.REDUCE) {
    return new ReduceTaskAttemptInfo(ta, task.getType());
  } else {
    return new TaskAttemptInfo(ta, task.getType(), true);
  }
}
Example 3
Source File: StartEndTimesBase.java    From big-c with Apache License 2.0 6 votes vote down vote up 
protected DataStatistics dataStatisticsForTask(TaskId taskID) {
  JobId jobID = taskID.getJobId();
  Job job = context.getJob(jobID);

  if (job == null) {
    return null;
  }

  Task task = job.getTask(taskID);

  if (task == null) {
    return null;
  }

  return task.getType() == TaskType.MAP
          ? mapperStatistics.get(job)
          : task.getType() == TaskType.REDUCE
              ? reducerStatistics.get(job)
              : null;
}
Example 4
Source File: TestAMWebServicesAttempts.java    From big-c with Apache License 2.0 6 votes vote down vote up 
public void verifyAMTaskAttemptXML(Element element, TaskAttempt att,
    TaskType ttype) {
  verifyTaskAttemptGeneric(att, ttype,
      WebServicesTestUtils.getXmlString(element, "id"),
      WebServicesTestUtils.getXmlString(element, "state"),
      WebServicesTestUtils.getXmlString(element, "type"),
      WebServicesTestUtils.getXmlString(element, "rack"),
      WebServicesTestUtils.getXmlString(element, "nodeHttpAddress"),
      WebServicesTestUtils.getXmlString(element, "diagnostics"),
      WebServicesTestUtils.getXmlString(element, "assignedContainerId"),
      WebServicesTestUtils.getXmlLong(element, "startTime"),
      WebServicesTestUtils.getXmlLong(element, "finishTime"),
      WebServicesTestUtils.getXmlLong(element, "elapsedTime"),
      WebServicesTestUtils.getXmlFloat(element, "progress"));

  if (ttype == TaskType.REDUCE) {
    verifyReduceTaskAttemptGeneric(att,
        WebServicesTestUtils.getXmlLong(element, "shuffleFinishTime"),
        WebServicesTestUtils.getXmlLong(element, "mergeFinishTime"),
        WebServicesTestUtils.getXmlLong(element, "elapsedShuffleTime"),
        WebServicesTestUtils.getXmlLong(element, "elapsedMergeTime"),
        WebServicesTestUtils.getXmlLong(element, "elapsedReduceTime"));
  }
}
Example 5
Source File: TestHsWebServicesAttempts.java    From big-c with Apache License 2.0 6 votes vote down vote up 
public void verifyHsTaskAttemptXML(Element element, TaskAttempt att,
    TaskType ttype) {
  verifyTaskAttemptGeneric(att, ttype,
      WebServicesTestUtils.getXmlString(element, "id"),
      WebServicesTestUtils.getXmlString(element, "state"),
      WebServicesTestUtils.getXmlString(element, "type"),
      WebServicesTestUtils.getXmlString(element, "rack"),
      WebServicesTestUtils.getXmlString(element, "nodeHttpAddress"),
      WebServicesTestUtils.getXmlString(element, "diagnostics"),
      WebServicesTestUtils.getXmlString(element, "assignedContainerId"),
      WebServicesTestUtils.getXmlLong(element, "startTime"),
      WebServicesTestUtils.getXmlLong(element, "finishTime"),
      WebServicesTestUtils.getXmlLong(element, "elapsedTime"),
      WebServicesTestUtils.getXmlFloat(element, "progress"));

  if (ttype == TaskType.REDUCE) {
    verifyReduceTaskAttemptGeneric(att,
        WebServicesTestUtils.getXmlLong(element, "shuffleFinishTime"),
        WebServicesTestUtils.getXmlLong(element, "mergeFinishTime"),
        WebServicesTestUtils.getXmlLong(element, "elapsedShuffleTime"),
        WebServicesTestUtils.getXmlLong(element, "elapsedMergeTime"),
        WebServicesTestUtils.getXmlLong(element, "elapsedReduceTime"));
  }
}
Example 6
Source File: TestHsWebServicesAttempts.java    From hadoop with Apache License 2.0 6 votes vote down vote up 
public void verifyHsTaskAttemptXML(Element element, TaskAttempt att,
    TaskType ttype) {
  verifyTaskAttemptGeneric(att, ttype,
      WebServicesTestUtils.getXmlString(element, "id"),
      WebServicesTestUtils.getXmlString(element, "state"),
      WebServicesTestUtils.getXmlString(element, "type"),
      WebServicesTestUtils.getXmlString(element, "rack"),
      WebServicesTestUtils.getXmlString(element, "nodeHttpAddress"),
      WebServicesTestUtils.getXmlString(element, "diagnostics"),
      WebServicesTestUtils.getXmlString(element, "assignedContainerId"),
      WebServicesTestUtils.getXmlLong(element, "startTime"),
      WebServicesTestUtils.getXmlLong(element, "finishTime"),
      WebServicesTestUtils.getXmlLong(element, "elapsedTime"),
      WebServicesTestUtils.getXmlFloat(element, "progress"));

  if (ttype == TaskType.REDUCE) {
    verifyReduceTaskAttemptGeneric(att,
        WebServicesTestUtils.getXmlLong(element, "shuffleFinishTime"),
        WebServicesTestUtils.getXmlLong(element, "mergeFinishTime"),
        WebServicesTestUtils.getXmlLong(element, "elapsedShuffleTime"),
        WebServicesTestUtils.getXmlLong(element, "elapsedMergeTime"),
        WebServicesTestUtils.getXmlLong(element, "elapsedReduceTime"));
  }
}
Example 7
Source File: StartEndTimesBase.java    From hadoop with Apache License 2.0 6 votes vote down vote up 
protected DataStatistics dataStatisticsForTask(TaskId taskID) {
  JobId jobID = taskID.getJobId();
  Job job = context.getJob(jobID);

  if (job == null) {
    return null;
  }

  Task task = job.getTask(taskID);

  if (task == null) {
    return null;
  }

  return task.getType() == TaskType.MAP
          ? mapperStatistics.get(job)
          : task.getType() == TaskType.REDUCE
              ? reducerStatistics.get(job)
              : null;
}
Example 8
Source File: TestHsWebServicesAttempts.java    From big-c with Apache License 2.0 6 votes vote down vote up 
public void verifyHsTaskAttempt(JSONObject info, TaskAttempt att,
    TaskType ttype) throws JSONException {
  if (ttype == TaskType.REDUCE) {
    assertEquals("incorrect number of elements", 17, info.length());
  } else {
    assertEquals("incorrect number of elements", 12, info.length());
  }

  verifyTaskAttemptGeneric(att, ttype, info.getString("id"),
      info.getString("state"), info.getString("type"),
      info.getString("rack"), info.getString("nodeHttpAddress"),
      info.getString("diagnostics"), info.getString("assignedContainerId"),
      info.getLong("startTime"), info.getLong("finishTime"),
      info.getLong("elapsedTime"), (float) info.getDouble("progress"));

  if (ttype == TaskType.REDUCE) {
    verifyReduceTaskAttemptGeneric(att, info.getLong("shuffleFinishTime"),
        info.getLong("mergeFinishTime"), info.getLong("elapsedShuffleTime"),
        info.getLong("elapsedMergeTime"), info.getLong("elapsedReduceTime"));
  }
}
Example 9
Source File: TestAMWebServicesAttempts.java    From hadoop with Apache License 2.0 6 votes vote down vote up 
public void verifyAMTaskAttemptXML(Element element, TaskAttempt att,
    TaskType ttype) {
  verifyTaskAttemptGeneric(att, ttype,
      WebServicesTestUtils.getXmlString(element, "id"),
      WebServicesTestUtils.getXmlString(element, "state"),
      WebServicesTestUtils.getXmlString(element, "type"),
      WebServicesTestUtils.getXmlString(element, "rack"),
      WebServicesTestUtils.getXmlString(element, "nodeHttpAddress"),
      WebServicesTestUtils.getXmlString(element, "diagnostics"),
      WebServicesTestUtils.getXmlString(element, "assignedContainerId"),
      WebServicesTestUtils.getXmlLong(element, "startTime"),
      WebServicesTestUtils.getXmlLong(element, "finishTime"),
      WebServicesTestUtils.getXmlLong(element, "elapsedTime"),
      WebServicesTestUtils.getXmlFloat(element, "progress"));

  if (ttype == TaskType.REDUCE) {
    verifyReduceTaskAttemptGeneric(att,
        WebServicesTestUtils.getXmlLong(element, "shuffleFinishTime"),
        WebServicesTestUtils.getXmlLong(element, "mergeFinishTime"),
        WebServicesTestUtils.getXmlLong(element, "elapsedShuffleTime"),
        WebServicesTestUtils.getXmlLong(element, "elapsedMergeTime"),
        WebServicesTestUtils.getXmlLong(element, "elapsedReduceTime"));
  }
}
Example 10
Source File: TestMRApps.java    From hadoop with Apache License 2.0 5 votes vote down vote up 
@Test (timeout = 120000)
public void testTaskIDtoString() {
  TaskId tid = RecordFactoryProvider.getRecordFactory(null).newRecordInstance(TaskId.class);
  tid.setJobId(RecordFactoryProvider.getRecordFactory(null).newRecordInstance(JobId.class));
  tid.getJobId().setAppId(ApplicationId.newInstance(0, 0));
  tid.setTaskType(TaskType.MAP);
  TaskType type = tid.getTaskType();
  System.err.println(type);
  type = TaskType.REDUCE;
  System.err.println(type);
  System.err.println(tid.getTaskType());
  assertEquals("task_0_0000_m_000000", MRApps.toString(tid));
  tid.setTaskType(TaskType.REDUCE);
  assertEquals("task_0_0000_r_000000", MRApps.toString(tid));
}
Example 11
Source File: TypeConverter.java    From hadoop with Apache License 2.0 5 votes vote down vote up 
public static TaskType
    toYarn(org.apache.hadoop.mapreduce.TaskType taskType) {
  switch (taskType) {
  case MAP:
    return TaskType.MAP;
  case REDUCE:
    return TaskType.REDUCE;
  default:
    throw new YarnRuntimeException("Unrecognized task type: " + taskType);
  }
}
Example 12
Source File: MockJobs.java    From hadoop with Apache License 2.0 5 votes vote down vote up 
public static TaskAttemptReport newTaskAttemptReport(TaskAttemptId id) {
  ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(
      id.getTaskId().getJobId().getAppId(), 0);
  ContainerId containerId = ContainerId.newContainerId(appAttemptId, 0);
  TaskAttemptReport report = Records.newRecord(TaskAttemptReport.class);
  report.setTaskAttemptId(id);
  report
      .setStartTime(System.currentTimeMillis() - (int) (Math.random() * DT));
  report.setFinishTime(System.currentTimeMillis()
      + (int) (Math.random() * DT) + 1);

  if (id.getTaskId().getTaskType() == TaskType.REDUCE) {
    report.setShuffleFinishTime(
        (report.getFinishTime() + report.getStartTime()) / 2);
    report.setSortFinishTime(
        (report.getFinishTime() + report.getShuffleFinishTime()) / 2);
  }

  report.setPhase(PHASES.next());
  report.setTaskAttemptState(TASK_ATTEMPT_STATES.next());
  report.setProgress((float) Math.random());
  report.setCounters(TypeConverter.toYarn(newCounters()));
  report.setContainerId(containerId);
  report.setDiagnosticInfo(DIAGS.next());
  report.setStateString("Moving average " + Math.random());
  return report;
}
Example 13
Source File: TaskAttemptImpl.java    From big-c with Apache License 2.0 5 votes vote down vote up 
private int getMemoryRequired(Configuration conf, TaskType taskType) {
  int memory = 1024;
  if (taskType == TaskType.MAP)  {
    memory =
        conf.getInt(MRJobConfig.MAP_MEMORY_MB,
            MRJobConfig.DEFAULT_MAP_MEMORY_MB);
  } else if (taskType == TaskType.REDUCE) {
    memory =
        conf.getInt(MRJobConfig.REDUCE_MEMORY_MB,
            MRJobConfig.DEFAULT_REDUCE_MEMORY_MB);
  }
  
  return memory;
}
Example 14
Source File: TaskAttemptImpl.java    From big-c with Apache License 2.0 5 votes vote down vote up 
private int getCpuRequired(Configuration conf, TaskType taskType) {
  int vcores = 1;
  if (taskType == TaskType.MAP)  {
    vcores =
        conf.getInt(MRJobConfig.MAP_CPU_VCORES,
            MRJobConfig.DEFAULT_MAP_CPU_VCORES);
  } else if (taskType == TaskType.REDUCE) {
    vcores =
        conf.getInt(MRJobConfig.REDUCE_CPU_VCORES,
            MRJobConfig.DEFAULT_REDUCE_CPU_VCORES);
  }
  
  return vcores;
}
Example 15
Source File: JobImpl.java    From hadoop with Apache License 2.0 5 votes vote down vote up 
private void taskFailed(JobImpl job, Task task) {
  if (task.getType() == TaskType.MAP) {
    job.failedMapTaskCount++;
  } else if (task.getType() == TaskType.REDUCE) {
    job.failedReduceTaskCount++;
  }
  job.addDiagnostic("Task failed " + task.getID());
  job.metrics.failedTask(task);
}
Example 16
Source File: TypeConverter.java    From big-c with Apache License 2.0 5 votes vote down vote up 
public static TaskType
    toYarn(org.apache.hadoop.mapreduce.TaskType taskType) {
  switch (taskType) {
  case MAP:
    return TaskType.MAP;
  case REDUCE:
    return TaskType.REDUCE;
  default:
    throw new YarnRuntimeException("Unrecognized task type: " + taskType);
  }
}
Example 17
Source File: ReduceTaskImpl.java    From hadoop with Apache License 2.0 5 votes vote down vote up 
public ReduceTaskImpl(JobId jobId, int partition,
    EventHandler eventHandler, Path jobFile, JobConf conf,
    int numMapTasks, TaskAttemptListener taskAttemptListener,
    Token<JobTokenIdentifier> jobToken,
    Credentials credentials, Clock clock,
    int appAttemptId, MRAppMetrics metrics, AppContext appContext) {
  super(jobId, TaskType.REDUCE, partition, eventHandler, jobFile, conf,
      taskAttemptListener, jobToken, credentials, clock,
      appAttemptId, metrics, appContext);
  this.numMapTasks = numMapTasks;
}
Example 18
Source File: TestKill.java    From big-c with Apache License 2.0 4 votes vote down vote up 
@Test
public void testKillTaskWait() throws Exception {
  final Dispatcher dispatcher = new AsyncDispatcher() {
    private TaskAttemptEvent cachedKillEvent;
    @Override
    protected void dispatch(Event event) {
      if (event instanceof TaskAttemptEvent) {
        TaskAttemptEvent killEvent = (TaskAttemptEvent) event;
        if (killEvent.getType() == TaskAttemptEventType.TA_KILL) {
          TaskAttemptId taID = killEvent.getTaskAttemptID();
          if (taID.getTaskId().getTaskType() == TaskType.REDUCE
              && taID.getTaskId().getId() == 0 && taID.getId() == 0) {
            // Task is asking the reduce TA to kill itself. 'Create' a race
            // condition. Make the task succeed and then inform the task that
            // TA has succeeded. Once Task gets the TA succeeded event at
            // KILL_WAIT, then relay the actual kill signal to TA
            super.dispatch(new TaskAttemptEvent(taID,
              TaskAttemptEventType.TA_DONE));
            super.dispatch(new TaskAttemptEvent(taID,
              TaskAttemptEventType.TA_CONTAINER_CLEANED));
            super.dispatch(new TaskTAttemptEvent(taID,
              TaskEventType.T_ATTEMPT_SUCCEEDED));
            this.cachedKillEvent = killEvent;
            return;
          }
        }
      } else if (event instanceof TaskEvent) {
        TaskEvent taskEvent = (TaskEvent) event;
        if (taskEvent.getType() == TaskEventType.T_ATTEMPT_SUCCEEDED
            && this.cachedKillEvent != null) {
          // When the TA comes and reports that it is done, send the
          // cachedKillEvent
          super.dispatch(this.cachedKillEvent);
          return;
        }

      }
      super.dispatch(event);
    }
  };
  MRApp app = new MRApp(1, 1, false, this.getClass().getName(), true) {
    @Override
    public Dispatcher createDispatcher() {
      return dispatcher;
    }
  };
  Job job = app.submit(new Configuration());
  JobId jobId = app.getJobId();
  app.waitForState(job, JobState.RUNNING);
  Assert.assertEquals("Num tasks not correct", 2, job.getTasks().size());
  Iterator<Task> it = job.getTasks().values().iterator();
  Task mapTask = it.next();
  Task reduceTask = it.next();
  app.waitForState(mapTask, TaskState.RUNNING);
  app.waitForState(reduceTask, TaskState.RUNNING);
  TaskAttempt mapAttempt = mapTask.getAttempts().values().iterator().next();
  app.waitForState(mapAttempt, TaskAttemptState.RUNNING);
  TaskAttempt reduceAttempt = reduceTask.getAttempts().values().iterator().next();
  app.waitForState(reduceAttempt, TaskAttemptState.RUNNING);

  // Finish map
  app.getContext().getEventHandler().handle(
      new TaskAttemptEvent(
          mapAttempt.getID(),
          TaskAttemptEventType.TA_DONE));
  app.waitForState(mapTask, TaskState.SUCCEEDED);

  // Now kill the job
  app.getContext().getEventHandler()
    .handle(new JobEvent(jobId, JobEventType.JOB_KILL));

  app.waitForInternalState((JobImpl)job, JobStateInternal.KILLED);
}
Example 19
Source File: ReduceTaskImpl.java    From hadoop with Apache License 2.0 4 votes vote down vote up 
@Override
public TaskType getType() {
  return TaskType.REDUCE;
}
Example 20
Source File: MRApps.java    From hadoop with Apache License 2.0 4 votes vote down vote up 
public static TaskType taskType(String symbol) {
  // JDK 7 supports switch on strings
  if (symbol.equals("m")) return TaskType.MAP;
  if (symbol.equals("r")) return TaskType.REDUCE;
  throw new YarnRuntimeException("Unknown task symbol: "+ symbol);
}