org.apache.hadoop.mapreduce.jobhistory.NormalizedResourceEvent Java Examples

The following examples show how to use org.apache.hadoop.mapreduce.jobhistory.NormalizedResourceEvent. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: MRApp.java    From hadoop with Apache License 2.0 5 votes vote down vote up 
@Override
public void handle(ContainerAllocatorEvent event) {
  ContainerId cId =
      ContainerId.newContainerId(getContext().getApplicationAttemptId(),
        containerCount++);
  NodeId nodeId = NodeId.newInstance(NM_HOST, NM_PORT);
  Resource resource = Resource.newInstance(1234, 2, 2);
  ContainerTokenIdentifier containerTokenIdentifier =
      new ContainerTokenIdentifier(cId, nodeId.toString(), "user",
      resource, System.currentTimeMillis() + 10000, 42, 42,
      Priority.newInstance(0), 0);
  Token containerToken = newContainerToken(nodeId, "password".getBytes(),
        containerTokenIdentifier);
  Container container = Container.newInstance(cId, nodeId,
      NM_HOST + ":" + NM_HTTP_PORT, resource, null, containerToken);
  JobID id = TypeConverter.fromYarn(applicationId);
  JobId jobId = TypeConverter.toYarn(id);
  getContext().getEventHandler().handle(new JobHistoryEvent(jobId, 
      new NormalizedResourceEvent(
          org.apache.hadoop.mapreduce.TaskType.REDUCE,
      100)));
  getContext().getEventHandler().handle(new JobHistoryEvent(jobId, 
      new NormalizedResourceEvent(
          org.apache.hadoop.mapreduce.TaskType.MAP,
      100)));
  getContext().getEventHandler().handle(
      new TaskAttemptContainerAssignedEvent(event.getAttemptID(),
          container, null));
}
Example #2
Source File: MRApp.java    From big-c with Apache License 2.0 5 votes vote down vote up 
@Override
public void handle(ContainerAllocatorEvent event) {
  ContainerId cId =
      ContainerId.newContainerId(getContext().getApplicationAttemptId(),
        containerCount++);
  NodeId nodeId = NodeId.newInstance(NM_HOST, NM_PORT);
  Resource resource = Resource.newInstance(1234, 2);
  ContainerTokenIdentifier containerTokenIdentifier =
      new ContainerTokenIdentifier(cId, nodeId.toString(), "user",
      resource, System.currentTimeMillis() + 10000, 42, 42,
      Priority.newInstance(0), 0);
  Token containerToken = newContainerToken(nodeId, "password".getBytes(),
        containerTokenIdentifier);
  Container container = Container.newInstance(cId, nodeId,
      NM_HOST + ":" + NM_HTTP_PORT, resource, null, containerToken);
  JobID id = TypeConverter.fromYarn(applicationId);
  JobId jobId = TypeConverter.toYarn(id);
  getContext().getEventHandler().handle(new JobHistoryEvent(jobId, 
      new NormalizedResourceEvent(
          org.apache.hadoop.mapreduce.TaskType.REDUCE,
      100)));
  getContext().getEventHandler().handle(new JobHistoryEvent(jobId, 
      new NormalizedResourceEvent(
          org.apache.hadoop.mapreduce.TaskType.MAP,
      100)));
  getContext().getEventHandler().handle(
      new TaskAttemptContainerAssignedEvent(event.getAttemptID(),
          container, null));
}
Example #3
Source File: JobBuilder.java    From hadoop with Apache License 2.0 4 votes vote down vote up 
/**
 * Process one {@link HistoryEvent}
 * 
 * @param event
 *          The {@link HistoryEvent} to be processed.
 */
public void process(HistoryEvent event) {
  if (finalized) {
    throw new IllegalStateException(
        "JobBuilder.process(HistoryEvent event) called after ParsedJob built");
  }

  // these are in lexicographical order by class name.
  if (event instanceof AMStartedEvent) {
    // ignore this event as Rumen currently doesnt need this event
    //TODO Enhance Rumen to process this event and capture restarts
    return;
  } else if (event instanceof NormalizedResourceEvent) {
    // Log an warn message as NormalizedResourceEvent shouldn't be written.
    LOG.warn("NormalizedResourceEvent should be ignored in history server.");
  } else if (event instanceof JobFinishedEvent) {
    processJobFinishedEvent((JobFinishedEvent) event);
  } else if (event instanceof JobInfoChangeEvent) {
    processJobInfoChangeEvent((JobInfoChangeEvent) event);
  } else if (event instanceof JobInitedEvent) {
    processJobInitedEvent((JobInitedEvent) event);
  } else if (event instanceof JobPriorityChangeEvent) {
    processJobPriorityChangeEvent((JobPriorityChangeEvent) event);
  } else if (event instanceof JobQueueChangeEvent) {
    processJobQueueChangeEvent((JobQueueChangeEvent) event);
  } else if (event instanceof JobStatusChangedEvent) {
    processJobStatusChangedEvent((JobStatusChangedEvent) event);
  } else if (event instanceof JobSubmittedEvent) {
    processJobSubmittedEvent((JobSubmittedEvent) event);
  } else if (event instanceof JobUnsuccessfulCompletionEvent) {
    processJobUnsuccessfulCompletionEvent((JobUnsuccessfulCompletionEvent) event);
  } else if (event instanceof MapAttemptFinishedEvent) {
    processMapAttemptFinishedEvent((MapAttemptFinishedEvent) event);
  } else if (event instanceof ReduceAttemptFinishedEvent) {
    processReduceAttemptFinishedEvent((ReduceAttemptFinishedEvent) event);
  } else if (event instanceof TaskAttemptFinishedEvent) {
    processTaskAttemptFinishedEvent((TaskAttemptFinishedEvent) event);
  } else if (event instanceof TaskAttemptStartedEvent) {
    processTaskAttemptStartedEvent((TaskAttemptStartedEvent) event);
  } else if (event instanceof TaskAttemptUnsuccessfulCompletionEvent) {
    processTaskAttemptUnsuccessfulCompletionEvent((TaskAttemptUnsuccessfulCompletionEvent) event);
  } else if (event instanceof TaskFailedEvent) {
    processTaskFailedEvent((TaskFailedEvent) event);
  } else if (event instanceof TaskFinishedEvent) {
    processTaskFinishedEvent((TaskFinishedEvent) event);
  } else if (event instanceof TaskStartedEvent) {
    processTaskStartedEvent((TaskStartedEvent) event);
  } else if (event instanceof TaskUpdatedEvent) {
    processTaskUpdatedEvent((TaskUpdatedEvent) event);
  } else
    throw new IllegalArgumentException(
        "JobBuilder.process(HistoryEvent): unknown event type:"
        + event.getEventType() + " for event:" + event);
}
Example #4
Source File: JobBuilder.java    From big-c with Apache License 2.0 4 votes vote down vote up 
/**
 * Process one {@link HistoryEvent}
 * 
 * @param event
 *          The {@link HistoryEvent} to be processed.
 */
public void process(HistoryEvent event) {
  if (finalized) {
    throw new IllegalStateException(
        "JobBuilder.process(HistoryEvent event) called after ParsedJob built");
  }

  // these are in lexicographical order by class name.
  if (event instanceof AMStartedEvent) {
    // ignore this event as Rumen currently doesnt need this event
    //TODO Enhance Rumen to process this event and capture restarts
    return;
  } else if (event instanceof NormalizedResourceEvent) {
    // Log an warn message as NormalizedResourceEvent shouldn't be written.
    LOG.warn("NormalizedResourceEvent should be ignored in history server.");
  } else if (event instanceof JobFinishedEvent) {
    processJobFinishedEvent((JobFinishedEvent) event);
  } else if (event instanceof JobInfoChangeEvent) {
    processJobInfoChangeEvent((JobInfoChangeEvent) event);
  } else if (event instanceof JobInitedEvent) {
    processJobInitedEvent((JobInitedEvent) event);
  } else if (event instanceof JobPriorityChangeEvent) {
    processJobPriorityChangeEvent((JobPriorityChangeEvent) event);
  } else if (event instanceof JobQueueChangeEvent) {
    processJobQueueChangeEvent((JobQueueChangeEvent) event);
  } else if (event instanceof JobStatusChangedEvent) {
    processJobStatusChangedEvent((JobStatusChangedEvent) event);
  } else if (event instanceof JobSubmittedEvent) {
    processJobSubmittedEvent((JobSubmittedEvent) event);
  } else if (event instanceof JobUnsuccessfulCompletionEvent) {
    processJobUnsuccessfulCompletionEvent((JobUnsuccessfulCompletionEvent) event);
  } else if (event instanceof MapAttemptFinishedEvent) {
    processMapAttemptFinishedEvent((MapAttemptFinishedEvent) event);
  } else if (event instanceof ReduceAttemptFinishedEvent) {
    processReduceAttemptFinishedEvent((ReduceAttemptFinishedEvent) event);
  } else if (event instanceof TaskAttemptFinishedEvent) {
    processTaskAttemptFinishedEvent((TaskAttemptFinishedEvent) event);
  } else if (event instanceof TaskAttemptStartedEvent) {
    processTaskAttemptStartedEvent((TaskAttemptStartedEvent) event);
  } else if (event instanceof TaskAttemptUnsuccessfulCompletionEvent) {
    processTaskAttemptUnsuccessfulCompletionEvent((TaskAttemptUnsuccessfulCompletionEvent) event);
  } else if (event instanceof TaskFailedEvent) {
    processTaskFailedEvent((TaskFailedEvent) event);
  } else if (event instanceof TaskFinishedEvent) {
    processTaskFinishedEvent((TaskFinishedEvent) event);
  } else if (event instanceof TaskStartedEvent) {
    processTaskStartedEvent((TaskStartedEvent) event);
  } else if (event instanceof TaskUpdatedEvent) {
    processTaskUpdatedEvent((TaskUpdatedEvent) event);
  } else
    throw new IllegalArgumentException(
        "JobBuilder.process(HistoryEvent): unknown event type:"
        + event.getEventType() + " for event:" + event);
}