Java Code Examples for org.apache.hadoop.yarn.api.records.ContainerStatus#newInstance()

The following examples show how to use org.apache.hadoop.yarn.api.records.ContainerStatus#newInstance() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: ResourceTrackerService.java    From hadoop with Apache License 2.0 5 votes vote down vote up 
/**
 * Helper method to handle received ContainerStatus. If this corresponds to
 * the completion of a master-container of a managed AM,
 * we call the handler for RMAppAttemptContainerFinishedEvent.
 */
@SuppressWarnings("unchecked")
@VisibleForTesting
void handleNMContainerStatus(NMContainerStatus containerStatus, NodeId nodeId) {
  ApplicationAttemptId appAttemptId =
      containerStatus.getContainerId().getApplicationAttemptId();
  RMApp rmApp =
      rmContext.getRMApps().get(appAttemptId.getApplicationId());
  if (rmApp == null) {
    LOG.error("Received finished container : "
        + containerStatus.getContainerId()
        + " for unknown application " + appAttemptId.getApplicationId()
        + " Skipping.");
    return;
  }

  if (rmApp.getApplicationSubmissionContext().getUnmanagedAM()) {
    if (LOG.isDebugEnabled()) {
      LOG.debug("Ignoring container completion status for unmanaged AM "
          + rmApp.getApplicationId());
    }
    return;
  }

  RMAppAttempt rmAppAttempt = rmApp.getRMAppAttempt(appAttemptId);
  Container masterContainer = rmAppAttempt.getMasterContainer();
  if (masterContainer.getId().equals(containerStatus.getContainerId())
      && containerStatus.getContainerState() == ContainerState.COMPLETE) {
    ContainerStatus status =
        ContainerStatus.newInstance(containerStatus.getContainerId(),
          containerStatus.getContainerState(), containerStatus.getDiagnostics(),
          containerStatus.getContainerExitStatus());
    // sending master container finished event.
    RMAppAttemptContainerFinishedEvent evt =
        new RMAppAttemptContainerFinishedEvent(appAttemptId, status,
            nodeId);
    rmContext.getDispatcher().getEventHandler().handle(evt);
  }
}
Example 2
Source File: RMContainerImpl.java    From hadoop with Apache License 2.0 5 votes vote down vote up 
@Override
public RMContainerState transition(RMContainerImpl container,
    RMContainerEvent event) {
  NMContainerStatus report =
      ((RMContainerRecoverEvent) event).getContainerReport();
  if (report.getContainerState().equals(ContainerState.COMPLETE)) {
    ContainerStatus status =
        ContainerStatus.newInstance(report.getContainerId(),
          report.getContainerState(), report.getDiagnostics(),
          report.getContainerExitStatus());

    new FinishedTransition().transition(container,
      new RMContainerFinishedEvent(container.containerId, status,
        RMContainerEventType.FINISHED));
    return RMContainerState.COMPLETED;
  } else if (report.getContainerState().equals(ContainerState.RUNNING)) {
    // Tell the app
    container.eventHandler.handle(new RMAppRunningOnNodeEvent(container
        .getApplicationAttemptId().getApplicationId(), container.nodeId));
    return RMContainerState.RUNNING;
  } else {
    // This can never happen.
    LOG.warn("RMContainer received unexpected recover event with container"
        + " state " + report.getContainerState() + " while recovering.");
    return RMContainerState.RUNNING;
  }
}
Example 3
Source File: RMNodeImpl.java    From hadoop with Apache License 2.0 5 votes vote down vote up 
private ContainerStatus createContainerStatus(
    NMContainerStatus remoteContainer) {
  ContainerStatus cStatus =
      ContainerStatus.newInstance(remoteContainer.getContainerId(),
          remoteContainer.getContainerState(),
          remoteContainer.getDiagnostics(),
          remoteContainer.getContainerExitStatus());
  return cStatus;
}
Example 4
Source File: TestRMAppAttemptTransitions.java    From hadoop with Apache License 2.0 5 votes vote down vote up 
@SuppressWarnings("deprecation")
@Test
public void testContainersCleanupForLastAttempt() {
  // create a failed attempt.
  applicationAttempt =
      new RMAppAttemptImpl(applicationAttempt.getAppAttemptId(), spyRMContext,
        scheduler, masterService, submissionContext, new Configuration(),
        true, BuilderUtils.newResourceRequest(
            RMAppAttemptImpl.AM_CONTAINER_PRIORITY, ResourceRequest.ANY,
            submissionContext.getResource(), 1));
  when(submissionContext.getKeepContainersAcrossApplicationAttempts())
    .thenReturn(true);
  when(submissionContext.getMaxAppAttempts()).thenReturn(1);
  Container amContainer = allocateApplicationAttempt();
  launchApplicationAttempt(amContainer);
  runApplicationAttempt(amContainer, "host", 8042, "oldtrackingurl", false);
  ContainerStatus cs1 =
      ContainerStatus.newInstance(amContainer.getId(),
        ContainerState.COMPLETE, "some error", 123);
  ApplicationAttemptId appAttemptId = applicationAttempt.getAppAttemptId();
  NodeId anyNodeId = NodeId.newInstance("host", 1234);
  applicationAttempt.handle(new RMAppAttemptContainerFinishedEvent(
    appAttemptId, cs1, anyNodeId));
  assertEquals(YarnApplicationAttemptState.RUNNING,
      applicationAttempt.createApplicationAttemptState());
  sendAttemptUpdateSavedEvent(applicationAttempt);
  assertEquals(RMAppAttemptState.FAILED,
    applicationAttempt.getAppAttemptState());
  assertFalse(transferStateFromPreviousAttempt);
  verifyApplicationAttemptFinished(RMAppAttemptState.FAILED);
}
Example 5
Source File: ResourceTrackerService.java    From big-c with Apache License 2.0 5 votes vote down vote up 
/**
 * Helper method to handle received ContainerStatus. If this corresponds to
 * the completion of a master-container of a managed AM,
 * we call the handler for RMAppAttemptContainerFinishedEvent.
 */
@SuppressWarnings("unchecked")
@VisibleForTesting
void handleNMContainerStatus(NMContainerStatus containerStatus, NodeId nodeId) {
  ApplicationAttemptId appAttemptId =
      containerStatus.getContainerId().getApplicationAttemptId();
  RMApp rmApp =
      rmContext.getRMApps().get(appAttemptId.getApplicationId());
  if (rmApp == null) {
    LOG.error("Received finished container : "
        + containerStatus.getContainerId()
        + " for unknown application " + appAttemptId.getApplicationId()
        + " Skipping.");
    return;
  }

  if (rmApp.getApplicationSubmissionContext().getUnmanagedAM()) {
    if (LOG.isDebugEnabled()) {
      LOG.debug("Ignoring container completion status for unmanaged AM "
          + rmApp.getApplicationId());
    }
    return;
  }

  RMAppAttempt rmAppAttempt = rmApp.getRMAppAttempt(appAttemptId);
  Container masterContainer = rmAppAttempt.getMasterContainer();
  if (masterContainer.getId().equals(containerStatus.getContainerId())
      && containerStatus.getContainerState() == ContainerState.COMPLETE) {
    ContainerStatus status =
        ContainerStatus.newInstance(containerStatus.getContainerId(),
          containerStatus.getContainerState(), containerStatus.getDiagnostics(),
          containerStatus.getContainerExitStatus());
    // sending master container finished event.
    RMAppAttemptContainerFinishedEvent evt =
        new RMAppAttemptContainerFinishedEvent(appAttemptId, status,
            nodeId);
    rmContext.getDispatcher().getEventHandler().handle(evt);
  }
}
Example 6
Source File: RMContainerImpl.java    From big-c with Apache License 2.0 5 votes vote down vote up 
@Override
public RMContainerState transition(RMContainerImpl container,
    RMContainerEvent event) {
  NMContainerStatus report =
      ((RMContainerRecoverEvent) event).getContainerReport();
  if (report.getContainerState().equals(ContainerState.COMPLETE)) {
    ContainerStatus status =
        ContainerStatus.newInstance(report.getContainerId(),
          report.getContainerState(), report.getDiagnostics(),
          report.getContainerExitStatus());

    new FinishedTransition().transition(container,
      new RMContainerFinishedEvent(container.containerId, status,
        RMContainerEventType.FINISHED));
    return RMContainerState.COMPLETED;
  } else if (report.getContainerState().equals(ContainerState.RUNNING)) {
    // Tell the app
    container.eventHandler.handle(new RMAppRunningOnNodeEvent(container
        .getApplicationAttemptId().getApplicationId(), container.nodeId));
    return RMContainerState.RUNNING;
  } else {
    // This can never happen.
    LOG.warn("RMContainer received unexpected recover event with container"
        + " state " + report.getContainerState() + " while recovering.");
    return RMContainerState.RUNNING;
  }
}
Example 7
Source File: RMNodeImpl.java    From big-c with Apache License 2.0 5 votes vote down vote up 
private ContainerStatus createContainerStatus(
    NMContainerStatus remoteContainer) {
  ContainerStatus cStatus =
      ContainerStatus.newInstance(remoteContainer.getContainerId(),
          remoteContainer.getContainerState(),
          remoteContainer.getDiagnostics(),
          remoteContainer.getContainerExitStatus());
  return cStatus;
}
Example 8
Source File: TestRMAppAttemptTransitions.java    From big-c with Apache License 2.0 5 votes vote down vote up 
@SuppressWarnings("deprecation")
@Test
public void testContainersCleanupForLastAttempt() {
  // create a failed attempt.
  applicationAttempt =
      new RMAppAttemptImpl(applicationAttempt.getAppAttemptId(), spyRMContext,
        scheduler, masterService, submissionContext, new Configuration(),
        true, BuilderUtils.newResourceRequest(
            RMAppAttemptImpl.AM_CONTAINER_PRIORITY, ResourceRequest.ANY,
            submissionContext.getResource(), 1));
  when(submissionContext.getKeepContainersAcrossApplicationAttempts())
    .thenReturn(true);
  when(submissionContext.getMaxAppAttempts()).thenReturn(1);
  Container amContainer = allocateApplicationAttempt();
  launchApplicationAttempt(amContainer);
  runApplicationAttempt(amContainer, "host", 8042, "oldtrackingurl", false);
  ContainerStatus cs1 =
      ContainerStatus.newInstance(amContainer.getId(),
        ContainerState.COMPLETE, "some error", 123);
  ApplicationAttemptId appAttemptId = applicationAttempt.getAppAttemptId();
  NodeId anyNodeId = NodeId.newInstance("host", 1234);
  applicationAttempt.handle(new RMAppAttemptContainerFinishedEvent(
    appAttemptId, cs1, anyNodeId));
  assertEquals(YarnApplicationAttemptState.RUNNING,
      applicationAttempt.createApplicationAttemptState());
  sendAttemptUpdateSavedEvent(applicationAttempt);
  assertEquals(RMAppAttemptState.FAILED,
    applicationAttempt.getAppAttemptState());
  assertFalse(transferStateFromPreviousAttempt);
  verifyApplicationAttemptFinished(RMAppAttemptState.FAILED);
}
Example 9
Source File: TestRMAppAttemptTransitions.java    From hadoop with Apache License 2.0 4 votes vote down vote up 
@Test
public void testFailedToFailed() {
  // create a failed attempt.
  when(submissionContext.getKeepContainersAcrossApplicationAttempts())
    .thenReturn(true);
  Container amContainer = allocateApplicationAttempt();
  launchApplicationAttempt(amContainer);
  runApplicationAttempt(amContainer, "host", 8042, "oldtrackingurl", false);
  ContainerStatus cs1 =
      ContainerStatus.newInstance(amContainer.getId(),
        ContainerState.COMPLETE, "some error", 123);
  ApplicationAttemptId appAttemptId = applicationAttempt.getAppAttemptId();
  NodeId anyNodeId = NodeId.newInstance("host", 1234);
  applicationAttempt.handle(new RMAppAttemptContainerFinishedEvent(
    appAttemptId, cs1, anyNodeId));
  assertEquals(YarnApplicationAttemptState.RUNNING,
      applicationAttempt.createApplicationAttemptState());
  sendAttemptUpdateSavedEvent(applicationAttempt);
  assertEquals(RMAppAttemptState.FAILED,
    applicationAttempt.getAppAttemptState());
  // should not kill containers when attempt fails.
  assertTrue(transferStateFromPreviousAttempt);
  verifyApplicationAttemptFinished(RMAppAttemptState.FAILED);

  // failed attempt captured the container finished event.
  assertEquals(0, applicationAttempt.getJustFinishedContainers().size());
  ContainerStatus cs2 =
      ContainerStatus.newInstance(ContainerId.newContainerId(appAttemptId, 2),
        ContainerState.COMPLETE, "", 0);
  applicationAttempt.handle(new RMAppAttemptContainerFinishedEvent(
    appAttemptId, cs2, anyNodeId));
  assertEquals(1, applicationAttempt.getJustFinishedContainers().size());
  boolean found = false;
  for (ContainerStatus containerStatus:applicationAttempt
      .getJustFinishedContainers()) {
    if (cs2.getContainerId().equals(containerStatus.getContainerId())) {
      found = true;
    }
  }
  assertTrue(found);
}
Example 10
Source File: TestRMContainerAllocator.java    From hadoop with Apache License 2.0 4 votes vote down vote up 
@Test
public void testCompletedContainerEvent() {
  RMContainerAllocator allocator = new RMContainerAllocator(
      mock(ClientService.class), mock(AppContext.class));
  
  TaskAttemptId attemptId = MRBuilderUtils.newTaskAttemptId(
      MRBuilderUtils.newTaskId(
          MRBuilderUtils.newJobId(1, 1, 1), 1, TaskType.MAP), 1);
  ApplicationId applicationId = ApplicationId.newInstance(1, 1);
  ApplicationAttemptId applicationAttemptId = ApplicationAttemptId.newInstance(
      applicationId, 1);
  ContainerId containerId = ContainerId.newContainerId(applicationAttemptId, 1);
  ContainerStatus status = ContainerStatus.newInstance(
      containerId, ContainerState.RUNNING, "", 0);

  ContainerStatus abortedStatus = ContainerStatus.newInstance(
      containerId, ContainerState.RUNNING, "",
      ContainerExitStatus.ABORTED);
  
  TaskAttemptEvent event = allocator.createContainerFinishedEvent(status,
      attemptId);
  Assert.assertEquals(TaskAttemptEventType.TA_CONTAINER_COMPLETED,
      event.getType());
  
  TaskAttemptEvent abortedEvent = allocator.createContainerFinishedEvent(
      abortedStatus, attemptId);
  Assert.assertEquals(TaskAttemptEventType.TA_KILL, abortedEvent.getType());
  
  ContainerId containerId2 = ContainerId.newContainerId(applicationAttemptId, 2);
  ContainerStatus status2 = ContainerStatus.newInstance(containerId2,
      ContainerState.RUNNING, "", 0);

  ContainerStatus preemptedStatus = ContainerStatus.newInstance(containerId2,
      ContainerState.RUNNING, "", ContainerExitStatus.PREEMPTED);

  TaskAttemptEvent event2 = allocator.createContainerFinishedEvent(status2,
      attemptId);
  Assert.assertEquals(TaskAttemptEventType.TA_CONTAINER_COMPLETED,
      event2.getType());

  TaskAttemptEvent abortedEvent2 = allocator.createContainerFinishedEvent(
      preemptedStatus, attemptId);
  Assert.assertEquals(TaskAttemptEventType.TA_KILL, abortedEvent2.getType());
}
Example 11
Source File: TestRMAppAttemptTransitions.java    From big-c with Apache License 2.0 4 votes vote down vote up 
@Test
public void testFailedToFailed() {
  // create a failed attempt.
  when(submissionContext.getKeepContainersAcrossApplicationAttempts())
    .thenReturn(true);
  Container amContainer = allocateApplicationAttempt();
  launchApplicationAttempt(amContainer);
  runApplicationAttempt(amContainer, "host", 8042, "oldtrackingurl", false);
  ContainerStatus cs1 =
      ContainerStatus.newInstance(amContainer.getId(),
        ContainerState.COMPLETE, "some error", 123);
  ApplicationAttemptId appAttemptId = applicationAttempt.getAppAttemptId();
  NodeId anyNodeId = NodeId.newInstance("host", 1234);
  applicationAttempt.handle(new RMAppAttemptContainerFinishedEvent(
    appAttemptId, cs1, anyNodeId));
  assertEquals(YarnApplicationAttemptState.RUNNING,
      applicationAttempt.createApplicationAttemptState());
  sendAttemptUpdateSavedEvent(applicationAttempt);
  assertEquals(RMAppAttemptState.FAILED,
    applicationAttempt.getAppAttemptState());
  // should not kill containers when attempt fails.
  assertTrue(transferStateFromPreviousAttempt);
  verifyApplicationAttemptFinished(RMAppAttemptState.FAILED);

  // failed attempt captured the container finished event.
  assertEquals(0, applicationAttempt.getJustFinishedContainers().size());
  ContainerStatus cs2 =
      ContainerStatus.newInstance(ContainerId.newContainerId(appAttemptId, 2),
        ContainerState.COMPLETE, "", 0);
  applicationAttempt.handle(new RMAppAttemptContainerFinishedEvent(
    appAttemptId, cs2, anyNodeId));
  assertEquals(1, applicationAttempt.getJustFinishedContainers().size());
  boolean found = false;
  for (ContainerStatus containerStatus:applicationAttempt
      .getJustFinishedContainers()) {
    if (cs2.getContainerId().equals(containerStatus.getContainerId())) {
      found = true;
    }
  }
  assertTrue(found);
}
Example 12
Source File: TestRMContainerAllocator.java    From big-c with Apache License 2.0 4 votes vote down vote up 
@Test
public void testCompletedContainerEvent() {
  RMContainerAllocator allocator = new RMContainerAllocator(
      mock(ClientService.class), mock(AppContext.class));
  
  TaskAttemptId attemptId = MRBuilderUtils.newTaskAttemptId(
      MRBuilderUtils.newTaskId(
          MRBuilderUtils.newJobId(1, 1, 1), 1, TaskType.MAP), 1);
  ApplicationId applicationId = ApplicationId.newInstance(1, 1);
  ApplicationAttemptId applicationAttemptId = ApplicationAttemptId.newInstance(
      applicationId, 1);
  ContainerId containerId = ContainerId.newContainerId(applicationAttemptId, 1);
  ContainerStatus status = ContainerStatus.newInstance(
      containerId, ContainerState.RUNNING, "", 0);

  ContainerStatus abortedStatus = ContainerStatus.newInstance(
      containerId, ContainerState.RUNNING, "",
      ContainerExitStatus.ABORTED);
  
  TaskAttemptEvent event = allocator.createContainerFinishedEvent(status,
      attemptId);
  Assert.assertEquals(TaskAttemptEventType.TA_CONTAINER_COMPLETED,
      event.getType());
  
  TaskAttemptEvent abortedEvent = allocator.createContainerFinishedEvent(
      abortedStatus, attemptId);
  Assert.assertEquals(TaskAttemptEventType.TA_KILL, abortedEvent.getType());
  
  ContainerId containerId2 = ContainerId.newContainerId(applicationAttemptId, 2);
  ContainerStatus status2 = ContainerStatus.newInstance(containerId2,
      ContainerState.RUNNING, "", 0);

  ContainerStatus preemptedStatus = ContainerStatus.newInstance(containerId2,
      ContainerState.RUNNING, "", ContainerExitStatus.PREEMPTED);

  TaskAttemptEvent event2 = allocator.createContainerFinishedEvent(status2,
      attemptId);
  Assert.assertEquals(TaskAttemptEventType.TA_CONTAINER_COMPLETED,
      event2.getType());

  TaskAttemptEvent abortedEvent2 = allocator.createContainerFinishedEvent(
      preemptedStatus, attemptId);
  Assert.assertEquals(TaskAttemptEventType.TA_KILL, abortedEvent2.getType());
}