org.apache.hadoop.yarn.api.records.ContainerStatus Java Examples

private String getAMContainerCrashedDiagnostics(
    RMAppAttemptContainerFinishedEvent finishEvent) {
  ContainerStatus status = finishEvent.getContainerStatus();
  StringBuilder diagnosticsBuilder = new StringBuilder();
  diagnosticsBuilder.append("AM Container for ").append(
    finishEvent.getApplicationAttemptId()).append(
    " exited with ").append(" exitCode: ").append(status.getExitStatus()).
    append("\n");
  if (this.getTrackingUrl() != null) {
    diagnosticsBuilder.append("For more detailed output,").append(
      " check application tracking page:").append(
      this.getTrackingUrl()).append(
      "Then, click on links to logs of each attempt.\n");
  }
  diagnosticsBuilder.append("Diagnostics: ").append(status.getDiagnostics())
      .append("Failing this attempt");
  return diagnosticsBuilder.toString();
}
 

@Override
public ContainerStatus getContainerStatus(ContainerId containerId,
    NodeId nodeId) throws YarnException, IOException {

  ContainerManagementProtocolProxyData proxy = null;
  List<ContainerId> containerIds = new ArrayList<ContainerId>();
  containerIds.add(containerId);
  try {
    proxy = cmProxy.getProxy(nodeId.toString(), containerId);
    GetContainerStatusesResponse response =
        proxy.getContainerManagementProtocol().getContainerStatuses(
            GetContainerStatusesRequest.newInstance(containerIds));
    if (response.getFailedRequests() != null
        && response.getFailedRequests().containsKey(containerId)) {
      Throwable t =
          response.getFailedRequests().get(containerId).deSerialize();
      parseAndThrowException(t);
    }
    ContainerStatus containerStatus = response.getContainerStatuses().get(0);
    return containerStatus;
  } finally {
    if (proxy != null) {
      cmProxy.mayBeCloseProxy(proxy);
    }
  }
}
 

@Override
public synchronized void containerCompleted(Object task, ContainerStatus containerStatus) {
  // Inform the Containers about completion.
  AMContainer amContainer = appContext.getAllContainers().get(containerStatus.getContainerId());
  if (amContainer != null) {
    String message = null;
    int exitStatus = containerStatus.getExitStatus();
    if (exitStatus == ContainerExitStatus.PREEMPTED) {
      message = "Container preempted externally. ";
    } else if (exitStatus == ContainerExitStatus.DISKS_FAILED) {
      message = "Container disk failed. ";
    } else {
      message = "Container failed. ";
    }
    if (containerStatus.getDiagnostics() != null) {
      message += containerStatus.getDiagnostics();
    }
    sendEvent(new AMContainerEventCompleted(amContainer.getContainerId(), exitStatus, message));
  }
}
 

private ContainerStatus getContainerStatusInternal(ContainerId containerID,
    NMTokenIdentifier nmTokenIdentifier) throws YarnException {
  String containerIDStr = containerID.toString();
  Container container = this.context.getContainers().get(containerID);

  LOG.info("Getting container-status for " + containerIDStr);
  authorizeGetAndStopContainerRequest(containerID, container, false,
    nmTokenIdentifier);

  if (container == null) {
    if (nodeStatusUpdater.isContainerRecentlyStopped(containerID)) {
      throw RPCUtil.getRemoteException("Container " + containerIDStr
        + " was recently stopped on node manager.");
    } else {
      throw RPCUtil.getRemoteException("Container " + containerIDStr
        + " is not handled by this NodeManager");
    }
  }
  ContainerStatus containerStatus = container.cloneAndGetContainerStatus();
  LOG.info("Returning " + containerStatus);
  return containerStatus;
}
 

private NodeStatus getNodeStatus(int responseId) throws IOException {

    NodeHealthStatus nodeHealthStatus = this.context.getNodeHealthStatus();
    nodeHealthStatus.setHealthReport(healthChecker.getHealthReport());
    nodeHealthStatus.setIsNodeHealthy(healthChecker.isHealthy());
    nodeHealthStatus.setLastHealthReportTime(healthChecker
      .getLastHealthReportTime());
    if (LOG.isDebugEnabled()) {
      LOG.debug("Node's health-status : " + nodeHealthStatus.getIsNodeHealthy()
          + ", " + nodeHealthStatus.getHealthReport());
    }
    List<ContainerStatus> containersStatuses = getContainerStatuses();
    NodeStatus nodeStatus =
        NodeStatus.newInstance(nodeId, responseId, containersStatuses,
          createKeepAliveApplicationList(), nodeHealthStatus);

    return nodeStatus;
  }
 

/**
 * Get a list of container statuses running on this NodeManager
 */
@Override
public GetContainerStatusesResponse getContainerStatuses(
    GetContainerStatusesRequest request) throws YarnException, IOException {

  List<ContainerStatus> succeededRequests = new ArrayList<ContainerStatus>();
  Map<ContainerId, SerializedException> failedRequests =
      new HashMap<ContainerId, SerializedException>();
  UserGroupInformation remoteUgi = getRemoteUgi();
  NMTokenIdentifier identifier = selectNMTokenIdentifier(remoteUgi);
  for (ContainerId id : request.getContainerIds()) {
    try {
      ContainerStatus status = getContainerStatusInternal(id, identifier);
      succeededRequests.add(status);
    } catch (YarnException e) {
      failedRequests.put(id, SerializedException.newInstance(e));
    }
  }
  return GetContainerStatusesResponse.newInstance(succeededRequests,
    failedRequests);
}
 

@Override
public AllocateResponse allocate(AllocateRequest request)
    throws YarnException, IOException {
  Assert.assertEquals("response ID mismatch",
      responseId, request.getResponseId());
  ++responseId;
  org.apache.hadoop.yarn.api.records.Token yarnToken = null;
  if (amToken != null) {
    yarnToken = org.apache.hadoop.yarn.api.records.Token.newInstance(
        amToken.getIdentifier(), amToken.getKind().toString(),
        amToken.getPassword(), amToken.getService().toString());
  }
  return AllocateResponse.newInstance(responseId,
      Collections.<ContainerStatus>emptyList(),
      Collections.<Container>emptyList(),
      Collections.<NodeReport>emptyList(),
      Resources.none(), null, 1, null,
      Collections.<NMToken>emptyList(),
      yarnToken,
      Collections.<ContainerResourceIncrease>emptyList(),
      Collections.<ContainerResourceDecrease>emptyList());
}
 

public static void waitForContainerState(ContainerManagementProtocol containerManager,
      ContainerId containerID, ContainerState finalState, int timeOutMax)
      throws InterruptedException, YarnException, IOException {
List<ContainerId> list = new ArrayList<ContainerId>();
list.add(containerID);
GetContainerStatusesRequest request =
    GetContainerStatusesRequest.newInstance(list);
ContainerStatus containerStatus =
    containerManager.getContainerStatuses(request).getContainerStatuses()
      .get(0);
int timeoutSecs = 0;
  while (!containerStatus.getState().equals(finalState)
      && timeoutSecs++ < timeOutMax) {
      Thread.sleep(1000);
      LOG.info("Waiting for container to get into state " + finalState
          + ". Current state is " + containerStatus.getState());
      containerStatus = containerManager.getContainerStatuses(request).getContainerStatuses().get(0);
    }
    LOG.info("Container state is " + containerStatus.getState());
    Assert.assertEquals("ContainerState is not correct (timedout)",
        finalState, containerStatus.getState());
  }
 

private String getAMContainerCrashedDiagnostics(
    RMAppAttemptContainerFinishedEvent finishEvent) {
  ContainerStatus status = finishEvent.getContainerStatus();
  StringBuilder diagnosticsBuilder = new StringBuilder();
  diagnosticsBuilder.append("AM Container for ").append(
    finishEvent.getApplicationAttemptId()).append(
    " exited with ").append(" exitCode: ").append(status.getExitStatus()).
    append("\n");
  if (this.getTrackingUrl() != null) {
    diagnosticsBuilder.append("For more detailed output,").append(
      " check application tracking page:").append(
      this.getTrackingUrl()).append(
      "Then, click on links to logs of each attempt.\n");
  }
  diagnosticsBuilder.append("Diagnostics: ").append(status.getDiagnostics())
      .append("Failing this attempt");
  return diagnosticsBuilder.toString();
}
 

private NodeStatus getNodeStatus(int responseId) throws IOException {

    NodeHealthStatus nodeHealthStatus = this.context.getNodeHealthStatus();
    nodeHealthStatus.setHealthReport(healthChecker.getHealthReport());
    nodeHealthStatus.setIsNodeHealthy(healthChecker.isHealthy());
    nodeHealthStatus.setLastHealthReportTime(healthChecker
      .getLastHealthReportTime());
    if (LOG.isDebugEnabled()) {
      LOG.debug("Node's health-status : " + nodeHealthStatus.getIsNodeHealthy()
          + ", " + nodeHealthStatus.getHealthReport());
    }
    List<ContainerStatus> containersStatuses = getContainerStatuses();
    NodeStatus nodeStatus =
        NodeStatus.newInstance(nodeId, responseId, containersStatuses,
          createKeepAliveApplicationList(), nodeHealthStatus);

    return nodeStatus;
  }
 

private static void publishContainerEndEvent(
    final TimelineClient timelineClient, ContainerStatus container,
    String domainId, UserGroupInformation ugi) {
  final TimelineEntity entity = new TimelineEntity();
  entity.setEntityId(container.getContainerId().toString());
  entity.setEntityType(DSEntity.DS_CONTAINER.toString());
  entity.setDomainId(domainId);
  entity.addPrimaryFilter("user", ugi.getShortUserName());
  TimelineEvent event = new TimelineEvent();
  event.setTimestamp(System.currentTimeMillis());
  event.setEventType(DSEvent.DS_CONTAINER_END.toString());
  event.addEventInfo("State", container.getState().name());
  event.addEventInfo("Exit Status", container.getExitStatus());
  entity.addEvent(event);
  try {
    timelineClient.putEntities(entity);
  } catch (YarnException | IOException e) {
    LOG.error("Container end event could not be published for "
        + container.getContainerId().toString(), e);
  }
}
 

public static AllocateResponse newAllocateResponse(int responseId,
    List<ContainerStatus> completedContainers,
    List<Container> allocatedContainers, List<NodeReport> updatedNodes,
    Resource availResources, AMCommand command, int numClusterNodes,
    PreemptionMessage preempt) {
  AllocateResponse response = recordFactory
      .newRecordInstance(AllocateResponse.class);
  response.setNumClusterNodes(numClusterNodes);
  response.setResponseId(responseId);
  response.setCompletedContainersStatuses(completedContainers);
  response.setAllocatedContainers(allocatedContainers);
  response.setUpdatedNodes(updatedNodes);
  response.setAvailableResources(availResources);
  response.setAMCommand(command);
  response.setPreemptionMessage(preempt);

  return response;
}
 

@Override
public void onContainersCompleted(final List<ContainerStatus> statuses) {
	runAsync(() -> {
			log.debug("YARN ResourceManager reported the following containers completed: {}.", statuses);
			for (final ContainerStatus containerStatus : statuses) {

				final ResourceID resourceId = new ResourceID(containerStatus.getContainerId().toString());
				final YarnWorkerNode yarnWorkerNode = workerNodeMap.remove(resourceId);

				if (yarnWorkerNode != null) {
					// Container completed unexpectedly ~> start a new one
					requestYarnContainerIfRequired();
				}
				// Eagerly close the connection with task manager.
				closeTaskManagerConnection(resourceId, new Exception(containerStatus.getDiagnostics()));
			}
		}
	);
}
 

@Test
public void testNodesDefaultWithUnHealthyNode() throws JSONException,
    Exception {

  WebResource r = resource();
  MockNM nm1 = rm.registerNode("h1:1234", 5120);
  MockNM nm2 = rm.registerNode("h2:1235", 5121);
  rm.sendNodeStarted(nm1);
  rm.NMwaitForState(nm1.getNodeId(), NodeState.RUNNING);
  rm.NMwaitForState(nm2.getNodeId(), NodeState.NEW);

  MockNM nm3 = rm.registerNode("h3:1236", 5122);
  rm.NMwaitForState(nm3.getNodeId(), NodeState.NEW);
  rm.sendNodeStarted(nm3);
  rm.NMwaitForState(nm3.getNodeId(), NodeState.RUNNING);
  RMNodeImpl node = (RMNodeImpl) rm.getRMContext().getRMNodes()
      .get(nm3.getNodeId());
  NodeHealthStatus nodeHealth = NodeHealthStatus.newInstance(false,
      "test health report", System.currentTimeMillis());
  node.handle(new RMNodeStatusEvent(nm3.getNodeId(), nodeHealth,
      new ArrayList<ContainerStatus>(), null, null));
  rm.NMwaitForState(nm3.getNodeId(), NodeState.UNHEALTHY);

  ClientResponse response =
      r.path("ws").path("v1").path("cluster").path("nodes")
        .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);

  assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
  JSONObject json = response.getEntity(JSONObject.class);
  assertEquals("incorrect number of elements", 1, json.length());
  JSONObject nodes = json.getJSONObject("nodes");
  assertEquals("incorrect number of elements", 1, nodes.length());
  JSONArray nodeArray = nodes.getJSONArray("node");
  // 3 nodes, including the unhealthy node and the new node.
  assertEquals("incorrect number of elements", 3, nodeArray.length());
}
 

@Test (timeout = 5000)
public void testContainerPreempted() throws IOException {
  Configuration conf = new Configuration(false);
  schedulerHandler.init(conf);
  schedulerHandler.start();
  
  String diagnostics = "Container preempted by RM.";
  TaskAttemptImpl mockTask = mock(TaskAttemptImpl.class);
  ContainerStatus mockStatus = mock(ContainerStatus.class);
  ContainerId mockCId = mock(ContainerId.class);
  AMContainer mockAMContainer = mock(AMContainer.class);
  when(mockAMContainerMap.get(mockCId)).thenReturn(mockAMContainer);
  when(mockAMContainer.getContainerId()).thenReturn(mockCId);
  when(mockStatus.getContainerId()).thenReturn(mockCId);
  when(mockStatus.getDiagnostics()).thenReturn(diagnostics);
  when(mockStatus.getExitStatus()).thenReturn(ContainerExitStatus.PREEMPTED);
  schedulerHandler.containerCompleted(0, mockTask, mockStatus);
  assertEquals(1, mockEventHandler.events.size());
  Event event = mockEventHandler.events.get(0);
  assertEquals(AMContainerEventType.C_COMPLETED, event.getType());
  AMContainerEventCompleted completedEvent = (AMContainerEventCompleted) event;
  assertEquals(mockCId, completedEvent.getContainerId());
  assertEquals("Container preempted externally. Container preempted by RM.",
      completedEvent.getDiagnostics());
  assertTrue(completedEvent.isPreempted());
  assertEquals(TaskAttemptTerminationCause.EXTERNAL_PREEMPTION,
      completedEvent.getTerminationCause());
  Assert.assertFalse(completedEvent.isDiskFailed());

  schedulerHandler.stop();
  schedulerHandler.close();
}
 

@Test (timeout = 5000)
public void testContainerDiskFailed() throws IOException {
  Configuration conf = new Configuration(false);
  schedulerHandler.init(conf);
  schedulerHandler.start();
  
  String diagnostics = "NM disk failed.";
  TaskAttemptImpl mockTask = mock(TaskAttemptImpl.class);
  ContainerStatus mockStatus = mock(ContainerStatus.class);
  ContainerId mockCId = mock(ContainerId.class);
  AMContainer mockAMContainer = mock(AMContainer.class);
  when(mockAMContainerMap.get(mockCId)).thenReturn(mockAMContainer);
  when(mockAMContainer.getContainerId()).thenReturn(mockCId);
  when(mockStatus.getContainerId()).thenReturn(mockCId);
  when(mockStatus.getDiagnostics()).thenReturn(diagnostics);
  when(mockStatus.getExitStatus()).thenReturn(ContainerExitStatus.DISKS_FAILED);
  schedulerHandler.containerCompleted(mockTask, mockStatus);
  Assert.assertEquals(1, mockEventHandler.events.size());
  Event event = mockEventHandler.events.get(0);
  Assert.assertEquals(AMContainerEventType.C_COMPLETED, event.getType());
  AMContainerEventCompleted completedEvent = (AMContainerEventCompleted) event;
  Assert.assertEquals(mockCId, completedEvent.getContainerId());
  Assert.assertEquals("Container disk failed. NM disk failed.", 
      completedEvent.getDiagnostics());
  Assert.assertFalse(completedEvent.isPreempted());
  Assert.assertTrue(completedEvent.isDiskFailed());

  schedulerHandler.stop();
  schedulerHandler.close();
}
 

public static org.apache.hadoop.yarn.server.api.records.NodeStatus 
createNodeStatus(NodeId nodeId, List<ContainerStatus> containers) {
  RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null);
  org.apache.hadoop.yarn.server.api.records.NodeStatus nodeStatus = 
      recordFactory.newRecordInstance(org.apache.hadoop.yarn.server.api.records.NodeStatus.class);
  nodeStatus.setNodeId(nodeId);
  nodeStatus.setContainersStatuses(containers);
  NodeHealthStatus nodeHealthStatus = 
    recordFactory.newRecordInstance(NodeHealthStatus.class);
  nodeHealthStatus.setIsNodeHealthy(true);
  nodeStatus.setNodeHealthStatus(nodeHealthStatus);
  return nodeStatus;
}
 

public NodeHeartbeatResponse nodeHeartbeat(ApplicationAttemptId attemptId,
    long containerId, ContainerState containerState) throws Exception {
  HashMap<ApplicationId, List<ContainerStatus>> nodeUpdate =
      new HashMap<ApplicationId, List<ContainerStatus>>(1);
  ContainerStatus containerStatus = BuilderUtils.newContainerStatus(
      BuilderUtils.newContainerId(attemptId, containerId), containerState,
      "Success", 0);
  ArrayList<ContainerStatus> containerStatusList =
      new ArrayList<ContainerStatus>(1);
  containerStatusList.add(containerStatus);
  Log.info("ContainerStatus: " + containerStatus);
  nodeUpdate.put(attemptId.getApplicationId(), containerStatusList);
  return nodeHeartbeat(nodeUpdate, true);
}
 

public static ContainerStatus newContainerStatus(ContainerId containerId,
    ContainerState containerState, String diagnostics, int exitStatus) {
  ContainerStatus containerStatus = recordFactory
    .newRecordInstance(ContainerStatus.class);
  containerStatus.setState(containerState);
  containerStatus.setContainerId(containerId);
  containerStatus.setDiagnostics(diagnostics);
  containerStatus.setExitStatus(exitStatus);
  return containerStatus;
}
 

public void waitForContainerToComplete(RMAppAttempt attempt,
    NMContainerStatus completedContainer) throws InterruptedException {
  while (true) {
    List<ContainerStatus> containers = attempt.getJustFinishedContainers();
    System.out.println("Received completed containers " + containers);
    for (ContainerStatus container : containers) {
      if (container.getContainerId().equals(
        completedContainer.getContainerId())) {
        return;
      }
    }
    Thread.sleep(200);
  }
}
 

public void waitForContainerToComplete(RMAppAttempt attempt,
    NMContainerStatus completedContainer) throws InterruptedException {
  while (true) {
    List<ContainerStatus> containers = attempt.getJustFinishedContainers();
    System.out.println("Received completed containers " + containers);
    for (ContainerStatus container : containers) {
      if (container.getContainerId().equals(
        completedContainer.getContainerId())) {
        return;
      }
    }
    Thread.sleep(200);
  }
}
 

@Override
public void onContainerStatusReceived(ContainerId containerId, ContainerStatus containerStatus)
{
  LOG.debug("Container Status: id={}, status={}", containerId, containerStatus);
  if (containerStatus.getState() != ContainerState.RUNNING) {
    recoverContainer(containerId);
  }
}
 

private ContainerStatus newContainerStatus(ContainerId cId, 
                                           ContainerState state,
                                           int exitState) {
  ContainerStatus cs = Records.newRecord(ContainerStatus.class);
  cs.setContainerId(cId);
  cs.setState(state);
  cs.setExitStatus(exitState);
  return cs;
}
 

@Override
public GetContainerStatusesResponse getContainerStatuses(
    GetContainerStatusesRequest request) throws IOException {
  List<ContainerStatus> statuses = new ArrayList<ContainerStatus>();
  statuses.add(status);
  return GetContainerStatusesResponse.newInstance(statuses, null);
}
 

public void onContainersCompleted(List<ContainerStatus> containerStatuses) {
  for (ContainerStatus status : containerStatuses) {
    log.info("Completed container " + status.getContainerId());
    synchronized (this) {
      numContainersToWaitFor--;
    }
  }
}
 

private void initLocalContainerStatuses() {
  if (this.containerStatuses != null) {
    return;
  }
  GetContainerStatusesResponseProtoOrBuilder p = viaProto ? proto : builder;
  List<ContainerStatusProto> statuses = p.getStatusList();
  this.containerStatuses = new ArrayList<ContainerStatus>();
  for (ContainerStatusProto status : statuses) {
    this.containerStatuses.add(convertFromProtoFormat(status));
  }
}
 

public static ContainerStatus createContainerStatus(int id,
    ContainerState containerState) {
  ApplicationId applicationId = ApplicationId.newInstance(0, 1);
  ApplicationAttemptId applicationAttemptId =
      ApplicationAttemptId.newInstance(applicationId, 1);
  ContainerId contaierId = ContainerId.newContainerId(applicationAttemptId, id);
  ContainerStatus containerStatus =
      BuilderUtils.newContainerStatus(contaierId, containerState,
        "test_containerStatus: id=" + id + ", containerState: "
            + containerState, 0);
  return containerStatus;
}
 

@VisibleForTesting
protected List<ContainerStatus> getContainerStatuses() throws IOException {
  List<ContainerStatus> containerStatuses = new ArrayList<ContainerStatus>();
  for (Container container : this.context.getContainers().values()) {
    ContainerId containerId = container.getContainerId();
    ApplicationId applicationId = containerId.getApplicationAttemptId()
        .getApplicationId();
    org.apache.hadoop.yarn.api.records.ContainerStatus containerStatus =
        container.cloneAndGetContainerStatus();
    if (containerStatus.getState() == ContainerState.COMPLETE) {
      if (isApplicationStopped(applicationId)) {
        if (LOG.isDebugEnabled()) {
          LOG.debug(applicationId + " is completing, " + " remove "
              + containerId + " from NM context.");
        }
        context.getContainers().remove(containerId);
        pendingCompletedContainers.put(containerId, containerStatus);
      } else {
        if (!isContainerRecentlyStopped(containerId)) {
          pendingCompletedContainers.put(containerId, containerStatus);
          // Adding to finished containers cache. Cache will keep it around at
          // least for #durationToTrackStoppedContainers duration. In the
          // subsequent call to stop container it will get removed from cache.
          addCompletedContainer(containerId);
        }
      }
    } else {
      containerStatuses.add(containerStatus);
    }
  }
  containerStatuses.addAll(pendingCompletedContainers.values());
  if (LOG.isDebugEnabled()) {
    LOG.debug("Sending out " + containerStatuses.size()
        + " container statuses: " + containerStatuses);
  }
  return containerStatuses;
}
 

private void addLocalContainerStatusesToProto() {
  maybeInitBuilder();
  builder.clearStatus();
  if (this.containerStatuses == null)
    return;
  List<ContainerStatusProto> protoList =
      new ArrayList<ContainerStatusProto>();
  for (ContainerStatus status : containerStatuses) {
    protoList.add(convertToProtoFormat(status));
  }
  builder.addAllStatus(protoList);
}
 

private ContainerStatus getContainerStatus(int applicationId,
    int containerID, int appAttemptId) {
  ContainerStatus status = recordFactory
      .newRecordInstance(ContainerStatus.class);
  status.setContainerId(getContainerId(containerID, appAttemptId));
  return status;
}