Java Code Examples for org.apache.hadoop.yarn.api.records.ContainerState#RUNNING

The following examples show how to use org.apache.hadoop.yarn.api.records.ContainerState#RUNNING . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: NMClientImpl.java    From hadoop with Apache License 2.0 6 votes vote down vote up 
@Override
public void stopContainer(ContainerId containerId, NodeId nodeId)
    throws YarnException, IOException {
  StartedContainer startedContainer = getStartedContainer(containerId);

  // Only allow one request of stopping the container to move forward
  // When entering the block, check whether the precursor has already stopped
  // the container
  if (startedContainer != null) {
    synchronized (startedContainer) {
      if (startedContainer.state != ContainerState.RUNNING) {
        return;
      }
      stopContainerInternal(containerId, nodeId);
      // Only after successful
      startedContainer.state = ContainerState.COMPLETE;
      removeStartedContainer(startedContainer);
    }
  } else {
    stopContainerInternal(containerId, nodeId);
  }

}
Example 2
Source File: NMClientImpl.java    From big-c with Apache License 2.0 6 votes vote down vote up 
@Override
public void stopContainer(ContainerId containerId, NodeId nodeId)
    throws YarnException, IOException {
  StartedContainer startedContainer = getStartedContainer(containerId);

  // Only allow one request of stopping the container to move forward
  // When entering the block, check whether the precursor has already stopped
  // the container
  if (startedContainer != null) {
    synchronized (startedContainer) {
      if (startedContainer.state != ContainerState.RUNNING) {
        return;
      }
      stopContainerInternal(containerId, nodeId);
      // Only after successful
      startedContainer.state = ContainerState.COMPLETE;
      removeStartedContainer(startedContainer);
    }
  } else {
    stopContainerInternal(containerId, nodeId);
  }

}
Example 3
Source File: YarnJobValidationTool.java    From samza with Apache License 2.0 6 votes vote down vote up 
public int validateContainerCount(ApplicationAttemptId attemptId) throws Exception {
  int runningContainerCount = 0;
  for (ContainerReport containerReport : this.client.getContainers(attemptId)) {
    if (containerReport.getContainerState() == ContainerState.RUNNING) {
      ++runningContainerCount;
    }
  }
  // expected containers to be the configured job containers plus the AppMaster container
  int containerExpected = this.config.getContainerCount() + 1;

  if (runningContainerCount == containerExpected) {
    log.info("Container count matches. " + runningContainerCount + " containers are running.");
    return runningContainerCount;
  } else {
    throw new SamzaException("Container count does not match. " + runningContainerCount + " containers are running, while " + containerExpected + " is expected.");
  }
}
Example 4
Source File: StreamingAppMasterService.java    From Bats with Apache License 2.0 5 votes vote down vote up 
@Override
public void onContainerStatusReceived(ContainerId containerId, ContainerStatus containerStatus)
{
  LOG.debug("Container Status: id={}, status={}", containerId, containerStatus);
  if (containerStatus.getState() != ContainerState.RUNNING) {
    recoverContainer(containerId);
  }
}
Example 5
Source File: RMContainerImpl.java    From hadoop with Apache License 2.0 5 votes vote down vote up 
@Override
public ContainerState getContainerState() {
  try {
    readLock.lock();
    if (getFinishedStatus() != null) {
      return getFinishedStatus().getState();
    } else {
      return ContainerState.RUNNING;
    }
  } finally {
    readLock.unlock();
  }
}
Example 6
Source File: RMNodeImpl.java    From hadoop with Apache License 2.0 5 votes vote down vote up 
@Override
public void transition(RMNodeImpl rmNode, RMNodeEvent event) {
  // Inform the scheduler
  RMNodeStartedEvent startEvent = (RMNodeStartedEvent) event;
  List<NMContainerStatus> containers = null;

  String host = rmNode.nodeId.getHost();
  if (rmNode.context.getInactiveRMNodes().containsKey(host)) {
    // Old node rejoining
    RMNode previouRMNode = rmNode.context.getInactiveRMNodes().get(host);
    rmNode.context.getInactiveRMNodes().remove(host);
    rmNode.updateMetricsForRejoinedNode(previouRMNode.getState());
  } else {
    // Increment activeNodes explicitly because this is a new node.
    ClusterMetrics.getMetrics().incrNumActiveNodes();
    containers = startEvent.getNMContainerStatuses();
    if (containers != null && !containers.isEmpty()) {
      for (NMContainerStatus container : containers) {
        if (container.getContainerState() == ContainerState.RUNNING) {
          rmNode.launchedContainers.add(container.getContainerId());
        }
      }
    }
  }
  
  if (null != startEvent.getRunningApplications()) {
    for (ApplicationId appId : startEvent.getRunningApplications()) {
      handleRunningAppOnNode(rmNode, rmNode.context, appId, rmNode.nodeId);
    }
  }

  rmNode.context.getDispatcher().getEventHandler()
    .handle(new NodeAddedSchedulerEvent(rmNode, containers));
  rmNode.context.getDispatcher().getEventHandler().handle(
    new NodesListManagerEvent(
        NodesListManagerEventType.NODE_USABLE, rmNode));
}
Example 7
Source File: RMContainerImpl.java    From big-c with Apache License 2.0 5 votes vote down vote up 
@Override
public ContainerState getContainerState() {
  try {
    readLock.lock();
    if (getFinishedStatus() != null) {
      return getFinishedStatus().getState();
    } else {
      return ContainerState.RUNNING;
    }
  } finally {
    readLock.unlock();
  }
}
Example 8
Source File: RMNodeImpl.java    From big-c with Apache License 2.0 5 votes vote down vote up 
@Override
public void transition(RMNodeImpl rmNode, RMNodeEvent event) {
  // Inform the scheduler
  RMNodeStartedEvent startEvent = (RMNodeStartedEvent) event;
  List<NMContainerStatus> containers = null;

  String host = rmNode.nodeId.getHost();
  if (rmNode.context.getInactiveRMNodes().containsKey(host)) {
    // Old node rejoining
    RMNode previouRMNode = rmNode.context.getInactiveRMNodes().get(host);
    rmNode.context.getInactiveRMNodes().remove(host);
    rmNode.updateMetricsForRejoinedNode(previouRMNode.getState());
  } else {
    // Increment activeNodes explicitly because this is a new node.
    ClusterMetrics.getMetrics().incrNumActiveNodes();
    containers = startEvent.getNMContainerStatuses();
    if (containers != null && !containers.isEmpty()) {
      for (NMContainerStatus container : containers) {
        if (container.getContainerState() == ContainerState.RUNNING) {
          rmNode.launchedContainers.add(container.getContainerId());
        }
      }
    }
  }
  
  if (null != startEvent.getRunningApplications()) {
    for (ApplicationId appId : startEvent.getRunningApplications()) {
      handleRunningAppOnNode(rmNode, rmNode.context, appId, rmNode.nodeId);
    }
  }

  rmNode.context.getDispatcher().getEventHandler()
    .handle(new NodeAddedSchedulerEvent(rmNode, containers));
  rmNode.context.getDispatcher().getEventHandler().handle(
    new NodesListManagerEvent(
        NodesListManagerEventType.NODE_USABLE, rmNode));
}
Example 9
Source File: StreamingAppMasterService.java    From attic-apex-core with Apache License 2.0 5 votes vote down vote up 
@Override
public void onContainerStatusReceived(ContainerId containerId, ContainerStatus containerStatus)
{
  LOG.debug("Container Status: id={}, status={}", containerId, containerStatus);
  if (containerStatus.getState() != ContainerState.RUNNING) {
    recoverContainer(containerId);
  }
}
Example 10
Source File: TestNodeStatusUpdater.java    From hadoop with Apache License 2.0 4 votes vote down vote up 
@Test(timeout = 90000)
public void testRemovePreviousCompletedContainersFromContext() throws Exception {
  NodeManager nm = new NodeManager();
  YarnConfiguration conf = new YarnConfiguration();
  conf.set(
      NodeStatusUpdaterImpl
          .YARN_NODEMANAGER_DURATION_TO_TRACK_STOPPED_CONTAINERS,
      "10000");
  nm.init(conf);
  NodeStatusUpdaterImpl nodeStatusUpdater =
      (NodeStatusUpdaterImpl) nm.getNodeStatusUpdater();
  ApplicationId appId = ApplicationId.newInstance(0, 0);
  ApplicationAttemptId appAttemptId =
      ApplicationAttemptId.newInstance(appId, 0);

  ContainerId cId = ContainerId.newContainerId(appAttemptId, 1);
  Token containerToken =
      BuilderUtils.newContainerToken(cId, "anyHost", 1234, "anyUser",
          BuilderUtils.newResource(1024, 1), 0, 123,
          "password".getBytes(), 0);
  Container anyCompletedContainer = new ContainerImpl(conf, null,
      null, null, null, null,
      BuilderUtils.newContainerTokenIdentifier(containerToken)) {

    @Override
    public ContainerState getCurrentState() {
      return ContainerState.COMPLETE;
    }

    @Override
    public org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerState getContainerState() {
      return org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerState.DONE;
    }
  };

  ContainerId runningContainerId =
      ContainerId.newContainerId(appAttemptId, 3);
  Token runningContainerToken =
      BuilderUtils.newContainerToken(runningContainerId, "anyHost",
        1234, "anyUser", BuilderUtils.newResource(1024, 1), 0, 123,
        "password".getBytes(), 0);
  Container runningContainer =
      new ContainerImpl(conf, null, null, null, null, null,
        BuilderUtils.newContainerTokenIdentifier(runningContainerToken)) {
        @Override
        public ContainerState getCurrentState() {
          return ContainerState.RUNNING;
        }

        @Override
        public org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerState getContainerState() {
          return org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerState.RUNNING;
        }
      };

  nm.getNMContext().getApplications().putIfAbsent(appId,
      mock(Application.class));
  nm.getNMContext().getContainers().put(cId, anyCompletedContainer);
  nm.getNMContext().getContainers()
    .put(runningContainerId, runningContainer);

  Assert.assertEquals(2, nodeStatusUpdater.getContainerStatuses().size());

  List<ContainerId> ackedContainers = new ArrayList<ContainerId>();
  ackedContainers.add(cId);
  ackedContainers.add(runningContainerId);

  nodeStatusUpdater.removeOrTrackCompletedContainersFromContext(ackedContainers);

  Set<ContainerId> containerIdSet = new HashSet<ContainerId>();
  List<ContainerStatus> containerStatuses = nodeStatusUpdater.getContainerStatuses();
  for (ContainerStatus status : containerStatuses) {
    containerIdSet.add(status.getContainerId());
  }

  Assert.assertEquals(1, containerStatuses.size());
  // completed container is removed;
  Assert.assertFalse(containerIdSet.contains(cId));
  // running container is not removed;
  Assert.assertTrue(containerIdSet.contains(runningContainerId));
}
Example 11
Source File: RMNodeImpl.java    From hadoop with Apache License 2.0 4 votes vote down vote up 
private void handleContainerStatus(List<ContainerStatus> containerStatuses) {
  // Filter the map to only obtain just launched containers and finished
  // containers.
  List<ContainerStatus> newlyLaunchedContainers =
      new ArrayList<ContainerStatus>();
  List<ContainerStatus> completedContainers =
      new ArrayList<ContainerStatus>();
  for (ContainerStatus remoteContainer : containerStatuses) {
    ContainerId containerId = remoteContainer.getContainerId();

    // Don't bother with containers already scheduled for cleanup, or for
    // applications already killed. The scheduler doens't need to know any
    // more about this container
    if (containersToClean.contains(containerId)) {
      LOG.info("Container " + containerId + " already scheduled for "
          + "cleanup, no further processing");
      continue;
    }
    if (finishedApplications.contains(containerId.getApplicationAttemptId()
        .getApplicationId())) {
      LOG.info("Container " + containerId
          + " belongs to an application that is already killed,"
          + " no further processing");
      continue;
    }

    // Process running containers
    if (remoteContainer.getState() == ContainerState.RUNNING) {
      if (!launchedContainers.contains(containerId)) {
        // Just launched container. RM knows about it the first time.
        launchedContainers.add(containerId);
        newlyLaunchedContainers.add(remoteContainer);
      }
    } else {
      // A finished container
      launchedContainers.remove(containerId);
      completedContainers.add(remoteContainer);
    }
  }
  if (newlyLaunchedContainers.size() != 0 || completedContainers.size() != 0) {
    nodeUpdateQueue.add(new UpdatedContainerInfo(newlyLaunchedContainers,
        completedContainers));
  }
}
Example 12
Source File: TestNodeStatusUpdater.java    From big-c with Apache License 2.0 4 votes vote down vote up 
@Test(timeout = 90000)
public void testRemovePreviousCompletedContainersFromContext() throws Exception {
  NodeManager nm = new NodeManager();
  YarnConfiguration conf = new YarnConfiguration();
  conf.set(
      NodeStatusUpdaterImpl
          .YARN_NODEMANAGER_DURATION_TO_TRACK_STOPPED_CONTAINERS,
      "10000");
  nm.init(conf);
  NodeStatusUpdaterImpl nodeStatusUpdater =
      (NodeStatusUpdaterImpl) nm.getNodeStatusUpdater();
  ApplicationId appId = ApplicationId.newInstance(0, 0);
  ApplicationAttemptId appAttemptId =
      ApplicationAttemptId.newInstance(appId, 0);

  ContainerId cId = ContainerId.newContainerId(appAttemptId, 1);
  Token containerToken =
      BuilderUtils.newContainerToken(cId, "anyHost", 1234, "anyUser",
          BuilderUtils.newResource(1024, 1), 0, 123,
          "password".getBytes(), 0);
  Container anyCompletedContainer = new ContainerImpl(null, conf, null,
      null, null, null, null,
      BuilderUtils.newContainerTokenIdentifier(containerToken), null, heartBeatID, null, triggered, null) {

    @Override
    public ContainerState getCurrentState() {
      return ContainerState.COMPLETE;
    }

    @Override
    public org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerState getContainerState() {
      return org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerState.DONE;
    }
  };

  ContainerId runningContainerId =
      ContainerId.newContainerId(appAttemptId, 3);
  Token runningContainerToken =
      BuilderUtils.newContainerToken(runningContainerId, "anyHost",
        1234, "anyUser", BuilderUtils.newResource(1024, 1), 0, 123,
        "password".getBytes(), 0);
  Container runningContainer =
      new ContainerImpl(null, conf, null, null, null, null, null,
        BuilderUtils.newContainerTokenIdentifier(runningContainerToken), null, heartBeatID, null, triggered, null) {
        @Override
        public ContainerState getCurrentState() {
          return ContainerState.RUNNING;
        }

        @Override
        public org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerState getContainerState() {
          return org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerState.RUNNING;
        }
      };

  nm.getNMContext().getApplications().putIfAbsent(appId,
      mock(Application.class));
  nm.getNMContext().getContainers().put(cId, anyCompletedContainer);
  nm.getNMContext().getContainers()
    .put(runningContainerId, runningContainer);

  Assert.assertEquals(2, nodeStatusUpdater.getContainerStatuses().size());

  List<ContainerId> ackedContainers = new ArrayList<ContainerId>();
  ackedContainers.add(cId);
  ackedContainers.add(runningContainerId);

  nodeStatusUpdater.removeOrTrackCompletedContainersFromContext(ackedContainers);

  Set<ContainerId> containerIdSet = new HashSet<ContainerId>();
  List<ContainerStatus> containerStatuses = nodeStatusUpdater.getContainerStatuses();
  for (ContainerStatus status : containerStatuses) {
    containerIdSet.add(status.getContainerId());
  }

  Assert.assertEquals(1, containerStatuses.size());
  // completed container is removed;
  Assert.assertFalse(containerIdSet.contains(cId));
  // running container is not removed;
  Assert.assertTrue(containerIdSet.contains(runningContainerId));
}
Example 13
Source File: RMNodeImpl.java    From big-c with Apache License 2.0 4 votes vote down vote up 
private void handleContainerStatus(List<ContainerStatus> containerStatuses) {
  
//LOG.info("RMNode hearbeat host"+this.getHostName());
   // Filter the map to only obtain just launched containers and finished
   // containers.
   List<ContainerStatus> newlyLaunchedContainers =
       new ArrayList<ContainerStatus>();
   List<ContainerStatus> completedContainers =
       new ArrayList<ContainerStatus>();
   for (ContainerStatus remoteContainer : containerStatuses) {
     ContainerId containerId = remoteContainer.getContainerId();

     // Don't bother with containers already scheduled for cleanup, or for
     // applications already killed. The scheduler doens't need to know any
     // more about this container
     if (containersToClean.contains(containerId)) {
       LOG.info("Container " + containerId + " already scheduled for "
           + "cleanup, no further processing");
       continue;
     }
     if (finishedApplications.contains(containerId.getApplicationAttemptId()
         .getApplicationId())) {
       LOG.info("Container " + containerId
           + " belongs to an application that is already killed,"
           + " no further processing");
       continue;
     }

     // Process running containers
     if (remoteContainer.getState() == ContainerState.RUNNING) {
       if (!launchedContainers.contains(containerId)) {
         // Just launched container. RM knows about it the first time.
         launchedContainers.add(containerId);
         newlyLaunchedContainers.add(remoteContainer);
       }
     } else {
       // A finished container
       launchedContainers.remove(containerId);
       completedContainers.add(remoteContainer);
     }
   }
   if (newlyLaunchedContainers.size() != 0 || completedContainers.size() != 0) {
     nodeUpdateQueue.add(new UpdatedContainerInfo(newlyLaunchedContainers,
         completedContainers));
   }
 }
Example 14
Source File: NMHeartBeatHandler.java    From incubator-myriad with Apache License 2.0 4 votes vote down vote up 
private boolean containerInUse(ContainerStatus status) {
  return (status.getState() == ContainerState.NEW || status.getState() == ContainerState.RUNNING);
}