org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container Java Examples

The following examples show how to use org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: TestLinuxContainerExecutor.java    From big-c with Apache License 2.0 6 votes vote down vote up
private int runAndBlock(ContainerId cId, String ... cmd) throws IOException {
  String appId = "APP_"+getNextId();
  Container container = mock(Container.class);
  ContainerLaunchContext context = mock(ContainerLaunchContext.class);
  HashMap<String, String> env = new HashMap<String,String>();

  when(container.getContainerId()).thenReturn(cId);
  when(container.getLaunchContext()).thenReturn(context);

  when(context.getEnvironment()).thenReturn(env);
  
  String script = writeScriptFile(cmd);

  Path scriptPath = new Path(script);
  Path tokensPath = new Path("/dev/null");
  Path workDir = new Path(workSpace.getAbsolutePath());
  Path pidFile = new Path(workDir, "pid.txt");

  exec.activateContainer(cId, pidFile);
  return exec.launchContainer(container, scriptPath, tokensPath,
      appSubmitter, appId, workDir, dirsHandler.getLocalDirs(),
      dirsHandler.getLogDirs());
}
 
Example #2
Source File: NodeStatusUpdaterImpl.java    From big-c with Apache License 2.0 6 votes vote down vote up
private List<NMContainerStatus> getNMContainerStatuses() throws IOException {
  List<NMContainerStatus> containerStatuses =
      new ArrayList<NMContainerStatus>();
  for (Container container : this.context.getContainers().values()) {
    ContainerId containerId = container.getContainerId();
    ApplicationId applicationId = containerId.getApplicationAttemptId()
        .getApplicationId();
    if (!this.context.getApplications().containsKey(applicationId)) {
      context.getContainers().remove(containerId);
      continue;
    }
    NMContainerStatus status =
        container.getNMContainerStatus();
    containerStatuses.add(status);
    if (status.getContainerState() == ContainerState.COMPLETE) {
      // Adding to finished containers cache. Cache will keep it around at
      // least for #durationToTrackStoppedContainers duration. In the
      // subsequent call to stop container it will get removed from cache.
      addCompletedContainer(containerId);
    }
  }
  LOG.info("Sending out " + containerStatuses.size()
    + " NM container statuses: " + containerStatuses);
  return containerStatuses;
}
 
Example #3
Source File: TestNMWebServicesApps.java    From big-c with Apache License 2.0 6 votes vote down vote up
private HashMap<String, String> addAppContainers(Application app) 
    throws IOException {
  Dispatcher dispatcher = new AsyncDispatcher();
  ApplicationAttemptId appAttemptId = BuilderUtils.newApplicationAttemptId(
      app.getAppId(), 1);
  Container container1 = new MockContainer(appAttemptId, dispatcher, conf,
      app.getUser(), app.getAppId(), 1);
  Container container2 = new MockContainer(appAttemptId, dispatcher, conf,
      app.getUser(), app.getAppId(), 2);
  nmContext.getContainers()
      .put(container1.getContainerId(), container1);
  nmContext.getContainers()
      .put(container2.getContainerId(), container2);

  app.getContainers().put(container1.getContainerId(), container1);
  app.getContainers().put(container2.getContainerId(), container2);
  HashMap<String, String> hash = new HashMap<String, String>();
  hash.put(container1.getContainerId().toString(), container1
      .getContainerId().toString());
  hash.put(container2.getContainerId().toString(), container2
      .getContainerId().toString());
  return hash;
}
 
Example #4
Source File: TestNMWebServicesContainers.java    From big-c with Apache License 2.0 6 votes vote down vote up
public void verifyContainersInfoXML(NodeList nodes, Container cont)
    throws JSONException, Exception {
  for (int i = 0; i < nodes.getLength(); i++) {
    Element element = (Element) nodes.item(i);

    verifyNodeContainerInfoGeneric(cont,
        WebServicesTestUtils.getXmlString(element, "id"),
        WebServicesTestUtils.getXmlString(element, "state"),
        WebServicesTestUtils.getXmlString(element, "user"),
        WebServicesTestUtils.getXmlInt(element, "exitCode"),
        WebServicesTestUtils.getXmlString(element, "diagnostics"),
        WebServicesTestUtils.getXmlString(element, "nodeId"),
        WebServicesTestUtils.getXmlInt(element, "totalMemoryNeededMB"),
        WebServicesTestUtils.getXmlInt(element, "totalVCoresNeeded"),
        WebServicesTestUtils.getXmlString(element, "containerLogsLink"));
  }
}
 
Example #5
Source File: ContainerManagerImpl.java    From hadoop with Apache License 2.0 6 votes vote down vote up
private void waitForRecoveredContainers() throws InterruptedException {
  final int sleepMsec = 100;
  int waitIterations = 100;
  List<ContainerId> newContainers = new ArrayList<ContainerId>();
  while (--waitIterations >= 0) {
    newContainers.clear();
    for (Container container : context.getContainers().values()) {
      if (container.getContainerState() == org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerState.NEW) {
        newContainers.add(container.getContainerId());
      }
    }
    if (newContainers.isEmpty()) {
      break;
    }
    LOG.info("Waiting for containers: " + newContainers);
    Thread.sleep(sleepMsec);
  }
  if (waitIterations < 0) {
    LOG.warn("Timeout waiting for recovered containers");
  }
}
 
Example #6
Source File: ContainerLaunch.java    From hadoop with Apache License 2.0 6 votes vote down vote up
public ContainerLaunch(Context context, Configuration configuration,
    Dispatcher dispatcher, ContainerExecutor exec, Application app,
    Container container, LocalDirsHandlerService dirsHandler,
    ContainerManagerImpl containerManager) {
  this.context = context;
  this.conf = configuration;
  this.app = app;
  this.exec = exec;
  this.container = container;
  this.dispatcher = dispatcher;
  this.dirsHandler = dirsHandler;
  this.containerManager = containerManager;
  this.sleepDelayBeforeSigKill =
      conf.getLong(YarnConfiguration.NM_SLEEP_DELAY_BEFORE_SIGKILL_MS,
          YarnConfiguration.DEFAULT_NM_SLEEP_DELAY_BEFORE_SIGKILL_MS);
  this.maxKillWaitTime =
      conf.getLong(YarnConfiguration.NM_PROCESS_KILL_WAIT_MS,
          YarnConfiguration.DEFAULT_NM_PROCESS_KILL_WAIT_MS);

  this.olr = new OwnLocalResources();
}
 
Example #7
Source File: ContainerLaunch.java    From big-c with Apache License 2.0 6 votes vote down vote up
public ContainerLaunch(Context context, Configuration configuration,
    Dispatcher dispatcher, ContainerExecutor exec, Application app,
    Container container, LocalDirsHandlerService dirsHandler,
    ContainerManagerImpl containerManager) {
  this.context = context;
  this.conf = configuration;
  this.app = app;
  this.exec = exec;
  this.container = container;
  this.dispatcher = dispatcher;
  this.dirsHandler = dirsHandler;
  this.containerManager = containerManager;
  this.sleepDelayBeforeSigKill =
      conf.getLong(YarnConfiguration.NM_SLEEP_DELAY_BEFORE_SIGKILL_MS,
          YarnConfiguration.DEFAULT_NM_SLEEP_DELAY_BEFORE_SIGKILL_MS);
  this.maxKillWaitTime =
      conf.getLong(YarnConfiguration.NM_PROCESS_KILL_WAIT_MS,
          YarnConfiguration.DEFAULT_NM_PROCESS_KILL_WAIT_MS);
}
 
Example #8
Source File: DockerLinuxContainerRuntime.java    From hadoop with Apache License 2.0 6 votes vote down vote up
private void executeDockerLoadCommand(DockerLoadCommand cmd,
    ContainerRuntimeContext ctx)
    throws ContainerExecutionException {
  Container container = ctx.getContainer();
  String containerIdStr = container.getContainerId().toString();
  String commandFile = dockerClient.writeCommandToTempFile(cmd,
      containerIdStr);
  PrivilegedOperation launchOp = new PrivilegedOperation(
      PrivilegedOperation.OperationType.RUN_DOCKER_CMD);

  launchOp.appendArgs(commandFile);

  try {
    privilegedOperationExecutor.executePrivilegedOperation(null,
        launchOp, null, container.getLaunchContext().getEnvironment(),
        false);
  } catch (PrivilegedOperationException e) {
    LOG.warn("Docker load operation failed. Exception: ", e);
    throw new ContainerExecutionException("Docker load operation failed", e
        .getExitCode(), e.getOutput(), e.getErrorOutput());
  }
}
 
Example #9
Source File: DummyContainerManager.java    From big-c with Apache License 2.0 6 votes vote down vote up
@Override
@SuppressWarnings("unchecked")
protected ContainersLauncher createContainersLauncher(Context context,
    ContainerExecutor exec) {
  return new ContainersLauncher(context, super.dispatcher, exec,
                                super.dirsHandler, this) {
    @Override
    public void handle(ContainersLauncherEvent event) {
      Container container = event.getContainer();
      ContainerId containerId = container.getContainerId();
      switch (event.getType()) {
      case LAUNCH_CONTAINER:
        dispatcher.getEventHandler().handle(
            new ContainerEvent(containerId,
                ContainerEventType.CONTAINER_LAUNCHED));
        break;
      case CLEANUP_CONTAINER:
        dispatcher.getEventHandler().handle(
            new ContainerExitEvent(containerId,
                ContainerEventType.CONTAINER_KILLED_ON_REQUEST, 0,
                "Container exited with exit code 0."));
        break;
      }
    }
  };
}
 
Example #10
Source File: NodeStatusUpdaterImpl.java    From hadoop with Apache License 2.0 6 votes vote down vote up
private List<NMContainerStatus> getNMContainerStatuses() throws IOException {
  List<NMContainerStatus> containerStatuses =
      new ArrayList<NMContainerStatus>();
  for (Container container : this.context.getContainers().values()) {
    ContainerId containerId = container.getContainerId();
    ApplicationId applicationId = containerId.getApplicationAttemptId()
        .getApplicationId();
    if (!this.context.getApplications().containsKey(applicationId)) {
      context.getContainers().remove(containerId);
      continue;
    }
    NMContainerStatus status =
        container.getNMContainerStatus();
    containerStatuses.add(status);
    if (status.getContainerState() == ContainerState.COMPLETE) {
      // Adding to finished containers cache. Cache will keep it around at
      // least for #durationToTrackStoppedContainers duration. In the
      // subsequent call to stop container it will get removed from cache.
      addCompletedContainer(containerId);
    }
  }
  LOG.info("Sending out " + containerStatuses.size()
    + " NM container statuses: " + containerStatuses);
  return containerStatuses;
}
 
Example #11
Source File: ResourceLocalizationService.java    From hadoop with Apache License 2.0 6 votes vote down vote up
/**
 * For each of the requested resources for a container, determines the
 * appropriate {@link LocalResourcesTracker} and forwards a 
 * {@link LocalResourceRequest} to that tracker.
 */
private void handleInitContainerResources(
    ContainerLocalizationRequestEvent rsrcReqs) {
  Container c = rsrcReqs.getContainer();
  // create a loading cache for the file statuses
  LoadingCache<Path,Future<FileStatus>> statCache =
      CacheBuilder.newBuilder().build(FSDownload.createStatusCacheLoader(getConfig()));
  LocalizerContext ctxt = new LocalizerContext(
      c.getUser(), c.getContainerId(), c.getCredentials(), statCache);
  Map<LocalResourceVisibility, Collection<LocalResourceRequest>> rsrcs =
    rsrcReqs.getRequestedResources();
  for (Map.Entry<LocalResourceVisibility, Collection<LocalResourceRequest>> e :
       rsrcs.entrySet()) {
    LocalResourcesTracker tracker =
        getLocalResourcesTracker(e.getKey(), c.getUser(),
            c.getContainerId().getApplicationAttemptId()
                .getApplicationId());
    for (LocalResourceRequest req : e.getValue()) {
      tracker.handle(new ResourceRequestEvent(req, e.getKey(), ctxt));
    }
  }
}
 
Example #12
Source File: NMWebServices.java    From hadoop with Apache License 2.0 6 votes vote down vote up
@GET
@Path("/containers")
@Produces({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML })
public ContainersInfo getNodeContainers() {
  init();
  ContainersInfo allContainers = new ContainersInfo();
  for (Entry<ContainerId, Container> entry : this.nmContext.getContainers()
      .entrySet()) {
    if (entry.getValue() == null) {
      // just skip it
      continue;
    }
    ContainerInfo info = new ContainerInfo(this.nmContext, entry.getValue(),
        uriInfo.getBaseUri().toString(), webapp.name());
    allContainers.add(info);
  }
  return allContainers;
}
 
Example #13
Source File: AllContainersPage.java    From hadoop with Apache License 2.0 6 votes vote down vote up
@Override
protected void render(Block html) {
  TBODY<TABLE<BODY<Hamlet>>> tableBody = html.body()
    .table("#containers")
      .thead()
        .tr()
          .td()._("ContainerId")._()
          .td()._("ContainerState")._()
          .td()._("logs")._()
        ._()
      ._().tbody();
  for (Entry<ContainerId, Container> entry : this.nmContext
      .getContainers().entrySet()) {
    ContainerInfo info = new ContainerInfo(this.nmContext, entry.getValue());
    tableBody
      .tr()
        .td().a(url("container", info.getId()), info.getId())
        ._()
        .td()._(info.getState())._()
        .td()
            .a(url(info.getShortLogLink()), "logs")._()
      ._();
  }
  tableBody._()._()._();
}
 
Example #14
Source File: ContainerLogsUtils.java    From big-c with Apache License 2.0 6 votes vote down vote up
/**
 * Finds the log file with the given filename for the given container.
 */
public static File getContainerLogFile(ContainerId containerId,
    String fileName, String remoteUser, Context context) throws YarnException {
  Container container = context.getContainers().get(containerId);
  
  Application application = getApplicationForContainer(containerId, context);
  checkAccess(remoteUser, application, context);
  if (container != null) {
    checkState(container.getContainerState());
  }
  
  try {
    LocalDirsHandlerService dirsHandler = context.getLocalDirsHandler();
    String relativeContainerLogDir = ContainerLaunch.getRelativeContainerLogDir(
        application.getAppId().toString(), containerId.toString());
    Path logPath = dirsHandler.getLogPathToRead(
        relativeContainerLogDir + Path.SEPARATOR + fileName);
    URI logPathURI = new File(logPath.toString()).toURI();
    File logFile = new File(logPathURI.getPath());
    return logFile;
  } catch (IOException e) {
    LOG.warn("Failed to find log file", e);
    throw new NotFoundException("Cannot find this log on the local disk.");
  }
}
 
Example #15
Source File: TestNMWebServicesContainers.java    From big-c with Apache License 2.0 6 votes vote down vote up
private HashMap<String, String> addAppContainers(Application app) 
    throws IOException {
  Dispatcher dispatcher = new AsyncDispatcher();
  ApplicationAttemptId appAttemptId = BuilderUtils.newApplicationAttemptId(
      app.getAppId(), 1);
  Container container1 = new MockContainer(appAttemptId, dispatcher, conf,
      app.getUser(), app.getAppId(), 1);
  Container container2 = new MockContainer(appAttemptId, dispatcher, conf,
      app.getUser(), app.getAppId(), 2);
  nmContext.getContainers()
      .put(container1.getContainerId(), container1);
  nmContext.getContainers()
      .put(container2.getContainerId(), container2);

  app.getContainers().put(container1.getContainerId(), container1);
  app.getContainers().put(container2.getContainerId(), container2);
  HashMap<String, String> hash = new HashMap<String, String>();
  hash.put(container1.getContainerId().toString(), container1
      .getContainerId().toString());
  hash.put(container2.getContainerId().toString(), container2
      .getContainerId().toString());
  return hash;
}
 
Example #16
Source File: ContainerLogsUtils.java    From hadoop with Apache License 2.0 6 votes vote down vote up
/**
 * Finds the local directories that logs for the given container are stored
 * on.
 */
public static List<File> getContainerLogDirs(ContainerId containerId,
    String remoteUser, Context context) throws YarnException {
  Container container = context.getContainers().get(containerId);

  Application application = getApplicationForContainer(containerId, context);
  checkAccess(remoteUser, application, context);
  // It is not required to have null check for container ( container == null )
  // and throw back exception.Because when container is completed, NodeManager
  // remove container information from its NMContext.Configuring log
  // aggregation to false, container log view request is forwarded to NM. NM
  // does not have completed container information,but still NM serve request for
  // reading container logs. 
  if (container != null) {
    checkState(container.getContainerState());
  }
  
  return getContainerLogDirs(containerId, context.getLocalDirsHandler());
}
 
Example #17
Source File: AllContainersPage.java    From big-c with Apache License 2.0 6 votes vote down vote up
@Override
protected void render(Block html) {
  TBODY<TABLE<BODY<Hamlet>>> tableBody = html.body()
    .table("#containers")
      .thead()
        .tr()
          .td()._("ContainerId")._()
          .td()._("ContainerState")._()
          .td()._("logs")._()
        ._()
      ._().tbody();
  for (Entry<ContainerId, Container> entry : this.nmContext
      .getContainers().entrySet()) {
    ContainerInfo info = new ContainerInfo(this.nmContext, entry.getValue());
    tableBody
      .tr()
        .td().a(url("container", info.getId()), info.getId())
        ._()
        .td()._(info.getState())._()
        .td()
            .a(url(info.getShortLogLink()), "logs")._()
      ._();
  }
  tableBody._()._()._();
}
 
Example #18
Source File: DelegatingLinuxContainerRuntime.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Override
public void reapContainer(ContainerRuntimeContext ctx)
    throws ContainerExecutionException {
  Container container = ctx.getContainer();
  LinuxContainerRuntime runtime = pickContainerRuntime(container);

  runtime.reapContainer(ctx);
}
 
Example #19
Source File: ContainerManagerImpl.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Override
public void handle(ContainerEvent event) {
  Map<ContainerId,Container> containers =
    ContainerManagerImpl.this.context.getContainers();
  Container c = containers.get(event.getContainerID());
  if (c != null) {
    c.handle(event);
  } else {
    LOG.warn("Event " + event + " sent to absent container " +
        event.getContainerID());
  }
}
 
Example #20
Source File: TestResourceLocalizationService.java    From hadoop with Apache License 2.0 5 votes vote down vote up
private static Container getMockContainer(ApplicationId appId, int id,
    String user) {
  Container c = mock(Container.class);
  ApplicationAttemptId appAttemptId =
      BuilderUtils.newApplicationAttemptId(appId, 1);
  ContainerId cId = BuilderUtils.newContainerId(appAttemptId, id);
  when(c.getUser()).thenReturn(user);
  when(c.getContainerId()).thenReturn(cId);
  Credentials creds = new Credentials();
  creds.addToken(new Text("tok" + id), getToken(id));
  when(c.getCredentials()).thenReturn(creds);
  when(c.toString()).thenReturn(cId.toString());
  return c;
}
 
Example #21
Source File: RecoveredContainerLaunch.java    From hadoop with Apache License 2.0 5 votes vote down vote up
public RecoveredContainerLaunch(Context context, Configuration configuration,
    Dispatcher dispatcher, ContainerExecutor exec, Application app,
    Container container, LocalDirsHandlerService dirsHandler,
    ContainerManagerImpl containerManager)
{
  super(context, configuration, dispatcher, exec, app, container, dirsHandler,
    containerManager);
  this.shouldLaunchContainer.set(true);
}
 
Example #22
Source File: TestNodeStatusUpdater.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Override
public ConcurrentMap<ContainerId, Container> getContainers() {
  if (heartBeatID == 0) {
    return containers;
  } else if (heartBeatID == 1) {
    ContainerStatus containerStatus2 =
        createContainerStatus(2, ContainerState.RUNNING);
    putMockContainer(containerStatus2);

    ContainerStatus containerStatus3 =
        createContainerStatus(3, ContainerState.COMPLETE);
    putMockContainer(containerStatus3);
    return containers;
  } else if (heartBeatID == 2) {
    ContainerStatus containerStatus4 =
        createContainerStatus(4, ContainerState.RUNNING);
    putMockContainer(containerStatus4);

    ContainerStatus containerStatus5 =
        createContainerStatus(5, ContainerState.COMPLETE);
    putMockContainer(containerStatus5);
    return containers;
  } else if (heartBeatID == 3 || heartBeatID == 4) {
    return containers;
  } else {
    containers.clear();
    return containers;
  }
}
 
Example #23
Source File: DelegatingLinuxContainerRuntime.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Override
public void prepareContainer(ContainerRuntimeContext ctx)
    throws ContainerExecutionException {
  Container container = ctx.getContainer();
  LinuxContainerRuntime runtime = pickContainerRuntime(container);

  runtime.prepareContainer(ctx);
}
 
Example #24
Source File: TestApplication.java    From big-c with Apache License 2.0 5 votes vote down vote up
private Container createMockedContainer(ApplicationId appId, int containerId) {
  ApplicationAttemptId appAttemptId =
      BuilderUtils.newApplicationAttemptId(appId, 1);
  ContainerId cId = BuilderUtils.newContainerId(appAttemptId, containerId);
  Container c = mock(Container.class);
  when(c.getContainerId()).thenReturn(cId);
  ContainerLaunchContext launchContext = mock(ContainerLaunchContext.class);
  when(c.getLaunchContext()).thenReturn(launchContext);
  when(launchContext.getApplicationACLs()).thenReturn(
      new HashMap<ApplicationAccessType, String>());
  return c;
}
 
Example #25
Source File: TestNMWebServicesContainers.java    From hadoop with Apache License 2.0 5 votes vote down vote up
public void verifyNodeContainerInfoGeneric(Container cont, String id,
    String state, String user, int exitCode, String diagnostics,
    String nodeId, int totalMemoryNeededMB, int totalVCoresNeeded,
    String logsLink)
    throws JSONException, Exception {
  WebServicesTestUtils.checkStringMatch("id", cont.getContainerId()
      .toString(), id);
  WebServicesTestUtils.checkStringMatch("state", cont.getContainerState()
      .toString(), state);
  WebServicesTestUtils.checkStringMatch("user", cont.getUser().toString(),
      user);
  assertEquals("exitCode wrong", 0, exitCode);
  WebServicesTestUtils
      .checkStringMatch("diagnostics", "testing", diagnostics);

  WebServicesTestUtils.checkStringMatch("nodeId", nmContext.getNodeId()
      .toString(), nodeId);
  assertEquals("totalMemoryNeededMB wrong",
    YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB,
    totalMemoryNeededMB);
  assertEquals("totalVCoresNeeded wrong",
    YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES,
    totalVCoresNeeded);
  String shortLink =
      ujoin("containerlogs", cont.getContainerId().toString(),
          cont.getUser());
  assertTrue("containerLogsLink wrong", logsLink.contains(shortLink));
}
 
Example #26
Source File: ApplicationImpl.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Override
public void transition(ApplicationImpl app, ApplicationEvent event) {
  // Start all the containers waiting for ApplicationInit
  for (Container container : app.containers.values()) {
    app.dispatcher.getEventHandler().handle(new ContainerInitEvent(
          container.getContainerId()));
  }
}
 
Example #27
Source File: TestDockerContainerExecutorWithMocks.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Test(expected = IllegalArgumentException.class)
public void testContainerLaunchNullImage() throws IOException {
  String appSubmitter = "nobody";
  String appId = "APP_ID";
  String containerId = "CONTAINER_ID";
  String testImage = "";

  Container container = mock(Container.class, RETURNS_DEEP_STUBS);
  ContainerId cId = mock(ContainerId.class, RETURNS_DEEP_STUBS);
  ContainerLaunchContext context = mock(ContainerLaunchContext.class);
  HashMap<String, String> env = new HashMap<String,String>();

  when(container.getContainerId()).thenReturn(cId);
  when(container.getLaunchContext()).thenReturn(context);
  when(cId.getApplicationAttemptId().getApplicationId().toString()).thenReturn(appId);
  when(cId.toString()).thenReturn(containerId);

  when(context.getEnvironment()).thenReturn(env);
  env.put(YarnConfiguration.NM_DOCKER_CONTAINER_EXECUTOR_IMAGE_NAME, testImage);
  dockerContainerExecutor.getConf()
      .set(YarnConfiguration.NM_DOCKER_CONTAINER_EXECUTOR_IMAGE_NAME, testImage);
  Path scriptPath = new Path("file:///bin/echo");
  Path tokensPath = new Path("file:///dev/null");

  Path pidFile = new Path(workDir, "pid.txt");

  dockerContainerExecutor.activateContainer(cId, pidFile);
  dockerContainerExecutor.launchContainer(container, scriptPath, tokensPath,
      appSubmitter, appId, workDir, dirsHandler.getLocalDirs(),
      dirsHandler.getLogDirs());
}
 
Example #28
Source File: NodeStatusUpdaterImpl.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@VisibleForTesting
@Private
public void removeOrTrackCompletedContainersFromContext(
    List<ContainerId> containerIds) throws IOException {
  Set<ContainerId> removedContainers = new HashSet<ContainerId>();

  pendingContainersToRemove.addAll(containerIds);
  Iterator<ContainerId> iter = pendingContainersToRemove.iterator();
  while (iter.hasNext()) {
    ContainerId containerId = iter.next();
    // remove the container only if the container is at DONE state
    Container nmContainer = context.getContainers().get(containerId);
    if (nmContainer == null) {
      iter.remove();
    } else if (nmContainer.getContainerState().equals(
      org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerState.DONE)) {
      context.getContainers().remove(containerId);
      removedContainers.add(containerId);
      iter.remove();
    }
  }

  if (!removedContainers.isEmpty()) {
    LOG.info("Removed completed containers from NM context: "
        + removedContainers);
  }
  pendingCompletedContainers.clear();
}
 
Example #29
Source File: NodeStatusUpdaterImpl.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@VisibleForTesting
protected List<ContainerStatus> getContainerStatuses() throws IOException {
  List<ContainerStatus> containerStatuses = new ArrayList<ContainerStatus>();
  for (Container container : this.context.getContainers().values()) {
    ContainerId containerId = container.getContainerId();
    ApplicationId applicationId = containerId.getApplicationAttemptId()
        .getApplicationId();
    org.apache.hadoop.yarn.api.records.ContainerStatus containerStatus =
        container.cloneAndGetContainerStatus();
    if (containerStatus.getState() == ContainerState.COMPLETE) {
      if (isApplicationStopped(applicationId)) {
        if (LOG.isDebugEnabled()) {
          LOG.debug(applicationId + " is completing, " + " remove "
              + containerId + " from NM context.");
        }
        context.getContainers().remove(containerId);
        pendingCompletedContainers.put(containerId, containerStatus);
      } else {
        if (!isContainerRecentlyStopped(containerId)) {
          pendingCompletedContainers.put(containerId, containerStatus);
          // Adding to finished containers cache. Cache will keep it around at
          // least for #durationToTrackStoppedContainers duration. In the
          // subsequent call to stop container it will get removed from cache.
          addCompletedContainer(containerId);
        }
      }
    } else {
      containerStatuses.add(containerStatus);
    }
  }
  containerStatuses.addAll(pendingCompletedContainers.values());
  if (LOG.isDebugEnabled()) {
    LOG.debug("Sending out " + containerStatuses.size()
        + " container statuses: " + containerStatuses);
  }
  return containerStatuses;
}
 
Example #30
Source File: ContainerPage.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Override
protected void render(Block html) {
  ContainerId containerID;
  try {
    containerID = ConverterUtils.toContainerId($(CONTAINER_ID));
  } catch (IllegalArgumentException e) {
    html.p()._("Invalid containerId " + $(CONTAINER_ID))._();
    return;
  }

  DIV<Hamlet> div = html.div("#content");
  Container container = this.nmContext.getContainers().get(containerID);
  if (container == null) {
    div.h1("Unknown Container. Container might have completed, "
            + "please go back to the previous page and retry.")._();
    return;
  }
  ContainerInfo info = new ContainerInfo(this.nmContext, container);

  info("Container information")
    ._("ContainerID", info.getId())
    ._("ContainerState", info.getState())
    ._("ExitStatus", info.getExitStatus())
    ._("Diagnostics", info.getDiagnostics())
    ._("User", info.getUser())
    ._("TotalMemoryNeeded", info.getMemoryNeeded())
    ._("TotalVCoresNeeded", info.getVCoresNeeded())
    ._("logs", info.getShortLogLink(), "Link to logs");
  html._(InfoBlock.class);
}