org.apache.hadoop.yarn.server.nodemanager.NodeHealthCheckerService Java Exaples

Source File: MiniYARNClusterSplice.java From spliceengine with GNU Affero General Public License v3.0

6 votes

@Override
protected NodeStatusUpdater createNodeStatusUpdater(Context context,
                                                    Dispatcher dispatcher, NodeHealthCheckerService healthChecker) {
    return new NodeStatusUpdaterImpl(context, dispatcher,
                                     healthChecker, metrics) {
        @Override
        protected ResourceTracker getRMClient() {
            final ResourceTrackerService rt =
                getResourceManager().getResourceTrackerService();
            final RecordFactory recordFactory =
                RecordFactoryProvider.getRecordFactory(null);

            // For in-process communication without RPC
            return Utils.getResourceTracker(rt);
        }

        @Override
        protected void stopRMProxy() { }
     };
}

Source File: BaseContainerManagerTest.java From hadoop with Apache License 2.0

5 votes

@Before
public void setup() throws IOException {
  localFS.delete(new Path(localDir.getAbsolutePath()), true);
  localFS.delete(new Path(tmpDir.getAbsolutePath()), true);
  localFS.delete(new Path(localLogDir.getAbsolutePath()), true);
  localFS.delete(new Path(remoteLogDir.getAbsolutePath()), true);
  localDir.mkdir();
  tmpDir.mkdir();
  localLogDir.mkdir();
  remoteLogDir.mkdir();
  LOG.info("Created localDir in " + localDir.getAbsolutePath());
  LOG.info("Created tmpDir in " + tmpDir.getAbsolutePath());

  String bindAddress = "0.0.0.0:12345";
  conf.set(YarnConfiguration.NM_ADDRESS, bindAddress);
  conf.set(YarnConfiguration.NM_LOCAL_DIRS, localDir.getAbsolutePath());
  conf.set(YarnConfiguration.NM_LOG_DIRS, localLogDir.getAbsolutePath());
  conf.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, remoteLogDir.getAbsolutePath());

  conf.setLong(YarnConfiguration.NM_LOG_RETAIN_SECONDS, 1);
  // Default delSrvc
  delSrvc = createDeletionService();
  delSrvc.init(conf);

  exec = createContainerExecutor();
  dirsHandler = new LocalDirsHandlerService();
  nodeHealthChecker = new NodeHealthCheckerService(
      NodeManager.getNodeHealthScriptRunner(conf), dirsHandler);
  nodeHealthChecker.init(conf);
  containerManager = createContainerManager(delSrvc);
  ((NMContext)context).setContainerManager(containerManager);
  nodeStatusUpdater.init(conf);
  containerManager.init(conf);
  nodeStatusUpdater.start();
}

Source File: BaseContainerManagerTest.java From big-c with Apache License 2.0

5 votes

@Before
public void setup() throws IOException {
  localFS.delete(new Path(localDir.getAbsolutePath()), true);
  localFS.delete(new Path(tmpDir.getAbsolutePath()), true);
  localFS.delete(new Path(localLogDir.getAbsolutePath()), true);
  localFS.delete(new Path(remoteLogDir.getAbsolutePath()), true);
  localDir.mkdir();
  tmpDir.mkdir();
  localLogDir.mkdir();
  remoteLogDir.mkdir();
  LOG.info("Created localDir in " + localDir.getAbsolutePath());
  LOG.info("Created tmpDir in " + tmpDir.getAbsolutePath());

  String bindAddress = "0.0.0.0:12345";
  conf.set(YarnConfiguration.NM_ADDRESS, bindAddress);
  conf.set(YarnConfiguration.NM_LOCAL_DIRS, localDir.getAbsolutePath());
  conf.set(YarnConfiguration.NM_LOG_DIRS, localLogDir.getAbsolutePath());
  conf.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, remoteLogDir.getAbsolutePath());

  conf.setLong(YarnConfiguration.NM_LOG_RETAIN_SECONDS, 1);
  // Default delSrvc
  delSrvc = createDeletionService();
  delSrvc.init(conf);

  exec = createContainerExecutor();
  nodeHealthChecker = new NodeHealthCheckerService();
  nodeHealthChecker.init(conf);
  dirsHandler = nodeHealthChecker.getDiskHandler();
  containerManager = createContainerManager(delSrvc);
  ((NMContext)context).setContainerManager(containerManager);
  nodeStatusUpdater.init(conf);
  containerManager.init(conf);
  nodeStatusUpdater.start();
}

Source File: TestNMWebServices.java From hadoop with Apache License 2.0

4 votes

@Override
protected void configureServlets() {
  Configuration conf = new Configuration();
  conf.set(YarnConfiguration.NM_LOCAL_DIRS, testRootDir.getAbsolutePath());
  conf.set(YarnConfiguration.NM_LOG_DIRS, testLogDir.getAbsolutePath());
  dirsHandler = new LocalDirsHandlerService();
  NodeHealthCheckerService healthChecker = new NodeHealthCheckerService(
      NodeManager.getNodeHealthScriptRunner(conf), dirsHandler);
  healthChecker.init(conf);
  aclsManager = new ApplicationACLsManager(conf);
  nmContext = new NodeManager.NMContext(null, null, dirsHandler,
      aclsManager, null);
  NodeId nodeId = NodeId.newInstance("testhost.foo.com", 8042);
  ((NodeManager.NMContext)nmContext).setNodeId(nodeId);
  resourceView = new ResourceView() {
    @Override
    public long getVmemAllocatedForContainers() {
      // 15.5G in bytes
      return new Long("16642998272");
    }

    @Override
    public long getPmemAllocatedForContainers() {
      // 16G in bytes
      return new Long("17179869184");
    }
    @Override
    public long getVCoresAllocatedForContainers() {
      return new Long("4000");
    }
    @Override
    public long getGCoresAllocatedForContainers() {
      return new Long("4000");
    }
    @Override
    public boolean isVmemCheckEnabled() {
      return true;
    }
    @Override
    public boolean isPmemCheckEnabled() {
      return true;
    }
  };
  nmWebApp = new NMWebApp(resourceView, aclsManager, dirsHandler);
  bind(JAXBContextResolver.class);
  bind(NMWebServices.class);
  bind(GenericExceptionHandler.class);
  bind(Context.class).toInstance(nmContext);
  bind(WebApp.class).toInstance(nmWebApp);
  bind(ResourceView.class).toInstance(resourceView);
  bind(ApplicationACLsManager.class).toInstance(aclsManager);
  bind(LocalDirsHandlerService.class).toInstance(dirsHandler);

  serve("/*").with(GuiceContainer.class);
}

Source File: TestNMWebServer.java From hadoop with Apache License 2.0

4 votes

private NodeHealthCheckerService createNodeHealthCheckerService(Configuration conf) {
  NodeHealthScriptRunner scriptRunner = NodeManager.getNodeHealthScriptRunner(conf);
  LocalDirsHandlerService dirsHandler = new LocalDirsHandlerService();
  return new NodeHealthCheckerService(scriptRunner, dirsHandler);
}

Source File: TestNMWebServer.java From hadoop with Apache License 2.0

4 votes

private int startNMWebAppServer(String webAddr) {
  Context nmContext = new NodeManager.NMContext(null, null, null, null,
      null);
  ResourceView resourceView = new ResourceView() {
    @Override
    public long getVmemAllocatedForContainers() {
      return 0;
    }
    @Override
    public long getPmemAllocatedForContainers() {
      return 0;
    }
    @Override
    public long getVCoresAllocatedForContainers() {
      return 0;
    }
    @Override
    public long getGCoresAllocatedForContainers() {
      return 0;
    }
    @Override
    public boolean isVmemCheckEnabled() {
      return true;
    }
    @Override
    public boolean isPmemCheckEnabled() {
      return true;
    }
  };
  Configuration conf = new Configuration();
  conf.set(YarnConfiguration.NM_LOCAL_DIRS, testRootDir.getAbsolutePath());
  conf.set(YarnConfiguration.NM_LOG_DIRS, testLogDir.getAbsolutePath());
  NodeHealthCheckerService healthChecker = createNodeHealthCheckerService(conf);
  healthChecker.init(conf);
  LocalDirsHandlerService dirsHandler = healthChecker.getDiskHandler();
  conf.set(YarnConfiguration.NM_WEBAPP_ADDRESS, webAddr);
  WebServer server = new WebServer(nmContext, resourceView,
      new ApplicationACLsManager(conf), dirsHandler);
  try {
    server.init(conf);
    server.start();
    return server.getPort();
  } finally {
    server.stop();
    healthChecker.stop();
  }
}

Source File: TestNMWebServer.java From hadoop with Apache License 2.0

4 votes

@Test
  public void testNMWebApp() throws IOException, YarnException {
    Context nmContext = new NodeManager.NMContext(null, null, null, null,
        null);
    ResourceView resourceView = new ResourceView() {
      @Override
      public long getVmemAllocatedForContainers() {
        return 0;
      }
      @Override
      public long getPmemAllocatedForContainers() {
        return 0;
      }
      @Override
      public long getVCoresAllocatedForContainers() {
        return 0;
      }
      @Override
      public long getGCoresAllocatedForContainers() {
        return 0;
      }
      @Override
      public boolean isVmemCheckEnabled() {
        return true;
      }
      @Override
      public boolean isPmemCheckEnabled() {
        return true;
      }
    };
    Configuration conf = new Configuration();
    conf.set(YarnConfiguration.NM_LOCAL_DIRS, testRootDir.getAbsolutePath());
    conf.set(YarnConfiguration.NM_LOG_DIRS, testLogDir.getAbsolutePath());
    NodeHealthCheckerService healthChecker = createNodeHealthCheckerService(conf);
    healthChecker.init(conf);
    LocalDirsHandlerService dirsHandler = healthChecker.getDiskHandler();

    WebServer server = new WebServer(nmContext, resourceView,
        new ApplicationACLsManager(conf), dirsHandler);
    server.init(conf);
    server.start();

    // Add an application and the corresponding containers
    RecordFactory recordFactory =
        RecordFactoryProvider.getRecordFactory(conf);
    Dispatcher dispatcher = new AsyncDispatcher();
    String user = "nobody";
    long clusterTimeStamp = 1234;
    ApplicationId appId =
        BuilderUtils.newApplicationId(recordFactory, clusterTimeStamp, 1);
    Application app = mock(Application.class);
    when(app.getUser()).thenReturn(user);
    when(app.getAppId()).thenReturn(appId);
    nmContext.getApplications().put(appId, app);
    ApplicationAttemptId appAttemptId = BuilderUtils.newApplicationAttemptId(
        appId, 1);
    ContainerId container1 =
        BuilderUtils.newContainerId(recordFactory, appId, appAttemptId, 0);
    ContainerId container2 =
        BuilderUtils.newContainerId(recordFactory, appId, appAttemptId, 1);
    NodeManagerMetrics metrics = mock(NodeManagerMetrics.class);
    NMStateStoreService stateStore = new NMNullStateStoreService();
    for (ContainerId containerId : new ContainerId[] { container1,
        container2}) {
      // TODO: Use builder utils
      ContainerLaunchContext launchContext =
          recordFactory.newRecordInstance(ContainerLaunchContext.class);
      long currentTime = System.currentTimeMillis();
      Token containerToken =
          BuilderUtils.newContainerToken(containerId, "127.0.0.1", 1234, user,
            BuilderUtils.newResource(1024, 1), currentTime + 10000L, 123,
            "password".getBytes(), currentTime);
      Container container =
          new ContainerImpl(conf, dispatcher, stateStore, launchContext,
            null, metrics,
            BuilderUtils.newContainerTokenIdentifier(containerToken)) {

            @Override
            public ContainerState getContainerState() {
              return ContainerState.RUNNING;
            };
          };
      nmContext.getContainers().put(containerId, container);
      //TODO: Gross hack. Fix in code.
      ApplicationId applicationId = 
          containerId.getApplicationAttemptId().getApplicationId();
      nmContext.getApplications().get(applicationId).getContainers()
          .put(containerId, container);
      writeContainerLogs(nmContext, containerId, dirsHandler);

    }
    // TODO: Pull logs and test contents.
//    Thread.sleep(1000000);
  }

Source File: TestNMWebServicesContainers.java From hadoop with Apache License 2.0

4 votes

@Override
protected void configureServlets() {
  resourceView = new ResourceView() {
    @Override
    public long getVmemAllocatedForContainers() {
      // 15.5G in bytes
      return new Long("16642998272");
    }

    @Override
    public long getPmemAllocatedForContainers() {
      // 16G in bytes
      return new Long("17179869184");
    }

    @Override
    public long getVCoresAllocatedForContainers() {
      return new Long("4000");
    }

    @Override
    public long getGCoresAllocatedForContainers() {
      return new Long("4000");
    }

    @Override
    public boolean isVmemCheckEnabled() {
      return true;
    }

    @Override
    public boolean isPmemCheckEnabled() {
      return true;
    }
  };
  conf.set(YarnConfiguration.NM_LOCAL_DIRS, testRootDir.getAbsolutePath());
  conf.set(YarnConfiguration.NM_LOG_DIRS, testLogDir.getAbsolutePath());
  LocalDirsHandlerService dirsHandler = new LocalDirsHandlerService();
  NodeHealthCheckerService healthChecker = new NodeHealthCheckerService(
      NodeManager.getNodeHealthScriptRunner(conf), dirsHandler);
  healthChecker.init(conf);
  dirsHandler = healthChecker.getDiskHandler();
  aclsManager = new ApplicationACLsManager(conf);
  nmContext = new NodeManager.NMContext(null, null, dirsHandler,
      aclsManager, null) {
    public NodeId getNodeId() {
      return NodeId.newInstance("testhost.foo.com", 8042);
    };

    public int getHttpPort() {
      return 1234;
    };
  };
  nmWebApp = new NMWebApp(resourceView, aclsManager, dirsHandler);
  bind(JAXBContextResolver.class);
  bind(NMWebServices.class);
  bind(GenericExceptionHandler.class);
  bind(Context.class).toInstance(nmContext);
  bind(WebApp.class).toInstance(nmWebApp);
  bind(ResourceView.class).toInstance(resourceView);
  bind(ApplicationACLsManager.class).toInstance(aclsManager);
  bind(LocalDirsHandlerService.class).toInstance(dirsHandler);

  serve("/*").with(GuiceContainer.class);
}

Source File: TestContainerLogsPage.java From hadoop with Apache License 2.0

4 votes

private NodeHealthCheckerService createNodeHealthCheckerService(Configuration conf) {
  NodeHealthScriptRunner scriptRunner = NodeManager.getNodeHealthScriptRunner(conf);
  LocalDirsHandlerService dirsHandler = new LocalDirsHandlerService();
  return new NodeHealthCheckerService(scriptRunner, dirsHandler);
}

Source File: TestContainerLogsPage.java From big-c with Apache License 2.0

4 votes

@Test(timeout=30000)
public void testContainerLogDirs() throws IOException, YarnException {
  File absLogDir = new File("target",
    TestNMWebServer.class.getSimpleName() + "LogDir").getAbsoluteFile();
  String logdirwithFile = absLogDir.toURI().toString();
  Configuration conf = new Configuration();
  conf.set(YarnConfiguration.NM_LOG_DIRS, logdirwithFile);
  NodeHealthCheckerService healthChecker = new NodeHealthCheckerService();
  healthChecker.init(conf);
  LocalDirsHandlerService dirsHandler = healthChecker.getDiskHandler();
  NMContext nmContext = new NodeManager.NMContext(null, null, dirsHandler,
      new ApplicationACLsManager(conf), new NMNullStateStoreService(), null);
  // Add an application and the corresponding containers
  RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(conf);
  String user = "nobody";
  long clusterTimeStamp = 1234;
  ApplicationId appId = BuilderUtils.newApplicationId(recordFactory,
      clusterTimeStamp, 1);
  Application app = mock(Application.class);
  when(app.getUser()).thenReturn(user);
  when(app.getAppId()).thenReturn(appId);
  ApplicationAttemptId appAttemptId = BuilderUtils.newApplicationAttemptId(
      appId, 1);
  ContainerId container1 = BuilderUtils.newContainerId(recordFactory, appId,
      appAttemptId, 0);
  nmContext.getApplications().put(appId, app);

  MockContainer container =
      new MockContainer(appAttemptId, new AsyncDispatcher(), conf, user,
          appId, 1);
  container.setState(ContainerState.RUNNING);
  nmContext.getContainers().put(container1, container);   
  List<File> files = null;
  files = ContainerLogsUtils.getContainerLogDirs(container1, user, nmContext);
  Assert.assertTrue(!(files.get(0).toString().contains("file:")));
  
  // After container is completed, it is removed from nmContext
  nmContext.getContainers().remove(container1);
  Assert.assertNull(nmContext.getContainers().get(container1));
  files = ContainerLogsUtils.getContainerLogDirs(container1, user, nmContext);
  Assert.assertTrue(!(files.get(0).toString().contains("file:")));

  // Create a new context to check if correct container log dirs are fetched
  // on full disk.
  LocalDirsHandlerService dirsHandlerForFullDisk = spy(dirsHandler);
  // good log dirs are empty and nm log dir is in the full log dir list.
  when(dirsHandlerForFullDisk.getLogDirs()).
      thenReturn(new ArrayList<String>());
  when(dirsHandlerForFullDisk.getLogDirsForRead()).
      thenReturn(Arrays.asList(new String[] {absLogDir.getAbsolutePath()}));
  nmContext = new NodeManager.NMContext(null, null, dirsHandlerForFullDisk,
      new ApplicationACLsManager(conf), new NMNullStateStoreService(), null);
  nmContext.getApplications().put(appId, app);
  container.setState(ContainerState.RUNNING);
  nmContext.getContainers().put(container1, container);
  List<File> dirs =
      ContainerLogsUtils.getContainerLogDirs(container1, user, nmContext);
  File containerLogDir = new File(absLogDir, appId + "/" + container1);
  Assert.assertTrue(dirs.contains(containerLogDir));
}

Source File: TestNMWebServicesApps.java From big-c with Apache License 2.0

4 votes

@Override
protected void configureServlets() {
  conf.set(YarnConfiguration.NM_LOCAL_DIRS, testRootDir.getAbsolutePath());
  conf.set(YarnConfiguration.NM_LOG_DIRS, testLogDir.getAbsolutePath());
  NodeHealthCheckerService healthChecker = new NodeHealthCheckerService();
  healthChecker.init(conf);
  dirsHandler = healthChecker.getDiskHandler();
  aclsManager = new ApplicationACLsManager(conf);
  nmContext = new NodeManager.NMContext(null, null, dirsHandler,
      aclsManager, null, null);
  NodeId nodeId = NodeId.newInstance("testhost.foo.com", 9999);
  ((NodeManager.NMContext)nmContext).setNodeId(nodeId);
  resourceView = new ResourceView() {
    @Override
    public long getVmemAllocatedForContainers() {
      // 15.5G in bytes
      return new Long("16642998272");
    }

    @Override
    public long getPmemAllocatedForContainers() {
      // 16G in bytes
      return new Long("17179869184");
    }

    @Override
    public long getVCoresAllocatedForContainers() {
      return new Long("4000");
    }


    @Override
    public boolean isVmemCheckEnabled() {
      return true;
    }

    @Override
    public boolean isPmemCheckEnabled() {
      return true;
    }
  };
  nmWebApp = new NMWebApp(resourceView, aclsManager, dirsHandler);
  bind(JAXBContextResolver.class);
  bind(NMWebServices.class);
  bind(GenericExceptionHandler.class);
  bind(Context.class).toInstance(nmContext);
  bind(WebApp.class).toInstance(nmWebApp);
  bind(ResourceView.class).toInstance(resourceView);
  bind(ApplicationACLsManager.class).toInstance(aclsManager);
  bind(LocalDirsHandlerService.class).toInstance(dirsHandler);

  serve("/*").with(GuiceContainer.class);
}

Source File: TestNMWebServices.java From big-c with Apache License 2.0

4 votes

@Override
protected void configureServlets() {
  Configuration conf = new Configuration();
  conf.set(YarnConfiguration.NM_LOCAL_DIRS, testRootDir.getAbsolutePath());
  conf.set(YarnConfiguration.NM_LOG_DIRS, testLogDir.getAbsolutePath());
  NodeHealthCheckerService healthChecker = new NodeHealthCheckerService();
  healthChecker.init(conf);
  dirsHandler = healthChecker.getDiskHandler();
  aclsManager = new ApplicationACLsManager(conf);
  nmContext = new NodeManager.NMContext(null, null, dirsHandler,
      aclsManager, null, null);
  NodeId nodeId = NodeId.newInstance("testhost.foo.com", 8042);
  ((NodeManager.NMContext)nmContext).setNodeId(nodeId);
  resourceView = new ResourceView() {
    @Override
    public long getVmemAllocatedForContainers() {
      // 15.5G in bytes
      return new Long("16642998272");
    }

    @Override
    public long getPmemAllocatedForContainers() {
      // 16G in bytes
      return new Long("17179869184");
    }
    @Override
    public long getVCoresAllocatedForContainers() {
      return new Long("4000");
    }
    @Override
    public boolean isVmemCheckEnabled() {
      return true;
    }
    @Override
    public boolean isPmemCheckEnabled() {
      return true;
    }
  };
  nmWebApp = new NMWebApp(resourceView, aclsManager, dirsHandler);
  bind(JAXBContextResolver.class);
  bind(NMWebServices.class);
  bind(GenericExceptionHandler.class);
  bind(Context.class).toInstance(nmContext);
  bind(WebApp.class).toInstance(nmWebApp);
  bind(ResourceView.class).toInstance(resourceView);
  bind(ApplicationACLsManager.class).toInstance(aclsManager);
  bind(LocalDirsHandlerService.class).toInstance(dirsHandler);

  serve("/*").with(GuiceContainer.class);
}

Source File: TestNMWebServer.java From big-c with Apache License 2.0

4 votes

private int startNMWebAppServer(String webAddr) {
  Context nmContext = new NodeManager.NMContext(null, null, null, null,
      null, null);
  ResourceView resourceView = new ResourceView() {
    @Override
    public long getVmemAllocatedForContainers() {
      return 0;
    }
    @Override
    public long getPmemAllocatedForContainers() {
      return 0;
    }
    @Override
    public long getVCoresAllocatedForContainers() {
      return 0;
    }
    @Override
    public boolean isVmemCheckEnabled() {
      return true;
    }
    @Override
    public boolean isPmemCheckEnabled() {
      return true;
    }
  };
  Configuration conf = new Configuration();
  conf.set(YarnConfiguration.NM_LOCAL_DIRS, testRootDir.getAbsolutePath());
  conf.set(YarnConfiguration.NM_LOG_DIRS, testLogDir.getAbsolutePath());
  NodeHealthCheckerService healthChecker = new NodeHealthCheckerService();
  healthChecker.init(conf);
  LocalDirsHandlerService dirsHandler = healthChecker.getDiskHandler();
  conf.set(YarnConfiguration.NM_WEBAPP_ADDRESS, webAddr);
  WebServer server = new WebServer(nmContext, resourceView,
      new ApplicationACLsManager(conf), dirsHandler);
  try {
    server.init(conf);
    server.start();
    return server.getPort();
  } finally {
    server.stop();
    healthChecker.stop();
  }
}

Source File: TestNMWebServer.java From big-c with Apache License 2.0

4 votes

@Test
  public void testNMWebApp() throws IOException, YarnException {
    Context nmContext = new NodeManager.NMContext(null, null, null, null,
        null, null);
    ResourceView resourceView = new ResourceView() {
      @Override
      public long getVmemAllocatedForContainers() {
        return 0;
      }
      @Override
      public long getPmemAllocatedForContainers() {
        return 0;
      }
      @Override
      public long getVCoresAllocatedForContainers() {
        return 0;
      }
      @Override
      public boolean isVmemCheckEnabled() {
        return true;
      }
      @Override
      public boolean isPmemCheckEnabled() {
        return true;
      }
    };
    Configuration conf = new Configuration();
    conf.set(YarnConfiguration.NM_LOCAL_DIRS, testRootDir.getAbsolutePath());
    conf.set(YarnConfiguration.NM_LOG_DIRS, testLogDir.getAbsolutePath());
    NodeHealthCheckerService healthChecker = new NodeHealthCheckerService();
    healthChecker.init(conf);
    LocalDirsHandlerService dirsHandler = healthChecker.getDiskHandler();

    WebServer server = new WebServer(nmContext, resourceView,
        new ApplicationACLsManager(conf), dirsHandler);
    server.init(conf);
    server.start();

    // Add an application and the corresponding containers
    RecordFactory recordFactory =
        RecordFactoryProvider.getRecordFactory(conf);
    Dispatcher dispatcher = new AsyncDispatcher();
    String user = "nobody";
    long clusterTimeStamp = 1234;
    ApplicationId appId =
        BuilderUtils.newApplicationId(recordFactory, clusterTimeStamp, 1);
    Application app = mock(Application.class);
    when(app.getUser()).thenReturn(user);
    when(app.getAppId()).thenReturn(appId);
    nmContext.getApplications().put(appId, app);
    ApplicationAttemptId appAttemptId = BuilderUtils.newApplicationAttemptId(
        appId, 1);
    ContainerId container1 =
        BuilderUtils.newContainerId(recordFactory, appId, appAttemptId, 0);
    ContainerId container2 =
        BuilderUtils.newContainerId(recordFactory, appId, appAttemptId, 1);
    NodeManagerMetrics metrics = mock(NodeManagerMetrics.class);
    NMStateStoreService stateStore = new NMNullStateStoreService();
    for (ContainerId containerId : new ContainerId[] { container1,
        container2}) {
      // TODO: Use builder utils
      ContainerLaunchContext launchContext =
          recordFactory.newRecordInstance(ContainerLaunchContext.class);
      long currentTime = System.currentTimeMillis();
      Token containerToken =
          BuilderUtils.newContainerToken(containerId, "127.0.0.1", 1234, user,
            BuilderUtils.newResource(1024, 1), currentTime + 10000L, 123,
            "password".getBytes(), currentTime);
      Container container =
          new ContainerImpl(nmContext, conf, dispatcher, stateStore, launchContext,
            null, metrics,
            BuilderUtils.newContainerTokenIdentifier(containerToken), null, 0, user, false, null) {

            @Override
            public ContainerState getContainerState() {
              return ContainerState.RUNNING;
            };
          };
      nmContext.getContainers().put(containerId, container);
      //TODO: Gross hack. Fix in code.
      ApplicationId applicationId = 
          containerId.getApplicationAttemptId().getApplicationId();
      nmContext.getApplications().get(applicationId).getContainers()
          .put(containerId, container);
      writeContainerLogs(nmContext, containerId, dirsHandler);

    }
    // TODO: Pull logs and test contents.
//    Thread.sleep(1000000);
  }

Source File: TestNMWebServicesContainers.java From big-c with Apache License 2.0

4 votes

protected void configureServlets() {
  resourceView = new ResourceView() {
    @Override
    public long getVmemAllocatedForContainers() {
      // 15.5G in bytes
      return new Long("16642998272");
    }

    @Override
    public long getPmemAllocatedForContainers() {
      // 16G in bytes
      return new Long("17179869184");
    }

    @Override
    public long getVCoresAllocatedForContainers() {
      return new Long("4000");
    }

    @Override
    public boolean isVmemCheckEnabled() {
      return true;
    }

    @Override
    public boolean isPmemCheckEnabled() {
      return true;
    }
  };
  conf.set(YarnConfiguration.NM_LOCAL_DIRS, testRootDir.getAbsolutePath());
  conf.set(YarnConfiguration.NM_LOG_DIRS, testLogDir.getAbsolutePath());
  NodeHealthCheckerService healthChecker = new NodeHealthCheckerService();
  healthChecker.init(conf);
  dirsHandler = healthChecker.getDiskHandler();
  aclsManager = new ApplicationACLsManager(conf);
  nmContext = new NodeManager.NMContext(null, null, dirsHandler,
      aclsManager, null, null) {
    public NodeId getNodeId() {
      return NodeId.newInstance("testhost.foo.com", 8042);
    };

    public int getHttpPort() {
      return 1234;
    };
  };
  nmWebApp = new NMWebApp(resourceView, aclsManager, dirsHandler);
  bind(JAXBContextResolver.class);
  bind(NMWebServices.class);
  bind(GenericExceptionHandler.class);
  bind(Context.class).toInstance(nmContext);
  bind(WebApp.class).toInstance(nmWebApp);
  bind(ResourceView.class).toInstance(resourceView);
  bind(ApplicationACLsManager.class).toInstance(aclsManager);
  bind(LocalDirsHandlerService.class).toInstance(dirsHandler);

  serve("/*").with(GuiceContainer.class);
}

Source File: TestNMWebServicesApps.java From hadoop with Apache License 2.0

4 votes

@Override
protected void configureServlets() {
  conf.set(YarnConfiguration.NM_LOCAL_DIRS, testRootDir.getAbsolutePath());
  conf.set(YarnConfiguration.NM_LOG_DIRS, testLogDir.getAbsolutePath());
  LocalDirsHandlerService dirsHandler = new LocalDirsHandlerService();
  NodeHealthCheckerService healthChecker = new NodeHealthCheckerService(
      NodeManager.getNodeHealthScriptRunner(conf), dirsHandler);
  healthChecker.init(conf);
  dirsHandler = healthChecker.getDiskHandler();
  aclsManager = new ApplicationACLsManager(conf);
  nmContext = new NodeManager.NMContext(null, null, dirsHandler,
      aclsManager, null);
  NodeId nodeId = NodeId.newInstance("testhost.foo.com", 9999);
  ((NodeManager.NMContext)nmContext).setNodeId(nodeId);
  resourceView = new ResourceView() {
    @Override
    public long getVmemAllocatedForContainers() {
      // 15.5G in bytes
      return new Long("16642998272");
    }

    @Override
    public long getPmemAllocatedForContainers() {
      // 16G in bytes
      return new Long("17179869184");
    }

    @Override
    public long getVCoresAllocatedForContainers() {
      return new Long("4000");
    }

    @Override
    public long getGCoresAllocatedForContainers() {
      return new Long("4000");
    }

    @Override
    public boolean isVmemCheckEnabled() {
      return true;
    }

    @Override
    public boolean isPmemCheckEnabled() {
      return true;
    }
  };
  nmWebApp = new NMWebApp(resourceView, aclsManager, dirsHandler);
  bind(JAXBContextResolver.class);
  bind(NMWebServices.class);
  bind(GenericExceptionHandler.class);
  bind(Context.class).toInstance(nmContext);
  bind(WebApp.class).toInstance(nmWebApp);
  bind(ResourceView.class).toInstance(resourceView);
  bind(ApplicationACLsManager.class).toInstance(aclsManager);
  bind(LocalDirsHandlerService.class).toInstance(dirsHandler);

  serve("/*").with(GuiceContainer.class);
}

Source File: TestContainerLogsPage.java From hadoop with Apache License 2.0

4 votes

@Test(timeout=30000)
public void testContainerLogDirs() throws IOException, YarnException {
  File absLogDir = new File("target",
    TestNMWebServer.class.getSimpleName() + "LogDir").getAbsoluteFile();
  String logdirwithFile = absLogDir.toURI().toString();
  Configuration conf = new Configuration();
  conf.set(YarnConfiguration.NM_LOG_DIRS, logdirwithFile);
  NodeHealthCheckerService healthChecker = createNodeHealthCheckerService(conf);
  healthChecker.init(conf);
  LocalDirsHandlerService dirsHandler = healthChecker.getDiskHandler();
  NMContext nmContext = new NodeManager.NMContext(null, null, dirsHandler,
      new ApplicationACLsManager(conf), new NMNullStateStoreService());
  // Add an application and the corresponding containers
  RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(conf);
  String user = "nobody";
  long clusterTimeStamp = 1234;
  ApplicationId appId = BuilderUtils.newApplicationId(recordFactory,
      clusterTimeStamp, 1);
  Application app = mock(Application.class);
  when(app.getUser()).thenReturn(user);
  when(app.getAppId()).thenReturn(appId);
  ApplicationAttemptId appAttemptId = BuilderUtils.newApplicationAttemptId(
      appId, 1);
  ContainerId container1 = BuilderUtils.newContainerId(recordFactory, appId,
      appAttemptId, 0);
  nmContext.getApplications().put(appId, app);

  MockContainer container =
      new MockContainer(appAttemptId, new AsyncDispatcher(), conf, user,
          appId, 1);
  container.setState(ContainerState.RUNNING);
  nmContext.getContainers().put(container1, container);   
  List<File> files = null;
  files = ContainerLogsUtils.getContainerLogDirs(container1, user, nmContext);
  Assert.assertTrue(!(files.get(0).toString().contains("file:")));
  
  // After container is completed, it is removed from nmContext
  nmContext.getContainers().remove(container1);
  Assert.assertNull(nmContext.getContainers().get(container1));
  files = ContainerLogsUtils.getContainerLogDirs(container1, user, nmContext);
  Assert.assertTrue(!(files.get(0).toString().contains("file:")));

  // Create a new context to check if correct container log dirs are fetched
  // on full disk.
  LocalDirsHandlerService dirsHandlerForFullDisk = spy(dirsHandler);
  // good log dirs are empty and nm log dir is in the full log dir list.
  when(dirsHandlerForFullDisk.getLogDirs()).
      thenReturn(new ArrayList<String>());
  when(dirsHandlerForFullDisk.getLogDirsForRead()).
      thenReturn(Arrays.asList(new String[] {absLogDir.getAbsolutePath()}));
  nmContext = new NodeManager.NMContext(null, null, dirsHandlerForFullDisk,
      new ApplicationACLsManager(conf), new NMNullStateStoreService());
  nmContext.getApplications().put(appId, app);
  container.setState(ContainerState.RUNNING);
  nmContext.getContainers().put(container1, container);
  List<File> dirs =
      ContainerLogsUtils.getContainerLogDirs(container1, user, nmContext);
  File containerLogDir = new File(absLogDir, appId + "/" + container1);
  Assert.assertTrue(dirs.contains(containerLogDir));
}

org.apache.hadoop.yarn.server.nodemanager.NodeHealthCheckerService Java Examples