org.apache.hadoop.yarn.server.nodemanager.security.NMContainerTokenSecretManager Java Examples

The following examples show how to use org.apache.hadoop.yarn.server.nodemanager.security.NMContainerTokenSecretManager. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: Utils.java    From spliceengine with GNU Affero General Public License v3.0 6 votes vote down vote up 
public static void waitForNMToRegister(NodeManager nm) throws Exception{
    NMTokenSecretManagerInNM nmTokenSecretManagerNM =
            nm.getNMContext().getNMTokenSecretManager();
    NMContainerTokenSecretManager containerTokenSecretManager = nm.getNMContext().getContainerTokenSecretManager();
    int attempt = 60;
    while(attempt-- > 0) {
        try {
            if (nmTokenSecretManagerNM.getCurrentKey() != null && containerTokenSecretManager.getCurrentKey() != null) {
                break;
            }
        } catch (Exception e) {

        }
        Thread.sleep(2000);
    }
}
Example #2
Source File: TestContainerManager.java    From big-c with Apache License 2.0 6 votes vote down vote up 
public static Token createContainerToken(ContainerId cId, long rmIdentifier,
    NodeId nodeId, String user,
    NMContainerTokenSecretManager containerTokenSecretManager,
    LogAggregationContext logAggregationContext)
    throws IOException {
  Resource r = BuilderUtils.newResource(1024, 1);
  ContainerTokenIdentifier containerTokenIdentifier =
      new ContainerTokenIdentifier(cId, nodeId.toString(), user, r,
        System.currentTimeMillis() + 100000L, 123, rmIdentifier,
        Priority.newInstance(0), 0, logAggregationContext);
  Token containerToken =
      BuilderUtils
        .newContainerToken(nodeId, containerTokenSecretManager
          .retrievePassword(containerTokenIdentifier),
          containerTokenIdentifier);
  return containerToken;
}
Example #3
Source File: TestResourceLocalizationService.java    From big-c with Apache License 2.0 6 votes vote down vote up 
private ResourceLocalizationService createSpyService(
    DrainDispatcher dispatcher, LocalDirsHandlerService dirsHandler,
    NMStateStoreService stateStore) {
  ContainerExecutor exec = mock(ContainerExecutor.class);
  LocalizerTracker mockLocalizerTracker = mock(LocalizerTracker.class);
  DeletionService delService = mock(DeletionService.class);
  NMContext nmContext =
      new NMContext(new NMContainerTokenSecretManager(conf),
        new NMTokenSecretManagerInNM(), null,
        new ApplicationACLsManager(conf), stateStore,null);
  ResourceLocalizationService rawService =
    new ResourceLocalizationService(dispatcher, exec, delService,
                                    dirsHandler, nmContext);
  ResourceLocalizationService spyService = spy(rawService);
  doReturn(mockServer).when(spyService).createServer();
  doReturn(mockLocalizerTracker).when(spyService).createLocalizerTracker(
      isA(Configuration.class));
  doReturn(lfs).when(spyService)
      .getLocalFileContext(isA(Configuration.class));
  return spyService;
}
Example #4
Source File: TestLocalCacheDirectoryManager.java    From hadoop with Apache License 2.0 6 votes vote down vote up 
@Test(timeout = 10000)
public void testMinimumPerDirectoryFileLimit() {
  YarnConfiguration conf = new YarnConfiguration();
  conf.set(YarnConfiguration.NM_LOCAL_CACHE_MAX_FILES_PER_DIRECTORY, "1");
  Exception e = null;
  NMContext nmContext =
      new NMContext(new NMContainerTokenSecretManager(conf),
        new NMTokenSecretManagerInNM(), null,
        new ApplicationACLsManager(conf), new NMNullStateStoreService());
  ResourceLocalizationService service =
      new ResourceLocalizationService(null, null, null, null, nmContext);
  try {
    service.init(conf);
  } catch (Exception e1) {
    e = e1;
  }
  Assert.assertNotNull(e);
  Assert.assertEquals(YarnRuntimeException.class, e.getClass());
  Assert.assertEquals(e.getMessage(),
    YarnConfiguration.NM_LOCAL_CACHE_MAX_FILES_PER_DIRECTORY
        + " parameter is configured with a value less than 37.");

}
Example #5
Source File: TestLocalCacheDirectoryManager.java    From big-c with Apache License 2.0 6 votes vote down vote up 
@Test(timeout = 10000)
public void testMinimumPerDirectoryFileLimit() {
  YarnConfiguration conf = new YarnConfiguration();
  conf.set(YarnConfiguration.NM_LOCAL_CACHE_MAX_FILES_PER_DIRECTORY, "1");
  Exception e = null;
  NMContext nmContext =
      new NMContext(new NMContainerTokenSecretManager(conf),
        new NMTokenSecretManagerInNM(), null,
        new ApplicationACLsManager(conf), new NMNullStateStoreService(), null);
  ResourceLocalizationService service =
      new ResourceLocalizationService(null, null, null, null, nmContext);
  try {
    service.init(conf);
  } catch (Exception e1) {
    e = e1;
  }
  Assert.assertNotNull(e);
  Assert.assertEquals(YarnRuntimeException.class, e.getClass());
  Assert.assertEquals(e.getMessage(),
    YarnConfiguration.NM_LOCAL_CACHE_MAX_FILES_PER_DIRECTORY
        + " parameter is configured with a value less than 37.");

}
Example #6
Source File: TestResourceLocalizationService.java    From hadoop with Apache License 2.0 6 votes vote down vote up 
private ResourceLocalizationService createSpyService(
    DrainDispatcher dispatcher, LocalDirsHandlerService dirsHandler,
    NMStateStoreService stateStore) {
  ContainerExecutor exec = mock(ContainerExecutor.class);
  LocalizerTracker mockLocalizerTracker = mock(LocalizerTracker.class);
  DeletionService delService = mock(DeletionService.class);
  NMContext nmContext =
      new NMContext(new NMContainerTokenSecretManager(conf),
        new NMTokenSecretManagerInNM(), null,
        new ApplicationACLsManager(conf), stateStore);
  ResourceLocalizationService rawService =
    new ResourceLocalizationService(dispatcher, exec, delService,
                                    dirsHandler, nmContext);
  ResourceLocalizationService spyService = spy(rawService);
  doReturn(mockServer).when(spyService).createServer();
  doReturn(mockLocalizerTracker).when(spyService).createLocalizerTracker(
      isA(Configuration.class));
  doReturn(lfs).when(spyService)
      .getLocalFileContext(isA(Configuration.class));
  return spyService;
}
Example #7
Source File: TestContainerManager.java    From hadoop with Apache License 2.0 6 votes vote down vote up 
public static Token createContainerToken(ContainerId cId, long rmIdentifier,
    NodeId nodeId, String user,
    NMContainerTokenSecretManager containerTokenSecretManager,
    LogAggregationContext logAggregationContext)
    throws IOException {
  Resource r = BuilderUtils.newResource(1024, 1);
  ContainerTokenIdentifier containerTokenIdentifier =
      new ContainerTokenIdentifier(cId, nodeId.toString(), user, r,
        System.currentTimeMillis() + 100000L, 123, rmIdentifier,
        Priority.newInstance(0), 0, logAggregationContext);
  Token containerToken =
      BuilderUtils
        .newContainerToken(nodeId, containerTokenSecretManager
          .retrievePassword(containerTokenIdentifier),
          containerTokenIdentifier);
  return containerToken;
}
Example #8
Source File: NodeManager.java    From hadoop with Apache License 2.0 5 votes vote down vote up 
protected NMContext createNMContext(
    NMContainerTokenSecretManager containerTokenSecretManager,
    NMTokenSecretManagerInNM nmTokenSecretManager,
    NMStateStoreService stateStore) {
  return new NMContext(containerTokenSecretManager, nmTokenSecretManager,
      dirsHandler, aclsManager, stateStore);
}
Example #9
Source File: TestContainerManager.java    From big-c with Apache License 2.0 5 votes vote down vote up 
public static Token createContainerToken(ContainerId cId, long rmIdentifier,
    NodeId nodeId, String user,
    NMContainerTokenSecretManager containerTokenSecretManager)
    throws IOException {
  return createContainerToken(cId, rmIdentifier, nodeId, user,
    containerTokenSecretManager, null);
}
Example #10
Source File: TestResourceLocalizationService.java    From big-c with Apache License 2.0 5 votes vote down vote up 
@Before
public void setup() throws IOException {
  conf = new Configuration();
  spylfs = spy(FileContext.getLocalFSFileContext().getDefaultFileSystem());
  lfs = FileContext.getFileContext(spylfs, conf);

  String logDir = lfs.makeQualified(new Path(basedir, "logdir ")).toString();
  conf.set(YarnConfiguration.NM_LOG_DIRS, logDir);
  nmContext = new NMContext(new NMContainerTokenSecretManager(
    conf), new NMTokenSecretManagerInNM(), null,
    new ApplicationACLsManager(conf), new NMNullStateStoreService(), null);
}
Example #11
Source File: NodeManager.java    From big-c with Apache License 2.0 5 votes vote down vote up 
public NMContext(NMContainerTokenSecretManager containerTokenSecretManager,
    NMTokenSecretManagerInNM nmTokenSecretManager,
    LocalDirsHandlerService dirsHandler, ApplicationACLsManager aclsManager,
    NMStateStoreService stateStore, CoresManager coresManager) {
  this.containerTokenSecretManager = containerTokenSecretManager;
  this.nmTokenSecretManager = nmTokenSecretManager;
  this.dirsHandler = dirsHandler;
  this.aclsManager = aclsManager;
  this.nodeHealthStatus.setIsNodeHealthy(true);
  this.nodeHealthStatus.setHealthReport("Healthy");
  this.nodeHealthStatus.setLastHealthReportTime(System.currentTimeMillis());
  this.stateStore = stateStore;
  this.coresManager = coresManager;
}
Example #12
Source File: NodeManager.java    From big-c with Apache License 2.0 5 votes vote down vote up 
private void recoverTokens(NMTokenSecretManagerInNM nmTokenSecretManager,
    NMContainerTokenSecretManager containerTokenSecretManager)
        throws IOException {
  if (nmStore.canRecover()) {
    nmTokenSecretManager.recover();
    containerTokenSecretManager.recover();
  }
}
Example #13
Source File: NodeManager.java    From big-c with Apache License 2.0 5 votes vote down vote up 
protected NMContext createNMContext(
    NMContainerTokenSecretManager containerTokenSecretManager,
    NMTokenSecretManagerInNM nmTokenSecretManager,
    NMStateStoreService stateStore,
    CoresManager coresManager) {
  return new NMContext(containerTokenSecretManager, nmTokenSecretManager,
      dirsHandler, aclsManager, stateStore,coresManager);
}
Example #14
Source File: TestContainerManager.java    From hadoop with Apache License 2.0 5 votes vote down vote up 
public static Token createContainerToken(ContainerId cId, long rmIdentifier,
    NodeId nodeId, String user,
    NMContainerTokenSecretManager containerTokenSecretManager)
    throws IOException {
  return createContainerToken(cId, rmIdentifier, nodeId, user,
    containerTokenSecretManager, null);
}
Example #15
Source File: TestResourceLocalizationService.java    From hadoop with Apache License 2.0 5 votes vote down vote up 
@Before
public void setup() throws IOException {
  conf = new Configuration();
  spylfs = spy(FileContext.getLocalFSFileContext().getDefaultFileSystem());
  lfs = FileContext.getFileContext(spylfs, conf);

  String logDir = lfs.makeQualified(new Path(basedir, "logdir ")).toString();
  conf.set(YarnConfiguration.NM_LOG_DIRS, logDir);
  nmContext = new NMContext(new NMContainerTokenSecretManager(
    conf), new NMTokenSecretManagerInNM(), null,
    new ApplicationACLsManager(conf), new NMNullStateStoreService());
}
Example #16
Source File: NodeManager.java    From hadoop with Apache License 2.0 5 votes vote down vote up 
public NMContext(NMContainerTokenSecretManager containerTokenSecretManager,
    NMTokenSecretManagerInNM nmTokenSecretManager,
    LocalDirsHandlerService dirsHandler, ApplicationACLsManager aclsManager,
    NMStateStoreService stateStore) {
  this.containerTokenSecretManager = containerTokenSecretManager;
  this.nmTokenSecretManager = nmTokenSecretManager;
  this.dirsHandler = dirsHandler;
  this.aclsManager = aclsManager;
  this.nodeHealthStatus.setIsNodeHealthy(true);
  this.nodeHealthStatus.setHealthReport("Healthy");
  this.nodeHealthStatus.setLastHealthReportTime(System.currentTimeMillis());
  this.stateStore = stateStore;
}
Example #17
Source File: NodeManager.java    From hadoop with Apache License 2.0 5 votes vote down vote up 
private void recoverTokens(NMTokenSecretManagerInNM nmTokenSecretManager,
    NMContainerTokenSecretManager containerTokenSecretManager)
        throws IOException {
  if (nmStore.canRecover()) {
    nmTokenSecretManager.recover();
    containerTokenSecretManager.recover();
  }
}
Example #18
Source File: TestNodeStatusUpdater.java    From hadoop with Apache License 2.0 4 votes vote down vote up 
public MyNMContext(
    NMContainerTokenSecretManager containerTokenSecretManager,
    NMTokenSecretManagerInNM nmTokenSecretManager) {
  super(containerTokenSecretManager, nmTokenSecretManager, null, null,
      new NMNullStateStoreService());
}
Example #19
Source File: TestNodeStatusUpdater.java    From hadoop with Apache License 2.0 4 votes vote down vote up 
/**
 * Test completed containerStatus get back up when heart beat lost, and will
 * be sent via next heart beat.
 */
@Test(timeout = 200000)
public void testCompletedContainerStatusBackup() throws Exception {
  nm = new NodeManager() {
    @Override
    protected NodeStatusUpdater createNodeStatusUpdater(Context context,
        Dispatcher dispatcher, NodeHealthCheckerService healthChecker) {
      MyNodeStatusUpdater2 myNodeStatusUpdater =
          new MyNodeStatusUpdater2(context, dispatcher, healthChecker,
              metrics);
      return myNodeStatusUpdater;
    }

    @Override
    protected NMContext createNMContext(
        NMContainerTokenSecretManager containerTokenSecretManager,
        NMTokenSecretManagerInNM nmTokenSecretManager,
        NMStateStoreService store) {
      return new MyNMContext(containerTokenSecretManager,
        nmTokenSecretManager);
    }
  };

  YarnConfiguration conf = createNMConfig();
  nm.init(conf);
  nm.start();

  int waitCount = 0;
  while (heartBeatID <= 4 && waitCount++ != 20) {
    Thread.sleep(500);
  }
  if (heartBeatID <= 4) {
    Assert.fail("Failed to get all heartbeats in time, " +
        "heartbeatID:" + heartBeatID);
  }
  if(assertionFailedInThread.get()) {
    Assert.fail("ContainerStatus Backup failed");
  }
  Assert.assertNotNull(nm.getNMContext().getSystemCredentialsForApps()
    .get(ApplicationId.newInstance(1234, 1)).getToken(new Text("token1")));
  nm.stop();
}
Example #20
Source File: NodeManager.java    From big-c with Apache License 2.0 4 votes vote down vote up 
@Override
protected void serviceInit(Configuration conf) throws Exception {

  conf.setBoolean(Dispatcher.DISPATCHER_EXIT_ON_ERROR_KEY, true);

  rmWorkPreservingRestartEnabled = conf.getBoolean(YarnConfiguration
          .RM_WORK_PRESERVING_RECOVERY_ENABLED,
      YarnConfiguration.DEFAULT_RM_WORK_PRESERVING_RECOVERY_ENABLED);

  initAndStartRecoveryStore(conf);

  NMContainerTokenSecretManager containerTokenSecretManager =
      new NMContainerTokenSecretManager(conf, nmStore);

  NMTokenSecretManagerInNM nmTokenSecretManager =
      new NMTokenSecretManagerInNM(nmStore);

  recoverTokens(nmTokenSecretManager, containerTokenSecretManager);
  
  this.aclsManager = new ApplicationACLsManager(conf);

  ContainerExecutor exec = ReflectionUtils.newInstance(
      conf.getClass(YarnConfiguration.NM_CONTAINER_EXECUTOR,
        DefaultContainerExecutor.class, ContainerExecutor.class), conf);
  try {
    exec.init();
  } catch (IOException e) {
    throw new YarnRuntimeException("Failed to initialize container executor", e);
  }    
  DeletionService del = createDeletionService(exec);
  addService(del);

  // NodeManager level dispatcher
  this.dispatcher = new AsyncDispatcher();
  
  this.coresManager = new CoresManagerImpl();
  this.coresManager.init(conf);
  
  nodeHealthChecker = new NodeHealthCheckerService();
  addService(nodeHealthChecker);
  dirsHandler = nodeHealthChecker.getDiskHandler();

  this.context = createNMContext(containerTokenSecretManager,
      nmTokenSecretManager, nmStore, coresManager);
  
  nodeStatusUpdater =
      createNodeStatusUpdater(context, dispatcher, nodeHealthChecker);

  NodeResourceMonitor nodeResourceMonitor = createNodeResourceMonitor();
  addService(nodeResourceMonitor);

  containerManager =
      createContainerManager(context, exec, del, nodeStatusUpdater,
      this.aclsManager, dirsHandler);
  addService(containerManager);
  ((NMContext) context).setContainerManager(containerManager);

  WebServer webServer = createWebServer(context, containerManager
      .getContainersMonitor(), this.aclsManager, dirsHandler);
  addService(webServer);
  ((NMContext) context).setWebServer(webServer);

  dispatcher.register(ContainerManagerEventType.class, containerManager);
  dispatcher.register(NodeManagerEventType.class, this);
  addService(dispatcher);
  
  DefaultMetricsSystem.initialize("NodeManager");

  // StatusUpdater should be added last so that it get started last 
  // so that we make sure everything is up before registering with RM. 
  addService(nodeStatusUpdater);
  
  super.serviceInit(conf);
  // TODO add local dirs to del
}
Example #21
Source File: TestApplication.java    From hadoop with Apache License 2.0 4 votes vote down vote up 
WrappedApplication(int id, long timestamp, String user, int numContainers) {
  Configuration conf = new Configuration();
  
  dispatcher = new DrainDispatcher();
  containerTokenIdentifierMap =
      new HashMap<ContainerId, ContainerTokenIdentifier>();
  dispatcher.init(conf);

  localizerBus = mock(EventHandler.class);
  launcherBus = mock(EventHandler.class);
  monitorBus = mock(EventHandler.class);
  auxBus = mock(EventHandler.class);
  containerBus = mock(EventHandler.class);
  logAggregationBus = mock(EventHandler.class);

  dispatcher.register(LocalizationEventType.class, localizerBus);
  dispatcher.register(ContainersLauncherEventType.class, launcherBus);
  dispatcher.register(ContainersMonitorEventType.class, monitorBus);
  dispatcher.register(AuxServicesEventType.class, auxBus);
  dispatcher.register(ContainerEventType.class, containerBus);
  dispatcher.register(LogHandlerEventType.class, logAggregationBus);

  nmTokenSecretMgr = mock(NMTokenSecretManagerInNM.class);

  context = mock(Context.class);
  
  when(context.getContainerTokenSecretManager()).thenReturn(
    new NMContainerTokenSecretManager(conf));
  when(context.getApplicationACLsManager()).thenReturn(
    new ApplicationACLsManager(conf));
  when(context.getNMTokenSecretManager()).thenReturn(nmTokenSecretMgr);
  
  // Setting master key
  MasterKey masterKey = new MasterKeyPBImpl();
  masterKey.setKeyId(123);
  masterKey.setBytes(ByteBuffer.wrap(new byte[] { (new Integer(123)
    .byteValue()) }));
  context.getContainerTokenSecretManager().setMasterKey(masterKey);
  
  this.user = user;
  this.appId = BuilderUtils.newApplicationId(timestamp, id);

  app = new ApplicationImpl(dispatcher, this.user, appId, null, context);
  containers = new ArrayList<Container>();
  for (int i = 0; i < numContainers; i++) {
    Container container = createMockedContainer(this.appId, i);
    containers.add(container);
    long currentTime = System.currentTimeMillis();
    ContainerTokenIdentifier identifier =
        new ContainerTokenIdentifier(container.getContainerId(), "", "",
          null, currentTime + 2000, masterKey.getKeyId(), currentTime,
          Priority.newInstance(0), 0);
    containerTokenIdentifierMap
      .put(identifier.getContainerID(), identifier);
    context.getContainerTokenSecretManager().startContainerSuccessful(
      identifier);
    Assert.assertFalse(context.getContainerTokenSecretManager()
      .isValidStartContainerRequest(identifier));
  }

  dispatcher.start();
}
Example #22
Source File: NodeManager.java    From big-c with Apache License 2.0 4 votes vote down vote up 
@Override
public NMContainerTokenSecretManager getContainerTokenSecretManager() {
  return this.containerTokenSecretManager;
}
Example #23
Source File: NodeManager.java    From hadoop with Apache License 2.0 4 votes vote down vote up 
@Override
public NMContainerTokenSecretManager getContainerTokenSecretManager() {
  return this.containerTokenSecretManager;
}
Example #24
Source File: TestApplication.java    From big-c with Apache License 2.0 4 votes vote down vote up 
WrappedApplication(int id, long timestamp, String user, int numContainers) {
  Configuration conf = new Configuration();
  
  dispatcher = new DrainDispatcher();
  containerTokenIdentifierMap =
      new HashMap<ContainerId, ContainerTokenIdentifier>();
  dispatcher.init(conf);

  localizerBus = mock(EventHandler.class);
  launcherBus = mock(EventHandler.class);
  monitorBus = mock(EventHandler.class);
  auxBus = mock(EventHandler.class);
  containerBus = mock(EventHandler.class);
  logAggregationBus = mock(EventHandler.class);

  dispatcher.register(LocalizationEventType.class, localizerBus);
  dispatcher.register(ContainersLauncherEventType.class, launcherBus);
  dispatcher.register(ContainersMonitorEventType.class, monitorBus);
  dispatcher.register(AuxServicesEventType.class, auxBus);
  dispatcher.register(ContainerEventType.class, containerBus);
  dispatcher.register(LogHandlerEventType.class, logAggregationBus);

  nmTokenSecretMgr = mock(NMTokenSecretManagerInNM.class);

  context = mock(Context.class);
  
  when(context.getContainerTokenSecretManager()).thenReturn(
    new NMContainerTokenSecretManager(conf));
  when(context.getApplicationACLsManager()).thenReturn(
    new ApplicationACLsManager(conf));
  when(context.getNMTokenSecretManager()).thenReturn(nmTokenSecretMgr);
  
  // Setting master key
  MasterKey masterKey = new MasterKeyPBImpl();
  masterKey.setKeyId(123);
  masterKey.setBytes(ByteBuffer.wrap(new byte[] { (new Integer(123)
    .byteValue()) }));
  context.getContainerTokenSecretManager().setMasterKey(masterKey);
  
  this.user = user;
  this.appId = BuilderUtils.newApplicationId(timestamp, id);

  app = new ApplicationImpl(dispatcher, this.user, appId, null, context);
  containers = new ArrayList<Container>();
  for (int i = 0; i < numContainers; i++) {
    Container container = createMockedContainer(this.appId, i);
    containers.add(container);
    long currentTime = System.currentTimeMillis();
    ContainerTokenIdentifier identifier =
        new ContainerTokenIdentifier(container.getContainerId(), "", "",
          null, currentTime + 2000, masterKey.getKeyId(), currentTime,
          Priority.newInstance(0), 0);
    containerTokenIdentifierMap
      .put(identifier.getContainerID(), identifier);
    context.getContainerTokenSecretManager().startContainerSuccessful(
      identifier);
    Assert.assertFalse(context.getContainerTokenSecretManager()
      .isValidStartContainerRequest(identifier));
  }

  dispatcher.start();
}
Example #25
Source File: NodeManager.java    From hadoop with Apache License 2.0 4 votes vote down vote up 
@Override
protected void serviceInit(Configuration conf) throws Exception {

  conf.setBoolean(Dispatcher.DISPATCHER_EXIT_ON_ERROR_KEY, true);

  rmWorkPreservingRestartEnabled = conf.getBoolean(YarnConfiguration
          .RM_WORK_PRESERVING_RECOVERY_ENABLED,
      YarnConfiguration.DEFAULT_RM_WORK_PRESERVING_RECOVERY_ENABLED);

  initAndStartRecoveryStore(conf);

  NMContainerTokenSecretManager containerTokenSecretManager =
      new NMContainerTokenSecretManager(conf, nmStore);

  NMTokenSecretManagerInNM nmTokenSecretManager =
      new NMTokenSecretManagerInNM(nmStore);

  recoverTokens(nmTokenSecretManager, containerTokenSecretManager);
  
  this.aclsManager = new ApplicationACLsManager(conf);

  ContainerExecutor exec = ReflectionUtils.newInstance(
      conf.getClass(YarnConfiguration.NM_CONTAINER_EXECUTOR,
        DefaultContainerExecutor.class, ContainerExecutor.class), conf);
  try {
    exec.init();
  } catch (IOException e) {
    throw new YarnRuntimeException("Failed to initialize container executor", e);
  }    
  DeletionService del = createDeletionService(exec);
  addService(del);

  // NodeManager level dispatcher
  this.dispatcher = new AsyncDispatcher();

  dirsHandler = new LocalDirsHandlerService(metrics);
  nodeHealthChecker =
      new NodeHealthCheckerService(
          getNodeHealthScriptRunner(conf), dirsHandler);
  addService(nodeHealthChecker);

  this.context = createNMContext(containerTokenSecretManager,
      nmTokenSecretManager, nmStore);
  
  nodeStatusUpdater =
      createNodeStatusUpdater(context, dispatcher, nodeHealthChecker);

  NodeResourceMonitor nodeResourceMonitor = createNodeResourceMonitor();
  addService(nodeResourceMonitor);

  containerManager =
      createContainerManager(context, exec, del, nodeStatusUpdater,
      this.aclsManager, dirsHandler);
  addService(containerManager);
  ((NMContext) context).setContainerManager(containerManager);

  WebServer webServer = createWebServer(context, containerManager
      .getContainersMonitor(), this.aclsManager, dirsHandler);
  addService(webServer);
  ((NMContext) context).setWebServer(webServer);

  dispatcher.register(ContainerManagerEventType.class, containerManager);
  dispatcher.register(NodeManagerEventType.class, this);
  addService(dispatcher);
  
  DefaultMetricsSystem.initialize("NodeManager");

  // StatusUpdater should be added last so that it get started last 
  // so that we make sure everything is up before registering with RM. 
  addService(nodeStatusUpdater);
  
  super.serviceInit(conf);
  // TODO add local dirs to del
}
Example #26
Source File: TestNodeStatusUpdater.java    From big-c with Apache License 2.0 4 votes vote down vote up 
/**
 * Test completed containerStatus get back up when heart beat lost, and will
 * be sent via next heart beat.
 */
@Test(timeout = 200000)
public void testCompletedContainerStatusBackup() throws Exception {
  nm = new NodeManager() {
    @Override
    protected NodeStatusUpdater createNodeStatusUpdater(Context context,
        Dispatcher dispatcher, NodeHealthCheckerService healthChecker) {
      MyNodeStatusUpdater2 myNodeStatusUpdater =
          new MyNodeStatusUpdater2(context, dispatcher, healthChecker,
              metrics);
      return myNodeStatusUpdater;
    }

    //@Override
    protected NMContext createNMContext(
        NMContainerTokenSecretManager containerTokenSecretManager,
        NMTokenSecretManagerInNM nmTokenSecretManager,
        NMStateStoreService store) {
      return new MyNMContext(containerTokenSecretManager,
        nmTokenSecretManager);
    }
  };

  YarnConfiguration conf = createNMConfig();
  nm.init(conf);
  nm.start();

  int waitCount = 0;
  while (heartBeatID <= 4 && waitCount++ != 20) {
    Thread.sleep(500);
  }
  if (heartBeatID <= 4) {
    Assert.fail("Failed to get all heartbeats in time, " +
        "heartbeatID:" + heartBeatID);
  }
  if(assertionFailedInThread.get()) {
    Assert.fail("ContainerStatus Backup failed");
  }
  Assert.assertNotNull(nm.getNMContext().getSystemCredentialsForApps()
    .get(ApplicationId.newInstance(1234, 1)).getToken(new Text("token1")));
  nm.stop();
}
Example #27
Source File: TestNodeStatusUpdater.java    From big-c with Apache License 2.0 4 votes vote down vote up 
public MyNMContext(
    NMContainerTokenSecretManager containerTokenSecretManager,
    NMTokenSecretManagerInNM nmTokenSecretManager) {
  super(containerTokenSecretManager, nmTokenSecretManager, null, null,
      new NMNullStateStoreService(), null);
}
Example #28
Source File: Context.java    From big-c with Apache License 2.0 votes vote down vote up 
NMContainerTokenSecretManager getContainerTokenSecretManager();
Example #29
Source File: Context.java    From hadoop with Apache License 2.0 votes vote down vote up 
NMContainerTokenSecretManager getContainerTokenSecretManager();