org.apache.hadoop.yarn.server.security.ApplicationACLsManager Java Examples

@BeforeClass
public static void setupClass() throws Exception {
  Configuration conf = new YarnConfiguration();
  TimelineStore store =
      TestApplicationHistoryManagerOnTimelineStore.createStore(MAX_APPS);
  TimelineACLsManager aclsManager = new TimelineACLsManager(conf);
  TimelineDataManager dataManager =
      new TimelineDataManager(store, aclsManager);
  conf.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, true);
  conf.set(YarnConfiguration.YARN_ADMIN_ACL, "foo");
  ApplicationACLsManager appAclsManager = new ApplicationACLsManager(conf);
  ApplicationHistoryManagerOnTimelineStore historyManager =
      new ApplicationHistoryManagerOnTimelineStore(dataManager, appAclsManager);
  historyManager.init(conf);
  historyClientService = new ApplicationHistoryClientService(historyManager) {
    @Override
    protected void serviceStart() throws Exception {
      // Do Nothing
    }
  };
  historyClientService.init(conf);
  historyClientService.start();
}
 

@Override
public void setup() {
  bind(NMWebServices.class);
  bind(GenericExceptionHandler.class);
  bind(JAXBContextResolver.class);
  bind(ResourceView.class).toInstance(this.resourceView);
  bind(ApplicationACLsManager.class).toInstance(this.aclsManager);
  bind(LocalDirsHandlerService.class).toInstance(dirsHandler);
  route("/", NMController.class, "info");
  route("/node", NMController.class, "node");
  route("/allApplications", NMController.class, "allApplications");
  route("/allContainers", NMController.class, "allContainers");
  route(pajoin("/application", APPLICATION_ID), NMController.class,
      "application");
  route(pajoin("/container", CONTAINER_ID), NMController.class,
      "container");
  route(
      pajoin("/containerlogs", CONTAINER_ID, APP_OWNER, CONTAINER_LOG_TYPE),
      NMController.class, "logs");
}
 

@BeforeClass
public static void setupClass() throws Exception {
  Configuration conf = new YarnConfiguration();
  TimelineStore store =
      TestApplicationHistoryManagerOnTimelineStore.createStore(MAX_APPS);
  TimelineACLsManager aclsManager = new TimelineACLsManager(conf);
  TimelineDataManager dataManager =
      new TimelineDataManager(store, aclsManager);
  conf.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, true);
  conf.set(YarnConfiguration.YARN_ADMIN_ACL, "foo");
  ApplicationACLsManager appAclsManager = new ApplicationACLsManager(conf);
  ApplicationHistoryManagerOnTimelineStore historyManager =
      new ApplicationHistoryManagerOnTimelineStore(dataManager, appAclsManager);
  historyManager.init(conf);
  historyClientService = new ApplicationHistoryClientService(historyManager) {
    @Override
    protected void serviceStart() throws Exception {
      // Do Nothing
    }
  };
  historyClientService.init(conf);
  historyClientService.start();
}
 

public RMAppManager(RMContext context,
    YarnScheduler scheduler, ApplicationMasterService masterService,
    ApplicationACLsManager applicationACLsManager, Configuration conf) {
  this.rmContext = context;
  this.scheduler = scheduler;
  this.masterService = masterService;
  this.applicationACLsManager = applicationACLsManager;
  this.conf = conf;
  this.maxCompletedAppsInMemory = conf.getInt(
      YarnConfiguration.RM_MAX_COMPLETED_APPLICATIONS,
      YarnConfiguration.DEFAULT_RM_MAX_COMPLETED_APPLICATIONS);
  this.maxCompletedAppsInStateStore =
      conf.getInt(
        YarnConfiguration.RM_STATE_STORE_MAX_COMPLETED_APPLICATIONS,
        YarnConfiguration.DEFAULT_RM_STATE_STORE_MAX_COMPLETED_APPLICATIONS);
  if (this.maxCompletedAppsInStateStore > this.maxCompletedAppsInMemory) {
    this.maxCompletedAppsInStateStore = this.maxCompletedAppsInMemory;
  }
}
 

private ResourceLocalizationService createSpyService(
    DrainDispatcher dispatcher, LocalDirsHandlerService dirsHandler,
    NMStateStoreService stateStore) {
  ContainerExecutor exec = mock(ContainerExecutor.class);
  LocalizerTracker mockLocalizerTracker = mock(LocalizerTracker.class);
  DeletionService delService = mock(DeletionService.class);
  NMContext nmContext =
      new NMContext(new NMContainerTokenSecretManager(conf),
        new NMTokenSecretManagerInNM(), null,
        new ApplicationACLsManager(conf), stateStore,null);
  ResourceLocalizationService rawService =
    new ResourceLocalizationService(dispatcher, exec, delService,
                                    dirsHandler, nmContext);
  ResourceLocalizationService spyService = spy(rawService);
  doReturn(mockServer).when(spyService).createServer();
  doReturn(mockLocalizerTracker).when(spyService).createLocalizerTracker(
      isA(Configuration.class));
  doReturn(lfs).when(spyService)
      .getLocalFileContext(isA(Configuration.class));
  return spyService;
}
 

@Test(timeout = 10000)
public void testMinimumPerDirectoryFileLimit() {
  YarnConfiguration conf = new YarnConfiguration();
  conf.set(YarnConfiguration.NM_LOCAL_CACHE_MAX_FILES_PER_DIRECTORY, "1");
  Exception e = null;
  NMContext nmContext =
      new NMContext(new NMContainerTokenSecretManager(conf),
        new NMTokenSecretManagerInNM(), null,
        new ApplicationACLsManager(conf), new NMNullStateStoreService());
  ResourceLocalizationService service =
      new ResourceLocalizationService(null, null, null, null, nmContext);
  try {
    service.init(conf);
  } catch (Exception e1) {
    e = e1;
  }
  Assert.assertNotNull(e);
  Assert.assertEquals(YarnRuntimeException.class, e.getClass());
  Assert.assertEquals(e.getMessage(),
    YarnConfiguration.NM_LOCAL_CACHE_MAX_FILES_PER_DIRECTORY
        + " parameter is configured with a value less than 37.");

}
 

public RMAppManager(RMContext context,
    YarnScheduler scheduler, ApplicationMasterService masterService,
    ApplicationACLsManager applicationACLsManager, Configuration conf) {
  this.rmContext = context;
  this.scheduler = scheduler;
  this.masterService = masterService;
  this.applicationACLsManager = applicationACLsManager;
  this.conf = conf;
  this.maxCompletedAppsInMemory = conf.getInt(
      YarnConfiguration.RM_MAX_COMPLETED_APPLICATIONS,
      YarnConfiguration.DEFAULT_RM_MAX_COMPLETED_APPLICATIONS);
  this.maxCompletedAppsInStateStore =
      conf.getInt(
        YarnConfiguration.RM_STATE_STORE_MAX_COMPLETED_APPLICATIONS,
        YarnConfiguration.DEFAULT_RM_STATE_STORE_MAX_COMPLETED_APPLICATIONS);
  if (this.maxCompletedAppsInStateStore > this.maxCompletedAppsInMemory) {
    this.maxCompletedAppsInStateStore = this.maxCompletedAppsInMemory;
  }
}
 

public ClientRMService createRMService() throws IOException {
  YarnScheduler yarnScheduler = mockYarnScheduler();
  RMContext rmContext = mock(RMContext.class);
  mockRMContext(yarnScheduler, rmContext);
  ConcurrentHashMap<ApplicationId, RMApp> apps = getRMApps(rmContext,
      yarnScheduler);
  when(rmContext.getRMApps()).thenReturn(apps);
  when(rmContext.getYarnConfiguration()).thenReturn(new Configuration());
  RMAppManager appManager = new RMAppManager(rmContext, yarnScheduler, null,
      mock(ApplicationACLsManager.class), new Configuration());
  when(rmContext.getDispatcher().getEventHandler()).thenReturn(
      new EventHandler<Event>() {
        public void handle(Event event) {
        }
      });

  ApplicationACLsManager mockAclsManager = mock(ApplicationACLsManager.class);
  QueueACLsManager mockQueueACLsManager = mock(QueueACLsManager.class);
  when(
      mockQueueACLsManager.checkAccess(any(UserGroupInformation.class),
          any(QueueACL.class), anyString())).thenReturn(true);
  return new ClientRMService(rmContext, yarnScheduler, appManager,
      mockAclsManager, mockQueueACLsManager, null);
}
 

@SuppressWarnings("deprecation")
@Before
public void setUp() {
  long now = System.currentTimeMillis();

  rmContext = mockRMContext(1, now - 10);
  ResourceScheduler scheduler = mockResourceScheduler();
  Configuration conf = new Configuration();
  ApplicationMasterService masterService =
      new ApplicationMasterService(rmContext, scheduler);
  appMonitor = new TestRMAppManager(rmContext,
      new ClientToAMTokenSecretManagerInRM(), scheduler, masterService,
      new ApplicationACLsManager(conf), conf);

  appId = MockApps.newAppID(1);
  RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null);
  asContext =
      recordFactory.newRecordInstance(ApplicationSubmissionContext.class);
  asContext.setApplicationId(appId);
  asContext.setAMContainerSpec(mockContainerLaunchContext(recordFactory));
  asContext.setResource(mockResource());
  setupDispatcher(rmContext, conf);
}
 

public ClientRMService createRMService() throws IOException {
  YarnScheduler yarnScheduler = mockYarnScheduler();
  RMContext rmContext = mock(RMContext.class);
  mockRMContext(yarnScheduler, rmContext);
  ConcurrentHashMap<ApplicationId, RMApp> apps = getRMApps(rmContext,
      yarnScheduler);
  when(rmContext.getRMApps()).thenReturn(apps);
  when(rmContext.getYarnConfiguration()).thenReturn(new Configuration());
  RMAppManager appManager = new RMAppManager(rmContext, yarnScheduler, null,
      mock(ApplicationACLsManager.class), new Configuration());
  when(rmContext.getDispatcher().getEventHandler()).thenReturn(
      new EventHandler<Event>() {
        public void handle(Event event) {
        }
      });

  ApplicationACLsManager mockAclsManager = mock(ApplicationACLsManager.class);
  QueueACLsManager mockQueueACLsManager = mock(QueueACLsManager.class);
  when(
      mockQueueACLsManager.checkAccess(any(UserGroupInformation.class),
          any(QueueACL.class), anyString())).thenReturn(true);
  return new ClientRMService(rmContext, yarnScheduler, appManager,
      mockAclsManager, mockQueueACLsManager, null);
}
 

@Before
public void setup() throws IOException {
  conf = new Configuration();
  spylfs = spy(FileContext.getLocalFSFileContext().getDefaultFileSystem());
  lfs = FileContext.getFileContext(spylfs, conf);

  String logDir = lfs.makeQualified(new Path(basedir, "logdir ")).toString();
  conf.set(YarnConfiguration.NM_LOG_DIRS, logDir);
  nmContext = new NMContext(new NMContainerTokenSecretManager(
    conf), new NMTokenSecretManagerInNM(), null,
    new ApplicationACLsManager(conf), new NMNullStateStoreService(), null);
}
 

@Test
public void testGetApplicationReport() throws Exception {
  YarnScheduler yarnScheduler = mock(YarnScheduler.class);
  RMContext rmContext = mock(RMContext.class);
  mockRMContext(yarnScheduler, rmContext);

  ApplicationId appId1 = getApplicationId(1);

  ApplicationACLsManager mockAclsManager = mock(ApplicationACLsManager.class);
  when(
      mockAclsManager.checkAccess(UserGroupInformation.getCurrentUser(),
          ApplicationAccessType.VIEW_APP, null, appId1)).thenReturn(true);

  ClientRMService rmService = new ClientRMService(rmContext, yarnScheduler,
      null, mockAclsManager, null, null);
  try {
    RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null);
    GetApplicationReportRequest request = recordFactory
        .newRecordInstance(GetApplicationReportRequest.class);
    request.setApplicationId(appId1);
    GetApplicationReportResponse response = 
        rmService.getApplicationReport(request);
    ApplicationReport report = response.getApplicationReport();
    ApplicationResourceUsageReport usageReport = 
        report.getApplicationResourceUsageReport();
    Assert.assertEquals(10, usageReport.getMemorySeconds());
    Assert.assertEquals(3, usageReport.getVcoreSeconds());
  } finally {
    rmService.close();
  }
}
 

public ClientRMService(RMContext rmContext, YarnScheduler scheduler,
    RMAppManager rmAppManager, ApplicationACLsManager applicationACLsManager,
    QueueACLsManager queueACLsManager,
    RMDelegationTokenSecretManager rmDTSecretManager) {
  this(rmContext, scheduler, rmAppManager, applicationACLsManager,
      queueACLsManager, rmDTSecretManager, new UTCClock());
}
 

@Override
protected ContainerManagerImpl createContainerManager(Context context,
    ContainerExecutor exec, DeletionService del,
    NodeStatusUpdater nodeStatusUpdater, ApplicationACLsManager aclsManager,
    LocalDirsHandlerService dirsHandler) {
  return new ContainerManagerImpl(context, exec, del, nodeStatusUpdater,
    metrics, aclsManager, dirsHandler){
    @Override
    public void setBlockNewContainerRequests(
        boolean blockNewContainerRequests) {
      if (blockNewContainerRequests) {
        // start test thread right after blockNewContainerRequests is set
        // true
        super.setBlockNewContainerRequests(blockNewContainerRequests);
        launchContainersThread = new RejectedContainersLauncherThread();
        launchContainersThread.start();
      } else {
        // join the test thread right before blockNewContainerRequests is
        // reset
        try {
          // stop the test thread
          ((RejectedContainersLauncherThread) launchContainersThread)
            .setStopThreadFlag(true);
          launchContainersThread.join();
          ((RejectedContainersLauncherThread) launchContainersThread)
          .setStopThreadFlag(false);
          super.setBlockNewContainerRequests(blockNewContainerRequests);
        } catch (InterruptedException e) {
          e.printStackTrace();
        }
      }
    }
  };
}
 

/**
 * Verifies that if for some reason NM fails to start ContainerManager RPC
 * server, RM is oblivious to NM's presence. The behaviour is like this
 * because otherwise, NM will report to RM even if all its servers are not
 * started properly, RM will think that the NM is alive and will retire the NM
 * only after NM_EXPIRY interval. See MAPREDUCE-2749.
 */
@Test
public void testNoRegistrationWhenNMServicesFail() throws Exception {

  nm = new NodeManager() {
    @Override
    protected NodeStatusUpdater createNodeStatusUpdater(Context context,
        Dispatcher dispatcher, NodeHealthCheckerService healthChecker) {
      return new MyNodeStatusUpdater(context, dispatcher, healthChecker,
                                     metrics);
    }

    @Override
    protected ContainerManagerImpl createContainerManager(Context context,
        ContainerExecutor exec, DeletionService del,
        NodeStatusUpdater nodeStatusUpdater,
        ApplicationACLsManager aclsManager,
        LocalDirsHandlerService diskhandler) {
      return new ContainerManagerImpl(context, exec, del, nodeStatusUpdater,
        metrics, aclsManager, diskhandler) {
        @Override
        protected void serviceStart() {
          // Simulating failure of starting RPC server
          throw new YarnRuntimeException("Starting of RPC Server failed");
        }
      };
    }
  };

  verifyNodeStartFailure("Starting of RPC Server failed");
}
 

public ApplicationHistoryManagerOnTimelineStore(
    TimelineDataManager timelineDataManager,
    ApplicationACLsManager aclsManager) {
  super(ApplicationHistoryManagerOnTimelineStore.class.getName());
  this.timelineDataManager = timelineDataManager;
  this.aclsManager = aclsManager;
}
 

public ApplicationHistoryManagerOnTimelineStore(
    TimelineDataManager timelineDataManager,
    ApplicationACLsManager aclsManager) {
  super(ApplicationHistoryManagerOnTimelineStore.class.getName());
  this.timelineDataManager = timelineDataManager;
  this.aclsManager = aclsManager;
}
 

public CustomedClientRMService(RMContext rmContext,
    YarnScheduler scheduler, RMAppManager rmAppManager,
    ApplicationACLsManager applicationACLsManager,
    QueueACLsManager queueACLsManager,
    RMDelegationTokenSecretManager rmDTSecretManager) {
  super(rmContext, scheduler, rmAppManager, applicationACLsManager,
      queueACLsManager, rmDTSecretManager);
}
 

public WebServer(Context nmContext, ResourceView resourceView,
    ApplicationACLsManager aclsManager,
    LocalDirsHandlerService dirsHandler) {
  super(WebServer.class.getName());
  this.nmContext = nmContext;
  this.nmWebApp = new NMWebApp(resourceView, aclsManager, dirsHandler);
}
 

@BeforeClass
public static void setup() throws Exception {
  Configuration conf = new YarnConfiguration();
  TimelineStore store =
      TestApplicationHistoryManagerOnTimelineStore.createStore(MAX_APPS);
  TimelineACLsManager aclsManager = new TimelineACLsManager(conf);
  TimelineDataManager dataManager =
      new TimelineDataManager(store, aclsManager);
  ApplicationACLsManager appAclsManager = new ApplicationACLsManager(conf);
  ApplicationHistoryManagerOnTimelineStore historyManager =
      new ApplicationHistoryManagerOnTimelineStore(dataManager, appAclsManager);
  historyManager.init(conf);
  historyManager.start();
  clientService = new ApplicationHistoryClientService(historyManager);
}
 

public NMContext(NMContainerTokenSecretManager containerTokenSecretManager,
    NMTokenSecretManagerInNM nmTokenSecretManager,
    LocalDirsHandlerService dirsHandler, ApplicationACLsManager aclsManager,
    NMStateStoreService stateStore, CoresManager coresManager) {
  this.containerTokenSecretManager = containerTokenSecretManager;
  this.nmTokenSecretManager = nmTokenSecretManager;
  this.dirsHandler = dirsHandler;
  this.aclsManager = aclsManager;
  this.nodeHealthStatus.setIsNodeHealthy(true);
  this.nodeHealthStatus.setHealthReport("Healthy");
  this.nodeHealthStatus.setLastHealthReportTime(System.currentTimeMillis());
  this.stateStore = stateStore;
  this.coresManager = coresManager;
}
 

@Before
public void setup() throws Exception {
  // Only test the ACLs of the generic history
  TimelineACLsManager aclsManager = new TimelineACLsManager(new YarnConfiguration());
  TimelineDataManager dataManager =
      new TimelineDataManager(store, aclsManager);
  ApplicationACLsManager appAclsManager = new ApplicationACLsManager(conf);
  historyManager =
      new ApplicationHistoryManagerOnTimelineStore(dataManager, appAclsManager);
  historyManager.init(conf);
  historyManager.start();
}
 

@Test (timeout = 30000)
public void testRMAppSubmitMaxAppAttempts() throws Exception {
  int[] globalMaxAppAttempts = new int[] { 10, 1 };
  int[][] individualMaxAppAttempts = new int[][]{
      new int[]{ 9, 10, 11, 0 },
      new int[]{ 1, 10, 0, -1 }};
  int[][] expectedNums = new int[][]{
      new int[]{ 9, 10, 10, 10 },
      new int[]{ 1, 1, 1, 1 }};
  for (int i = 0; i < globalMaxAppAttempts.length; ++i) {
    for (int j = 0; j < individualMaxAppAttempts.length; ++j) {
      ResourceScheduler scheduler = mockResourceScheduler();
      Configuration conf = new Configuration();
      conf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, globalMaxAppAttempts[i]);
      ApplicationMasterService masterService =
          new ApplicationMasterService(rmContext, scheduler);
      TestRMAppManager appMonitor = new TestRMAppManager(rmContext,
          new ClientToAMTokenSecretManagerInRM(), scheduler, masterService,
          new ApplicationACLsManager(conf), conf);

      ApplicationId appID = MockApps.newAppID(i * 4 + j + 1);
      asContext.setApplicationId(appID);
      if (individualMaxAppAttempts[i][j] != 0) {
        asContext.setMaxAppAttempts(individualMaxAppAttempts[i][j]);
      }
      appMonitor.submitApplication(asContext, "test");
      RMApp app = rmContext.getRMApps().get(appID);
      Assert.assertEquals("max application attempts doesn't match",
          expectedNums[i][j], app.getMaxAppAttempts());

      // wait for event to be processed
      int timeoutSecs = 0;
      while ((getAppEventType() == RMAppEventType.KILL) &&
          timeoutSecs++ < 20) {
        Thread.sleep(1000);
      }
      setAppEventType(RMAppEventType.KILL);
    }
  }
}
 

@Test
public void testGetApplicationReport() throws Exception {
  YarnScheduler yarnScheduler = mock(YarnScheduler.class);
  RMContext rmContext = mock(RMContext.class);
  mockRMContext(yarnScheduler, rmContext);

  ApplicationId appId1 = getApplicationId(1);

  ApplicationACLsManager mockAclsManager = mock(ApplicationACLsManager.class);
  when(
      mockAclsManager.checkAccess(UserGroupInformation.getCurrentUser(),
          ApplicationAccessType.VIEW_APP, null, appId1)).thenReturn(true);

  ClientRMService rmService = new ClientRMService(rmContext, yarnScheduler,
      null, mockAclsManager, null, null);
  try {
    RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null);
    GetApplicationReportRequest request = recordFactory
        .newRecordInstance(GetApplicationReportRequest.class);
    request.setApplicationId(appId1);
    GetApplicationReportResponse response = 
        rmService.getApplicationReport(request);
    ApplicationReport report = response.getApplicationReport();
    ApplicationResourceUsageReport usageReport = 
        report.getApplicationResourceUsageReport();
    Assert.assertEquals(10, usageReport.getMemorySeconds());
    Assert.assertEquals(3, usageReport.getVcoreSeconds());
    Assert.assertEquals(3, usageReport.getGcoreSeconds());
  } finally {
    rmService.close();
  }
}
 

public MyClientRMService(RMContext rmContext, YarnScheduler scheduler,
    RMAppManager rmAppManager,
    ApplicationACLsManager applicationACLsManager,
    QueueACLsManager queueACLsManager,
    RMDelegationTokenSecretManager rmDTSecretManager) {
  super(rmContext, scheduler, rmAppManager, applicationACLsManager,
      queueACLsManager, rmDTSecretManager);
  this.rmContext = rmContext;
}
 

public WebServer(Context nmContext, ResourceView resourceView,
    ApplicationACLsManager aclsManager,
    LocalDirsHandlerService dirsHandler) {
  super(WebServer.class.getName());
  this.nmContext = nmContext;
  this.nmWebApp = new NMWebApp(resourceView, aclsManager, dirsHandler);
}
 

public ClientRMServiceForTest(Configuration conf,
    ResourceScheduler scheduler,
    RMDelegationTokenSecretManager rmDTSecretManager) {
  super(mock(RMContext.class), scheduler, mock(RMAppManager.class),
      new ApplicationACLsManager(conf), new QueueACLsManager(scheduler,
          conf), rmDTSecretManager);
}
 

public MyRMAppManager(RMContext context, YarnScheduler scheduler,
    ApplicationMasterService masterService,
    ApplicationACLsManager applicationACLsManager, Configuration conf) {
  super(context, scheduler, masterService, applicationACLsManager, conf);
  this.conf = conf;
  this.rmContext = context;
}
 

public static ResourceManager mockRm(RMContext rmContext) throws IOException {
  ResourceManager rm = mock(ResourceManager.class);
  ResourceScheduler rs = mockCapacityScheduler();
  ApplicationACLsManager aclMgr = mockAppACLsManager();
  ClientRMService clientRMService = mockClientRMService(rmContext);
  when(rm.getResourceScheduler()).thenReturn(rs);
  when(rm.getRMContext()).thenReturn(rmContext);
  when(rm.getApplicationACLsManager()).thenReturn(aclMgr);
  when(rm.getClientRMService()).thenReturn(clientRMService);
  return rm;
}
 

@Before
public void setup() throws Exception {
  // Only test the ACLs of the generic history
  TimelineACLsManager aclsManager = new TimelineACLsManager(new YarnConfiguration());
  TimelineDataManager dataManager =
      new TimelineDataManager(store, aclsManager);
  ApplicationACLsManager appAclsManager = new ApplicationACLsManager(conf);
  historyManager =
      new ApplicationHistoryManagerOnTimelineStore(dataManager, appAclsManager);
  historyManager.init(conf);
  historyManager.start();
}