org.apache.hadoop.yarn.server.resourcemanager.ResourceManager Java Examples

@Before
public void setUp() throws Exception {
  final RMContext mockRMContext =
      TestRMWebApp.mockRMContext(3, numberOfRacks, numberOfNodesPerRack,
        8 * TestRMWebApp.GiB);
  injector =
      WebAppTests.createMockInjector(RMContext.class, mockRMContext,
        new Module() {
          @Override
          public void configure(Binder binder) {
            try {
              binder.bind(ResourceManager.class).toInstance(
                TestRMWebApp.mockRm(mockRMContext));
            } catch (IOException e) {
              throw new IllegalStateException(e);
            }
          }
        });
}
 

@Override
protected void configureServlets() {
  bind(JAXBContextResolver.class);
  bind(RMWebServices.class);
  bind(GenericExceptionHandler.class);
  Configuration rmconf = new Configuration();
  rmconf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS,
    YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS);
  rmconf.setClass(YarnConfiguration.RM_SCHEDULER, FifoScheduler.class,
    ResourceScheduler.class);
  rmconf.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, true);
  rm = new MockRM(rmconf);
  bind(ResourceManager.class).toInstance(rm);
  if (isKerberosAuth == true) {
    filter("/*").through(TestKerberosAuthFilter.class);
  } else {
    filter("/*").through(TestSimpleAuthFilter.class);
  }
  serve("/*").with(GuiceContainer.class);
}
 

@Override
protected void render(Block html) {
  html._(MetricsOverviewTable.class);
  ResourceManager rm = getInstance(ResourceManager.class);
  ClusterInfo cinfo = new ClusterInfo(rm);
  info("Cluster overview").
    _("Cluster ID:", cinfo.getClusterId()).
    _("ResourceManager state:", cinfo.getState()).
    _("ResourceManager HA state:", cinfo.getHAState()).
    _("ResourceManager RMStateStore:", cinfo.getRMStateStore()).
    _("ResourceManager started on:", Times.format(cinfo.getStartedOn())).
    _("ResourceManager version:", cinfo.getRMBuildVersion() +
        " on " + cinfo.getRMVersionBuiltOn()).
    _("Hadoop version:", cinfo.getHadoopBuildVersion() +
        " on " + cinfo.getHadoopVersionBuiltOn());
  html._(InfoBlock.class);
}
 

public void init(int id, int heartbeatInterval,
    List<ContainerSimulator> containerList, ResourceManager rm, SLSRunner se,
    long traceStartTime, long traceFinishTime, String user, String queue, 
    boolean isTracked, String oldAppId) {
  super.init(id, heartbeatInterval, containerList, rm, se, 
            traceStartTime, traceFinishTime, user, queue,
            isTracked, oldAppId);
  amtype = "mapreduce";
  
  // get map/reduce tasks
  for (ContainerSimulator cs : containerList) {
    if (cs.getType().equals("map")) {
      cs.setPriority(PRIORITY_MAP);
      pendingMaps.add(cs);
    } else if (cs.getType().equals("reduce")) {
      cs.setPriority(PRIORITY_REDUCE);
      pendingReduces.add(cs);
    }
  }
  allMaps.addAll(pendingMaps);
  allReduces.addAll(pendingReduces);
  mapTotal = pendingMaps.size();
  reduceTotal = pendingReduces.size();
  totalContainers = mapTotal + reduceTotal;
}
 

public void init(int id, int heartbeatInterval,
    List<ContainerSimulator> containerList, ResourceManager rm, SLSRunner se,
    long traceStartTime, long traceFinishTime, String user, String queue, 
    boolean isTracked, String oldAppId) {
  super.init(id, heartbeatInterval, containerList, rm, se, 
            traceStartTime, traceFinishTime, user, queue,
            isTracked, oldAppId);
  amtype = "mapreduce";
  
  // get map/reduce tasks
  for (ContainerSimulator cs : containerList) {
    if (cs.getType().equals("map")) {
      cs.setPriority(PRIORITY_MAP);
      pendingMaps.add(cs);
    } else if (cs.getType().equals("reduce")) {
      cs.setPriority(PRIORITY_REDUCE);
      pendingReduces.add(cs);
    }
  }
  allMaps.addAll(pendingMaps);
  allReduces.addAll(pendingReduces);
  mapTotal = pendingMaps.size();
  reduceTotal = pendingReduces.size();
  totalContainers = mapTotal + reduceTotal;
}
 

/**
 * Wait for all the NodeManagers to connect to the ResourceManager.
 *
 * @param timeout Time to wait (sleeps in 100 ms intervals) in milliseconds.
 * @return true if all NodeManagers connect to the (Active)
 * ResourceManager, false otherwise.
 * @throws YarnException
 * @throws InterruptedException
 */
public boolean waitForNodeManagersToConnect(long timeout)
    throws YarnException, InterruptedException {
  GetClusterMetricsRequest req = GetClusterMetricsRequest.newInstance();
  for (int i = 0; i < timeout / 100; i++) {
    ResourceManager rm = getResourceManager();
    if (rm == null) {
      throw new YarnException("Can not find the active RM.");
    }
    else if (nodeManagers.length == rm.getClientRMService()
          .getClusterMetrics(req).getClusterMetrics().getNumNodeManagers()) {
      return true;
    }
    Thread.sleep(100);
  }
  return false;
}
 

/**
 * Wait for all the NodeManagers to connect to the ResourceManager.
 *
 * @param timeout Time to wait (sleeps in 100 ms intervals) in milliseconds.
 * @return true if all NodeManagers connect to the (Active)
 * ResourceManager, false otherwise.
 * @throws YarnException
 * @throws InterruptedException
 */
public boolean waitForNodeManagersToConnect(long timeout)
    throws YarnException, InterruptedException {
  GetClusterMetricsRequest req = GetClusterMetricsRequest.newInstance();
  for (int i = 0; i < timeout / 100; i++) {
    ResourceManager rm = getResourceManager();
    if (rm == null) {
      throw new YarnException("Can not find the active RM.");
    }
    else if (nodeManagers.length == rm.getClientRMService()
          .getClusterMetrics(req).getClusterMetrics().getNumNodeManagers()) {
      return true;
    }
    Thread.sleep(100);
  }
  return false;
}
 

@Override
public Resource getMaximumResourceCapability() {
  Resource maxResource;
  maxAllocReadLock.lock();
  try {
    if (useConfiguredMaximumAllocationOnly) {
      if (System.currentTimeMillis() - ResourceManager.getClusterTimeStamp()
          > configuredMaximumAllocationWaitTime) {
        useConfiguredMaximumAllocationOnly = false;
      }
      maxResource = Resources.clone(configuredMaximumAllocation);
    } else {
      maxResource = Resources.clone(maximumAllocation);
    }
  } finally {
    maxAllocReadLock.unlock();
  }
  return maxResource;
}
 

public void scheduler() {
  // limit applications to those in states relevant to scheduling
  set(YarnWebParams.APP_STATE, StringHelper.cjoin(
      YarnApplicationState.NEW.toString(),
      YarnApplicationState.NEW_SAVING.toString(),
      YarnApplicationState.SUBMITTED.toString(),
      YarnApplicationState.ACCEPTED.toString(),
      YarnApplicationState.RUNNING.toString()));

  ResourceManager rm = getInstance(ResourceManager.class);
  ResourceScheduler rs = rm.getResourceScheduler();
  if (rs == null || rs instanceof CapacityScheduler) {
    setTitle("Capacity Scheduler");
    render(CapacitySchedulerPage.class);
    return;
  }
  
  if (rs instanceof FairScheduler) {
    setTitle("Fair Scheduler");
    render(FairSchedulerPage.class);
    return;
  }
  
  setTitle("Default Scheduler");
  render(DefaultSchedulerPage.class);
}
 

@Before
public void setUp() throws Exception {
  resourceManager = new ResourceManager() {
    @Override
    protected RMNodeLabelsManager createNodeLabelManager() {
      RMNodeLabelsManager mgr = new NullRMNodeLabelsManager();
      mgr.init(getConfig());
      return mgr;
    }
  };
  CapacitySchedulerConfiguration csConf 
     = new CapacitySchedulerConfiguration();
  setupQueueConfiguration(csConf);
  YarnConfiguration conf = new YarnConfiguration(csConf);
  conf.setClass(YarnConfiguration.RM_SCHEDULER, 
      CapacityScheduler.class, ResourceScheduler.class);
  resourceManager.init(conf);
  resourceManager.getRMContext().getContainerTokenSecretManager().rollMasterKey();
  resourceManager.getRMContext().getNMTokenSecretManager().rollMasterKey();
  ((AsyncDispatcher)resourceManager.getRMContext().getDispatcher()).start();
  mockContext = mock(RMContext.class);
  when(mockContext.getConfigurationProvider()).thenReturn(
      new LocalConfigurationProvider());
}
 

@Before
public void setUp() throws IOException {
  scheduler = new FairScheduler();
  
  Configuration conf = new YarnConfiguration();
  conf.setClass(YarnConfiguration.RM_SCHEDULER, FairScheduler.class,
      ResourceScheduler.class);
  conf.set("yarn.scheduler.fair.event-log-enabled", "true");

  // All tests assume only one assignment per node update
  conf.set(FairSchedulerConfiguration.ASSIGN_MULTIPLE, "false");
  resourceManager = new ResourceManager();
  resourceManager.init(conf);
  ((AsyncDispatcher)resourceManager.getRMContext().getDispatcher()).start();
  scheduler.init(conf);
  scheduler.start();
  scheduler.reinitialize(conf, resourceManager.getRMContext());
}
 

@Override
public void setup() {
  bind(JAXBContextResolver.class);
  bind(RMWebServices.class);
  bind(GenericExceptionHandler.class);
  bind(RMWebApp.class).toInstance(this);

  if (rm != null) {
    bind(ResourceManager.class).toInstance(rm);
    bind(ApplicationBaseProtocol.class).toInstance(rm.getClientRMService());
  }
  route("/", RmController.class);
  route(pajoin("/nodes", NODE_STATE), RmController.class, "nodes");
  route(pajoin("/apps", APP_STATE), RmController.class);
  route("/cluster", RmController.class, "about");
  route(pajoin("/app", APPLICATION_ID), RmController.class, "app");
  route("/scheduler", RmController.class, "scheduler");
  route(pajoin("/queue", QUEUE_NAME), RmController.class, "queue");
  route("/nodelabels", RmController.class, "nodelabels");
  route(pajoin("/appattempt", APPLICATION_ATTEMPT_ID), RmController.class,
    "appattempt");
  route(pajoin("/container", CONTAINER_ID), RmController.class, "container");
}
 

@Inject
public FairSchedulerAppsBlock(ResourceManager rm, ViewContext ctx,
    Configuration conf) {
  super(ctx);
  FairScheduler scheduler = (FairScheduler) rm.getResourceScheduler();
  fsinfo = new FairSchedulerInfo(scheduler);
  apps = new ConcurrentHashMap<ApplicationId, RMApp>();
  for (Map.Entry<ApplicationId, RMApp> entry : rm.getRMContext().getRMApps()
      .entrySet()) {
    if (!(RMAppState.NEW.equals(entry.getValue().getState())
        || RMAppState.NEW_SAVING.equals(entry.getValue().getState())
        || RMAppState.SUBMITTED.equals(entry.getValue().getState()))) {
      apps.put(entry.getKey(), entry.getValue());
    }
  }
  this.conf = conf;
  this.rm = rm;
}
 

protected void releaseContainers(List<ContainerId> containers,
    SchedulerApplicationAttempt attempt) {
  for (ContainerId containerId : containers) {
    RMContainer rmContainer = getRMContainer(containerId);
    if (rmContainer == null) {
      if (System.currentTimeMillis() - ResourceManager.getClusterTimeStamp()
          < nmExpireInterval) {
        LOG.info(containerId + " doesn't exist. Add the container"
            + " to the release request cache as it maybe on recovery.");
        synchronized (attempt) {
          attempt.getPendingRelease().add(containerId);
        }
      } else {
        RMAuditLogger.logFailure(attempt.getUser(),
          AuditConstants.RELEASE_CONTAINER,
          "Unauthorized access or invalid container", "Scheduler",
          "Trying to release container not owned by app or with invalid id.",
          attempt.getApplicationId(), containerId);
      }
    }
    completedContainer(rmContainer,
      SchedulerUtils.createAbnormalContainerStatus(containerId,
        SchedulerUtils.RELEASED_CONTAINER), RMContainerEventType.RELEASED);
  }
}
 

@Override
public Resource getMaximumResourceCapability() {
  Resource maxResource;
  maxAllocReadLock.lock();
  try {
    if (useConfiguredMaximumAllocationOnly) {
      if (System.currentTimeMillis() - ResourceManager.getClusterTimeStamp()
          > configuredMaximumAllocationWaitTime) {
        useConfiguredMaximumAllocationOnly = false;
      }
      maxResource = Resources.clone(configuredMaximumAllocation);
    } else {
      maxResource = Resources.clone(maximumAllocation);
    }
  } finally {
    maxAllocReadLock.unlock();
  }
  return maxResource;
}
 

public void scheduler() {
  // limit applications to those in states relevant to scheduling
  set(YarnWebParams.APP_STATE, StringHelper.cjoin(
      YarnApplicationState.NEW.toString(),
      YarnApplicationState.NEW_SAVING.toString(),
      YarnApplicationState.SUBMITTED.toString(),
      YarnApplicationState.ACCEPTED.toString(),
      YarnApplicationState.RUNNING.toString()));

  ResourceManager rm = getInstance(ResourceManager.class);
  ResourceScheduler rs = rm.getResourceScheduler();
  if (rs == null || rs instanceof CapacityScheduler) {
    setTitle("Capacity Scheduler");
    render(CapacitySchedulerPage.class);
    return;
  }
  
  if (rs instanceof FairScheduler) {
    setTitle("Fair Scheduler");
    render(FairSchedulerPage.class);
    return;
  }
  
  setTitle("Default Scheduler");
  render(DefaultSchedulerPage.class);
}
 

@Before
public void setUp() throws Exception {
  final RMContext mockRMContext =
      TestRMWebApp.mockRMContext(3, numberOfRacks, numberOfNodesPerRack,
        8 * TestRMWebApp.GiB);
  injector =
      WebAppTests.createMockInjector(RMContext.class, mockRMContext,
        new Module() {
          @Override
          public void configure(Binder binder) {
            try {
              binder.bind(ResourceManager.class).toInstance(
                TestRMWebApp.mockRm(mockRMContext));
            } catch (IOException e) {
              throw new IllegalStateException(e);
            }
          }
        });
}
 

@Override
public void setup() {
  bind(JAXBContextResolver.class);
  bind(RMWebServices.class);
  bind(GenericExceptionHandler.class);
  bind(RMWebApp.class).toInstance(this);

  if (rm != null) {
    bind(ResourceManager.class).toInstance(rm);
    bind(ApplicationBaseProtocol.class).toInstance(rm.getClientRMService());
  }
  route("/", RmController.class);
  route(pajoin("/nodes", NODE_STATE), RmController.class, "nodes");
  route(pajoin("/apps", APP_STATE), RmController.class);
  route("/cluster", RmController.class, "about");
  route(pajoin("/app", APPLICATION_ID), RmController.class, "app");
  route("/scheduler", RmController.class, "scheduler");
  route(pajoin("/queue", QUEUE_NAME), RmController.class, "queue");
  route("/nodelabels", RmController.class, "nodelabels");
  route(pajoin("/appattempt", APPLICATION_ATTEMPT_ID), RmController.class,
    "appattempt");
  route(pajoin("/container", CONTAINER_ID), RmController.class, "container");
}
 

@Before
public void setUp() throws IOException {
  scheduler = new FairScheduler();
  conf = createConfiguration();
  resourceManager = new ResourceManager();
  resourceManager.init(conf);

  // TODO: This test should really be using MockRM. For now starting stuff
  // that is needed at a bare minimum.
  ((AsyncDispatcher)resourceManager.getRMContext().getDispatcher()).start();
  resourceManager.getRMContext().getStateStore().start();

  // to initialize the master key
  resourceManager.getRMContext().getContainerTokenSecretManager().rollMasterKey();

  scheduler.setRMContext(resourceManager.getRMContext());
}
 

public static ResourceManager mockRm(RMContext rmContext) throws IOException {
  ResourceManager rm = mock(ResourceManager.class);
  ResourceScheduler rs = mockCapacityScheduler();
  ApplicationACLsManager aclMgr = mockAppACLsManager();
  ClientRMService clientRMService = mockClientRMService(rmContext);
  when(rm.getResourceScheduler()).thenReturn(rs);
  when(rm.getRMContext()).thenReturn(rmContext);
  when(rm.getApplicationACLsManager()).thenReturn(aclMgr);
  when(rm.getClientRMService()).thenReturn(clientRMService);
  return rm;
}
 

protected ResourceManager createResourceManager() {
  return new ResourceManager(){
    @Override
    protected void doSecureLogin() throws IOException {
      // Don't try to login using keytab in the testcases.
    }
  };
}
 

@Override
protected void configureServlets() {
  bind(JAXBContextResolver.class);
  bind(RMWebServices.class);
  bind(GenericExceptionHandler.class);
  Configuration conf = new Configuration();
  conf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS,
      YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS);
  conf.setClass(YarnConfiguration.RM_SCHEDULER, FifoScheduler.class,
      ResourceScheduler.class);
  rm = new MockRM(conf);
  bind(ResourceManager.class).toInstance(rm);
  serve("/*").with(GuiceContainer.class);
}
 

private void checkTaskContainersHost(ApplicationAttemptId attemptId,
    ContainerId containerId, ResourceManager rm, String host) {
  YarnScheduler scheduler = rm.getRMContext().getScheduler();
  SchedulerAppReport appReport = scheduler.getSchedulerAppInfo(attemptId);

  Assert.assertTrue(appReport.getLiveContainers().size() > 0);
  for (RMContainer c : appReport.getLiveContainers()) {
    if (c.getContainerId().equals(containerId)) {
      Assert.assertEquals(host, c.getAllocatedNode().getHost());
    }
  }
}
 

public static ResourceManager mockFifoRm(int apps, int racks, int nodes,
                                     int mbsPerNode)
throws Exception {
  ResourceManager rm = mock(ResourceManager.class);
  RMContext rmContext = mockRMContext(apps, racks, nodes,
      mbsPerNode);
  ResourceScheduler rs = mockFifoScheduler(rmContext);
  when(rm.getResourceScheduler()).thenReturn(rs);
  when(rm.getRMContext()).thenReturn(rmContext);
  return rm;
}
 

public static ResourceManager mockRm(RMContext rmContext) throws IOException {
  ResourceManager rm = mock(ResourceManager.class);
  ResourceScheduler rs = mockCapacityScheduler();
  ApplicationACLsManager aclMgr = mockAppACLsManager();
  ClientRMService clientRMService = mockClientRMService(rmContext);
  when(rm.getResourceScheduler()).thenReturn(rs);
  when(rm.getRMContext()).thenReturn(rmContext);
  when(rm.getApplicationACLsManager()).thenReturn(aclMgr);
  when(rm.getClientRMService()).thenReturn(clientRMService);
  return rm;
}
 

@Test
public void testFairSchedulerWebAppPage() {
  List<RMAppState> appStates = Arrays.asList(RMAppState.NEW,
      RMAppState.NEW_SAVING, RMAppState.SUBMITTED);
  final RMContext rmContext = mockRMContext(appStates);
  Injector injector = WebAppTests.createMockInjector(RMContext.class,
      rmContext,
      new Module() {
        @Override
        public void configure(Binder binder) {
          try {
            ResourceManager mockRmWithFairScheduler =
                mockRm(rmContext);
            binder.bind(ResourceManager.class).toInstance
                (mockRmWithFairScheduler);
            binder.bind(ApplicationBaseProtocol.class).toInstance(
              mockRmWithFairScheduler.getClientRMService());
          } catch (IOException e) {
            throw new IllegalStateException(e);
          }
        }
      });
  FairSchedulerPage fsViewInstance = injector.getInstance(FairSchedulerPage
      .class);
  fsViewInstance.render();
  WebAppTests.flushOutput(injector);
}
 

@Test(timeout = 10000)
public void testRMStarts() {
  Configuration conf = new YarnConfiguration();
  conf.setBoolean(YarnConfiguration.RM_SCHEDULER_ENABLE_MONITORS, true);
  conf.set(YarnConfiguration.RM_SCHEDULER_MONITOR_POLICIES,
      ProportionalCapacityPreemptionPolicy.class.getCanonicalName());

  ResourceManager rm = new ResourceManager();
  try {
    rm.init(conf);
  } catch (Exception e) {
    fail("ResourceManager does not start when " +
        YarnConfiguration.RM_SCHEDULER_ENABLE_MONITORS + " is set to true");
  }
}
 

@Override
protected void configureServlets() {
  bind(JAXBContextResolver.class);
  bind(RMWebServices.class);
  bind(GenericExceptionHandler.class);
  Configuration conf = new Configuration();
  conf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS,
      YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS);
  conf.setClass(YarnConfiguration.RM_SCHEDULER, FifoScheduler.class,
      ResourceScheduler.class);
  rm = new MockRM(conf);
  bind(ResourceManager.class).toInstance(rm);
  serve("/*").with(GuiceContainer.class);
}
 

@Override
protected void configureServlets() {
  bind(JAXBContextResolver.class);
  bind(RMWebServices.class);
  bind(GenericExceptionHandler.class);
  rm = new MockRM(new Configuration());
  rm.getRMContext().getContainerTokenSecretManager().rollMasterKey();
  rm.getRMContext().getNMTokenSecretManager().rollMasterKey();
  bind(ResourceManager.class).toInstance(rm);
  serve("/*").with(GuiceContainer.class);
}
 

private static ResourceManager mockRmWithApps(RMContext rmContext) throws
    IOException {
  ResourceManager rm = mock(ResourceManager.class);
  ResourceScheduler rs =  mockFairSchedulerWithoutApps(rmContext);
  ClientRMService clientRMService = mockClientRMService(rmContext);
  when(rm.getResourceScheduler()).thenReturn(rs);
  when(rm.getRMContext()).thenReturn(rmContext);
  when(rm.getClientRMService()).thenReturn(clientRMService);
  return rm;
}