org.apache.hadoop.yarn.server.resourcemanager.RMContextImpl Java Examples

The following examples show how to use org.apache.hadoop.yarn.server.resourcemanager.RMContextImpl. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: TestQueueParsing.java    From hadoop with Apache License 2.0 6 votes vote down vote up 
/**
 * Test init a queue configuration, children's capacity for a given label
 * doesn't equals to 100%. This expect IllegalArgumentException thrown.
 */
@Test(expected = IllegalArgumentException.class)
public void testQueueParsingFailWhenSumOfChildrenNonLabeledCapacityNot100Percent()
    throws IOException {
  nodeLabelManager.addToCluserNodeLabels(ImmutableSet
      .of("red", "blue"));
  
  YarnConfiguration conf = new YarnConfiguration();
  CapacitySchedulerConfiguration csConf =
      new CapacitySchedulerConfiguration(conf);
  setupQueueConfiguration(csConf);
  csConf.setCapacity(CapacitySchedulerConfiguration.ROOT + ".c.c2", 5);

  CapacityScheduler capacityScheduler = new CapacityScheduler();
  RMContextImpl rmContext =
      new RMContextImpl(null, null, null, null, null, null,
          new RMContainerTokenSecretManager(csConf),
          new NMTokenSecretManagerInRM(csConf),
          new ClientToAMTokenSecretManagerInRM(), null);
  rmContext.setNodeLabelManager(nodeLabelManager);
  capacityScheduler.setConf(csConf);
  capacityScheduler.setRMContext(rmContext);
  capacityScheduler.init(csConf);
  capacityScheduler.start();
  ServiceOperations.stopQuietly(capacityScheduler);
}
Example #2
Source File: TestCapacityScheduler.java    From hadoop with Apache License 2.0 6 votes vote down vote up 
/** Test that parseQueue throws an exception when two leaf queues have the
  *  same name
* @throws IOException
  */
 @Test(expected=IOException.class)
 public void testParseQueue() throws IOException {
   CapacityScheduler cs = new CapacityScheduler();
   cs.setConf(new YarnConfiguration());
   cs.setRMContext(resourceManager.getRMContext());
   CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration();
   setupQueueConfiguration(conf);
   cs.init(conf);
   cs.start();

   conf.setQueues(CapacitySchedulerConfiguration.ROOT + ".a.a1", new String[] {"b1"} );
   conf.setCapacity(CapacitySchedulerConfiguration.ROOT + ".a.a1.b1", 100.0f);
   conf.setUserLimitFactor(CapacitySchedulerConfiguration.ROOT + ".a.a1.b1", 100.0f);

   cs.reinitialize(conf, new RMContextImpl(null, null, null, null, null,
     null, new RMContainerTokenSecretManager(conf),
     new NMTokenSecretManagerInRM(conf),
     new ClientToAMTokenSecretManagerInRM(), null));
 }
Example #3
Source File: TestCapacityScheduler.java    From hadoop with Apache License 2.0 6 votes vote down vote up 
@Test
public void testRefreshQueues() throws Exception {
  CapacityScheduler cs = new CapacityScheduler();
  CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration();
  RMContextImpl rmContext =  new RMContextImpl(null, null, null, null, null,
      null, new RMContainerTokenSecretManager(conf),
      new NMTokenSecretManagerInRM(conf),
      new ClientToAMTokenSecretManagerInRM(), null);
  setupQueueConfiguration(conf);
  cs.setConf(new YarnConfiguration());
  cs.setRMContext(resourceManager.getRMContext());
  cs.init(conf);
  cs.start();
  cs.reinitialize(conf, rmContext);
  checkQueueCapacities(cs, A_CAPACITY, B_CAPACITY);

  conf.setCapacity(A, 80f);
  conf.setCapacity(B, 20f);
  cs.reinitialize(conf, mockContext);
  checkQueueCapacities(cs, 80f, 20f);
  cs.stop();
}
Example #4
Source File: TestQueueParsing.java    From hadoop with Apache License 2.0 6 votes vote down vote up 
@Test
public void testQueueParsingWithLabels() throws IOException {
  nodeLabelManager.addToCluserNodeLabels(ImmutableSet.of("red", "blue"));
  
  YarnConfiguration conf = new YarnConfiguration();
  CapacitySchedulerConfiguration csConf =
      new CapacitySchedulerConfiguration(conf);
  setupQueueConfigurationWithLabels(csConf);

  CapacityScheduler capacityScheduler = new CapacityScheduler();
  RMContextImpl rmContext =
      new RMContextImpl(null, null, null, null, null, null,
          new RMContainerTokenSecretManager(csConf),
          new NMTokenSecretManagerInRM(csConf),
          new ClientToAMTokenSecretManagerInRM(), null);
  rmContext.setNodeLabelManager(nodeLabelManager);
  capacityScheduler.setConf(csConf);
  capacityScheduler.setRMContext(rmContext);
  capacityScheduler.init(csConf);
  capacityScheduler.start();
  checkQueueLabels(capacityScheduler);
  ServiceOperations.stopQuietly(capacityScheduler);
}
Example #5
Source File: TestCapacityScheduler.java    From big-c with Apache License 2.0 6 votes vote down vote up 
@Test
public void testRefreshQueues() throws Exception {
  CapacityScheduler cs = new CapacityScheduler();
  CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration();
  RMContextImpl rmContext =  new RMContextImpl(null, null, null, null, null,
      null, new RMContainerTokenSecretManager(conf),
      new NMTokenSecretManagerInRM(conf),
      new ClientToAMTokenSecretManagerInRM(), null);
  setupQueueConfiguration(conf);
  cs.setConf(new YarnConfiguration());
  cs.setRMContext(resourceManager.getRMContext());
  cs.init(conf);
  cs.start();
  cs.reinitialize(conf, rmContext);
  checkQueueCapacities(cs, A_CAPACITY, B_CAPACITY);

  conf.setCapacity(A, 80f);
  conf.setCapacity(B, 20f);
  cs.reinitialize(conf, mockContext);
  checkQueueCapacities(cs, 80f, 20f);
  cs.stop();
}
Example #6
Source File: TestCapacityScheduler.java    From big-c with Apache License 2.0 6 votes vote down vote up 
/** Test that parseQueue throws an exception when two leaf queues have the
  *  same name
* @throws IOException
  */
 @Test(expected=IOException.class)
 public void testParseQueue() throws IOException {
   CapacityScheduler cs = new CapacityScheduler();
   cs.setConf(new YarnConfiguration());
   cs.setRMContext(resourceManager.getRMContext());
   CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration();
   setupQueueConfiguration(conf);
   cs.init(conf);
   cs.start();

   conf.setQueues(CapacitySchedulerConfiguration.ROOT + ".a.a1", new String[] {"b1"} );
   conf.setCapacity(CapacitySchedulerConfiguration.ROOT + ".a.a1.b1", 100.0f);
   conf.setUserLimitFactor(CapacitySchedulerConfiguration.ROOT + ".a.a1.b1", 100.0f);

   cs.reinitialize(conf, new RMContextImpl(null, null, null, null, null,
     null, new RMContainerTokenSecretManager(conf),
     new NMTokenSecretManagerInRM(conf),
     new ClientToAMTokenSecretManagerInRM(), null));
 }
Example #7
Source File: TestQueueParsing.java    From hadoop with Apache License 2.0 6 votes vote down vote up 
@Test
public void testQueueParsingWithLabelsInherit() throws IOException {
  nodeLabelManager.addToCluserNodeLabels(ImmutableSet.of("red", "blue"));

  YarnConfiguration conf = new YarnConfiguration();
  CapacitySchedulerConfiguration csConf =
      new CapacitySchedulerConfiguration(conf);
  setupQueueConfigurationWithLabelsInherit(csConf);

  CapacityScheduler capacityScheduler = new CapacityScheduler();
  RMContextImpl rmContext =
      new RMContextImpl(null, null, null, null, null, null,
          new RMContainerTokenSecretManager(csConf),
          new NMTokenSecretManagerInRM(csConf),
          new ClientToAMTokenSecretManagerInRM(), null);
  rmContext.setNodeLabelManager(nodeLabelManager);
  capacityScheduler.setConf(csConf);
  capacityScheduler.setRMContext(rmContext);
  capacityScheduler.init(csConf);
  capacityScheduler.start();
  checkQueueLabelsInheritConfig(capacityScheduler);
  ServiceOperations.stopQuietly(capacityScheduler);
}
Example #8
Source File: TestQueueParsing.java    From big-c with Apache License 2.0 6 votes vote down vote up 
@Test
public void testQueueParsingWithLabels() throws IOException {
  nodeLabelManager.addToCluserNodeLabels(ImmutableSet.of("red", "blue"));
  
  YarnConfiguration conf = new YarnConfiguration();
  CapacitySchedulerConfiguration csConf =
      new CapacitySchedulerConfiguration(conf);
  setupQueueConfigurationWithLabels(csConf);

  CapacityScheduler capacityScheduler = new CapacityScheduler();
  RMContextImpl rmContext =
      new RMContextImpl(null, null, null, null, null, null,
          new RMContainerTokenSecretManager(csConf),
          new NMTokenSecretManagerInRM(csConf),
          new ClientToAMTokenSecretManagerInRM(), null);
  rmContext.setNodeLabelManager(nodeLabelManager);
  capacityScheduler.setConf(csConf);
  capacityScheduler.setRMContext(rmContext);
  capacityScheduler.init(csConf);
  capacityScheduler.start();
  checkQueueLabels(capacityScheduler);
  ServiceOperations.stopQuietly(capacityScheduler);
}
Example #9
Source File: TestQueueParsing.java    From big-c with Apache License 2.0 6 votes vote down vote up 
@Test
public void testQueueParsingWithLabelsInherit() throws IOException {
  nodeLabelManager.addToCluserNodeLabels(ImmutableSet.of("red", "blue"));

  YarnConfiguration conf = new YarnConfiguration();
  CapacitySchedulerConfiguration csConf =
      new CapacitySchedulerConfiguration(conf);
  setupQueueConfigurationWithLabelsInherit(csConf);

  CapacityScheduler capacityScheduler = new CapacityScheduler();
  RMContextImpl rmContext =
      new RMContextImpl(null, null, null, null, null, null,
          new RMContainerTokenSecretManager(csConf),
          new NMTokenSecretManagerInRM(csConf),
          new ClientToAMTokenSecretManagerInRM(), null);
  rmContext.setNodeLabelManager(nodeLabelManager);
  capacityScheduler.setConf(csConf);
  capacityScheduler.setRMContext(rmContext);
  capacityScheduler.init(csConf);
  capacityScheduler.start();
  checkQueueLabelsInheritConfig(capacityScheduler);
  ServiceOperations.stopQuietly(capacityScheduler);
}
Example #10
Source File: TestRMWebApp.java    From hadoop with Apache License 2.0 6 votes vote down vote up 
public static CapacityScheduler mockCapacityScheduler() throws IOException {
  // stolen from TestCapacityScheduler
  CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration();
  setupQueueConfiguration(conf);

  CapacityScheduler cs = new CapacityScheduler();
  cs.setConf(new YarnConfiguration());
  RMContext rmContext = new RMContextImpl(null, null, null, null, null,
      null, new RMContainerTokenSecretManager(conf),
      new NMTokenSecretManagerInRM(conf),
      new ClientToAMTokenSecretManagerInRM(), null);
  rmContext.setNodeLabelManager(new NullRMNodeLabelsManager());
  cs.setRMContext(rmContext);
  cs.init(conf);
  return cs;
}
Example #11
Source File: TestQueueParsing.java    From big-c with Apache License 2.0 6 votes vote down vote up 
@Test
public void testQueueParsingWhenLabelsNotExist() throws IOException {
  YarnConfiguration conf = new YarnConfiguration();
  CapacitySchedulerConfiguration csConf =
      new CapacitySchedulerConfiguration(conf);
  setupQueueConfigurationWithLabels(csConf);

  CapacityScheduler capacityScheduler = new CapacityScheduler();
  RMContextImpl rmContext =
      new RMContextImpl(null, null, null, null, null, null,
          new RMContainerTokenSecretManager(csConf),
          new NMTokenSecretManagerInRM(csConf),
          new ClientToAMTokenSecretManagerInRM(), null);
  
  RMNodeLabelsManager nodeLabelsManager = new NullRMNodeLabelsManager();
  nodeLabelsManager.init(conf);
  nodeLabelsManager.start();
  
  rmContext.setNodeLabelManager(nodeLabelsManager);
  capacityScheduler.setConf(csConf);
  capacityScheduler.setRMContext(rmContext);
  capacityScheduler.init(csConf);
  capacityScheduler.start();
  ServiceOperations.stopQuietly(capacityScheduler);
  ServiceOperations.stopQuietly(nodeLabelsManager);
}
Example #12
Source File: TestQueueParsing.java    From big-c with Apache License 2.0 6 votes vote down vote up 
/**
 * Test init a queue configuration, children's capacity for a given label
 * doesn't equals to 100%. This expect IllegalArgumentException thrown.
 */
@Test(expected = IllegalArgumentException.class)
public void testQueueParsingFailWhenSumOfChildrenNonLabeledCapacityNot100Percent()
    throws IOException {
  nodeLabelManager.addToCluserNodeLabels(ImmutableSet
      .of("red", "blue"));
  
  YarnConfiguration conf = new YarnConfiguration();
  CapacitySchedulerConfiguration csConf =
      new CapacitySchedulerConfiguration(conf);
  setupQueueConfiguration(csConf);
  csConf.setCapacity(CapacitySchedulerConfiguration.ROOT + ".c.c2", 5);

  CapacityScheduler capacityScheduler = new CapacityScheduler();
  RMContextImpl rmContext =
      new RMContextImpl(null, null, null, null, null, null,
          new RMContainerTokenSecretManager(csConf),
          new NMTokenSecretManagerInRM(csConf),
          new ClientToAMTokenSecretManagerInRM(), null);
  rmContext.setNodeLabelManager(nodeLabelManager);
  capacityScheduler.setConf(csConf);
  capacityScheduler.setRMContext(rmContext);
  capacityScheduler.init(csConf);
  capacityScheduler.start();
  ServiceOperations.stopQuietly(capacityScheduler);
}
Example #13
Source File: TestQueueParsing.java    From hadoop with Apache License 2.0 6 votes vote down vote up 
@Test
public void testQueueParsingWhenLabelsNotExist() throws IOException {
  YarnConfiguration conf = new YarnConfiguration();
  CapacitySchedulerConfiguration csConf =
      new CapacitySchedulerConfiguration(conf);
  setupQueueConfigurationWithLabels(csConf);

  CapacityScheduler capacityScheduler = new CapacityScheduler();
  RMContextImpl rmContext =
      new RMContextImpl(null, null, null, null, null, null,
          new RMContainerTokenSecretManager(csConf),
          new NMTokenSecretManagerInRM(csConf),
          new ClientToAMTokenSecretManagerInRM(), null);
  
  RMNodeLabelsManager nodeLabelsManager = new NullRMNodeLabelsManager();
  nodeLabelsManager.init(conf);
  nodeLabelsManager.start();
  
  rmContext.setNodeLabelManager(nodeLabelsManager);
  capacityScheduler.setConf(csConf);
  capacityScheduler.setRMContext(rmContext);
  capacityScheduler.init(csConf);
  capacityScheduler.start();
  ServiceOperations.stopQuietly(capacityScheduler);
  ServiceOperations.stopQuietly(nodeLabelsManager);
}
Example #14
Source File: TestRMWebApp.java    From big-c with Apache License 2.0 6 votes vote down vote up 
public static CapacityScheduler mockCapacityScheduler() throws IOException {
  // stolen from TestCapacityScheduler
  CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration();
  setupQueueConfiguration(conf);

  CapacityScheduler cs = new CapacityScheduler();
  cs.setConf(new YarnConfiguration());
  RMContext rmContext = new RMContextImpl(null, null, null, null, null,
      null, new RMContainerTokenSecretManager(conf),
      new NMTokenSecretManagerInRM(conf),
      new ClientToAMTokenSecretManagerInRM(), null);
  rmContext.setNodeLabelManager(new NullRMNodeLabelsManager());
  cs.setRMContext(rmContext);
  cs.init(conf);
  return cs;
}
Example #15
Source File: TestRMWebAppFairScheduler.java    From big-c with Apache License 2.0 5 votes vote down vote up 
private static FairScheduler mockFairScheduler() throws IOException {
  FairScheduler fs = new FairScheduler();
  FairSchedulerConfiguration conf = new FairSchedulerConfiguration();
  fs.setRMContext(new RMContextImpl(null, null, null, null, null,
      null, new RMContainerTokenSecretManager(conf),
      new NMTokenSecretManagerInRM(conf),
      new ClientToAMTokenSecretManagerInRM(), null));
  fs.init(conf);
  return fs;
}
Example #16
Source File: TestRMNMRPCResponseId.java    From big-c with Apache License 2.0 5 votes vote down vote up 
@Before
public void setUp() {
  Configuration conf = new Configuration();
  // Dispatcher that processes events inline
  Dispatcher dispatcher = new InlineDispatcher();
  dispatcher.register(SchedulerEventType.class, new EventHandler<Event>() {
    @Override
    public void handle(Event event) {
      ; // ignore
    }
  });
  RMContext context =
      new RMContextImpl(dispatcher, null, null, null, null,
        null, new RMContainerTokenSecretManager(conf),
        new NMTokenSecretManagerInRM(conf), null, null);
  dispatcher.register(RMNodeEventType.class,
      new ResourceManager.NodeEventDispatcher(context));
  NodesListManager nodesListManager = new NodesListManager(context);
  nodesListManager.init(conf);
  
  context.getContainerTokenSecretManager().rollMasterKey();
  context.getNMTokenSecretManager().rollMasterKey();
  resourceTrackerService = new ResourceTrackerService(context,
      nodesListManager, new NMLivelinessMonitor(dispatcher),
      context.getContainerTokenSecretManager(),
      context.getNMTokenSecretManager());
  resourceTrackerService.init(conf);
}
Example #17
Source File: TestNMReconnect.java    From big-c with Apache License 2.0 5 votes vote down vote up 
@Before
public void setUp() {
  Configuration conf = new Configuration();
  // Dispatcher that processes events inline
  Dispatcher dispatcher = new InlineDispatcher();

  dispatcher.register(RMNodeEventType.class,
      new TestRMNodeEventDispatcher());

  RMContext context = new RMContextImpl(dispatcher, null,
      null, null, null, null, null, null, null, null);
  dispatcher.register(SchedulerEventType.class,
      new InlineDispatcher.EmptyEventHandler());
  dispatcher.register(RMNodeEventType.class,
      new NodeEventDispatcher(context));
  NMLivelinessMonitor nmLivelinessMonitor = new NMLivelinessMonitor(
      dispatcher);
  nmLivelinessMonitor.init(conf);
  nmLivelinessMonitor.start();
  NodesListManager nodesListManager = new NodesListManager(context);
  nodesListManager.init(conf);
  RMContainerTokenSecretManager containerTokenSecretManager =
      new RMContainerTokenSecretManager(conf);
  containerTokenSecretManager.start();
  NMTokenSecretManagerInRM nmTokenSecretManager =
      new NMTokenSecretManagerInRM(conf);
  nmTokenSecretManager.start();
  resourceTrackerService = new ResourceTrackerService(context,
      nodesListManager, nmLivelinessMonitor, containerTokenSecretManager,
      nmTokenSecretManager);
  
  resourceTrackerService.init(conf);
  resourceTrackerService.start();
}
Example #18
Source File: TestNMExpiry.java    From big-c with Apache License 2.0 5 votes vote down vote up 
@Before
public void setUp() {
  Configuration conf = new Configuration();
  // Dispatcher that processes events inline
  Dispatcher dispatcher = new InlineDispatcher();
  RMContext context = new RMContextImpl(dispatcher, null,
      null, null, null, null, null, null, null, null);
  dispatcher.register(SchedulerEventType.class,
      new InlineDispatcher.EmptyEventHandler());
  dispatcher.register(RMNodeEventType.class,
      new NodeEventDispatcher(context));
  NMLivelinessMonitor nmLivelinessMonitor = new TestNmLivelinessMonitor(
      dispatcher);
  nmLivelinessMonitor.init(conf);
  nmLivelinessMonitor.start();
  NodesListManager nodesListManager = new NodesListManager(context);
  nodesListManager.init(conf);
  RMContainerTokenSecretManager containerTokenSecretManager =
      new RMContainerTokenSecretManager(conf);
  containerTokenSecretManager.start();
  NMTokenSecretManagerInRM nmTokenSecretManager =
      new NMTokenSecretManagerInRM(conf);
  nmTokenSecretManager.start();
  resourceTrackerService = new ResourceTrackerService(context,
      nodesListManager, nmLivelinessMonitor, containerTokenSecretManager,
      nmTokenSecretManager);
  
  resourceTrackerService.init(conf);
  resourceTrackerService.start();
}
Example #19
Source File: TestQueueParsing.java    From hadoop with Apache License 2.0 5 votes vote down vote up 
/**
 * Test init a queue configuration, children's capacity for a given label
 * doesn't equals to 100%. This expect IllegalArgumentException thrown.
 */
@Test(expected = IllegalArgumentException.class)
public void testQueueParsingFailWhenSumOfChildrenLabeledCapacityNot100Percent()
    throws IOException {
  nodeLabelManager.addToCluserNodeLabels(ImmutableSet
      .of("red", "blue"));
  
  YarnConfiguration conf = new YarnConfiguration();
  CapacitySchedulerConfiguration csConf =
      new CapacitySchedulerConfiguration(conf);
  setupQueueConfigurationWithLabels(csConf);
  csConf.setCapacityByLabel(CapacitySchedulerConfiguration.ROOT + ".b.b3",
      "red", 24);

  CapacityScheduler capacityScheduler = new CapacityScheduler();
  RMContextImpl rmContext =
      new RMContextImpl(null, null, null, null, null, null,
          new RMContainerTokenSecretManager(csConf),
          new NMTokenSecretManagerInRM(csConf),
          new ClientToAMTokenSecretManagerInRM(), null);
  rmContext.setNodeLabelManager(nodeLabelManager);
  capacityScheduler.setConf(csConf);
  capacityScheduler.setRMContext(rmContext);
  capacityScheduler.init(csConf);
  capacityScheduler.start();
  ServiceOperations.stopQuietly(capacityScheduler);
}
Example #20
Source File: TestQueueParsing.java    From big-c with Apache License 2.0 5 votes vote down vote up 
@Test
public void testQueueParsingWhenLabelsInheritedNotExistedInNodeLabelManager()
    throws IOException {
  YarnConfiguration conf = new YarnConfiguration();
  CapacitySchedulerConfiguration csConf =
      new CapacitySchedulerConfiguration(conf);
  setupQueueConfigurationWithLabelsInherit(csConf);

  CapacityScheduler capacityScheduler = new CapacityScheduler();
  RMContextImpl rmContext =
      new RMContextImpl(null, null, null, null, null, null,
          new RMContainerTokenSecretManager(csConf),
          new NMTokenSecretManagerInRM(csConf),
          new ClientToAMTokenSecretManagerInRM(), null);
  
  RMNodeLabelsManager nodeLabelsManager = new NullRMNodeLabelsManager();
  nodeLabelsManager.init(conf);
  nodeLabelsManager.start();
  
  rmContext.setNodeLabelManager(nodeLabelsManager);
  capacityScheduler.setConf(csConf);
  capacityScheduler.setRMContext(rmContext);
  capacityScheduler.init(csConf);
  capacityScheduler.start();
  ServiceOperations.stopQuietly(capacityScheduler);
  ServiceOperations.stopQuietly(nodeLabelsManager);
}
Example #21
Source File: TestQueueParsing.java    From big-c with Apache License 2.0 5 votes vote down vote up 
@Test
public void testSingleLevelQueueParsingWhenLabelsNotExistedInNodeLabelManager()
    throws IOException {
  YarnConfiguration conf = new YarnConfiguration();
  CapacitySchedulerConfiguration csConf =
      new CapacitySchedulerConfiguration(conf);
  setupQueueConfigurationWithSingleLevel(csConf);

  CapacityScheduler capacityScheduler = new CapacityScheduler();
  RMContextImpl rmContext =
      new RMContextImpl(null, null, null, null, null, null,
          new RMContainerTokenSecretManager(csConf),
          new NMTokenSecretManagerInRM(csConf),
          new ClientToAMTokenSecretManagerInRM(), null);
  
  RMNodeLabelsManager nodeLabelsManager = new NullRMNodeLabelsManager();
  nodeLabelsManager.init(conf);
  nodeLabelsManager.start();
  
  rmContext.setNodeLabelManager(nodeLabelsManager);
  capacityScheduler.setConf(csConf);
  capacityScheduler.setRMContext(rmContext);
  capacityScheduler.init(csConf);
  capacityScheduler.start();
  ServiceOperations.stopQuietly(capacityScheduler);
  ServiceOperations.stopQuietly(nodeLabelsManager);
}
Example #22
Source File: TestQueueParsing.java    From big-c with Apache License 2.0 5 votes vote down vote up 
/**
 * Test init a queue configuration, children's capacity for a given label
 * doesn't equals to 100%. This expect IllegalArgumentException thrown.
 */
@Test(expected = IllegalArgumentException.class)
public void testQueueParsingFailWhenSumOfChildrenLabeledCapacityNot100Percent()
    throws IOException {
  nodeLabelManager.addToCluserNodeLabels(ImmutableSet
      .of("red", "blue"));
  
  YarnConfiguration conf = new YarnConfiguration();
  CapacitySchedulerConfiguration csConf =
      new CapacitySchedulerConfiguration(conf);
  setupQueueConfigurationWithLabels(csConf);
  csConf.setCapacityByLabel(CapacitySchedulerConfiguration.ROOT + ".b.b3",
      "red", 24);

  CapacityScheduler capacityScheduler = new CapacityScheduler();
  RMContextImpl rmContext =
      new RMContextImpl(null, null, null, null, null, null,
          new RMContainerTokenSecretManager(csConf),
          new NMTokenSecretManagerInRM(csConf),
          new ClientToAMTokenSecretManagerInRM(), null);
  rmContext.setNodeLabelManager(nodeLabelManager);
  capacityScheduler.setConf(csConf);
  capacityScheduler.setRMContext(rmContext);
  capacityScheduler.init(csConf);
  capacityScheduler.start();
  ServiceOperations.stopQuietly(capacityScheduler);
}
Example #23
Source File: TestQueueParsing.java    From big-c with Apache License 2.0 5 votes vote down vote up 
/**
 * Test init a queue configuration, children's capacity for a given label
 * doesn't equals to 100%. This expect IllegalArgumentException thrown.
 */
@Test(expected = IllegalArgumentException.class)
public void testQueueParsingWithSumOfChildLabelCapacityNot100PercentWithWildCard()
    throws IOException {
  nodeLabelManager.addToCluserNodeLabels(ImmutableSet
      .of("red", "blue"));
  
  YarnConfiguration conf = new YarnConfiguration();
  CapacitySchedulerConfiguration csConf =
      new CapacitySchedulerConfiguration(conf);
  setupQueueConfigurationWithLabels(csConf);
  csConf.setCapacityByLabel(CapacitySchedulerConfiguration.ROOT + ".b.b3",
      "red", 24);
  csConf.setAccessibleNodeLabels(CapacitySchedulerConfiguration.ROOT,
      ImmutableSet.of(RMNodeLabelsManager.ANY));
  csConf.setAccessibleNodeLabels(CapacitySchedulerConfiguration.ROOT + ".b",
      ImmutableSet.of(RMNodeLabelsManager.ANY));

  CapacityScheduler capacityScheduler = new CapacityScheduler();
  RMContextImpl rmContext =
      new RMContextImpl(null, null, null, null, null, null,
          new RMContainerTokenSecretManager(csConf),
          new NMTokenSecretManagerInRM(csConf),
          new ClientToAMTokenSecretManagerInRM(), null);
  rmContext.setNodeLabelManager(nodeLabelManager);
  capacityScheduler.setConf(csConf);
  capacityScheduler.setRMContext(rmContext);
  capacityScheduler.init(csConf);
  capacityScheduler.start();
  ServiceOperations.stopQuietly(capacityScheduler);
}
Example #24
Source File: TestQueueParsing.java    From big-c with Apache License 2.0 5 votes vote down vote up 
@Test(expected = IOException.class)
public void testQueueParsingWithMoveQueue()
    throws IOException {
  YarnConfiguration conf = new YarnConfiguration();
  CapacitySchedulerConfiguration csConf =
      new CapacitySchedulerConfiguration(conf);
  csConf.setQueues("root", new String[] { "a" });
  csConf.setQueues("root.a", new String[] { "x", "y" });
  csConf.setCapacity("root.a", 100);
  csConf.setCapacity("root.a.x", 50);
  csConf.setCapacity("root.a.y", 50);

  CapacityScheduler capacityScheduler = new CapacityScheduler();
  RMContextImpl rmContext =
      new RMContextImpl(null, null, null, null, null, null,
          new RMContainerTokenSecretManager(csConf),
          new NMTokenSecretManagerInRM(csConf),
          new ClientToAMTokenSecretManagerInRM(), null);
  rmContext.setNodeLabelManager(nodeLabelManager);
  capacityScheduler.setConf(csConf);
  capacityScheduler.setRMContext(rmContext);
  capacityScheduler.init(csConf);
  capacityScheduler.start();
  
  csConf.setQueues("root", new String[] { "a", "x" });
  csConf.setQueues("root.a", new String[] { "y" });
  csConf.setCapacity("root.x", 50);
  csConf.setCapacity("root.a", 50);
  csConf.setCapacity("root.a.y", 100);
  
  capacityScheduler.reinitialize(csConf, rmContext);
}
Example #25
Source File: TestQueueParsing.java    From big-c with Apache License 2.0 5 votes vote down vote up 
@Test
public void testQueueParsingWhenLabelsNotExistedInNodeLabelManager()
    throws IOException {
  YarnConfiguration conf = new YarnConfiguration();
  CapacitySchedulerConfiguration csConf =
      new CapacitySchedulerConfiguration(conf);
  setupQueueConfigurationWithLabels(csConf);

  CapacityScheduler capacityScheduler = new CapacityScheduler();
  RMContextImpl rmContext =
      new RMContextImpl(null, null, null, null, null, null,
          new RMContainerTokenSecretManager(csConf),
          new NMTokenSecretManagerInRM(csConf),
          new ClientToAMTokenSecretManagerInRM(), null);
  
  RMNodeLabelsManager nodeLabelsManager = new NullRMNodeLabelsManager();
  nodeLabelsManager.init(conf);
  nodeLabelsManager.start();
  
  rmContext.setNodeLabelManager(nodeLabelsManager);
  capacityScheduler.setConf(csConf);
  capacityScheduler.setRMContext(rmContext);
  capacityScheduler.init(csConf);
  capacityScheduler.start();
  ServiceOperations.stopQuietly(capacityScheduler);
  ServiceOperations.stopQuietly(nodeLabelsManager);
}
Example #26
Source File: TestFifoScheduler.java    From big-c with Apache License 2.0 5 votes vote down vote up 
@Test(timeout=5000)
public void testAppAttemptMetrics() throws Exception {
  AsyncDispatcher dispatcher = new InlineDispatcher();
  
  FifoScheduler scheduler = new FifoScheduler();
  RMApplicationHistoryWriter writer = mock(RMApplicationHistoryWriter.class);
  RMContext rmContext = new RMContextImpl(dispatcher, null,
      null, null, null, null, null, null, null, writer, scheduler);
  ((RMContextImpl) rmContext).setSystemMetricsPublisher(
      mock(SystemMetricsPublisher.class));

  Configuration conf = new Configuration();
  scheduler.setRMContext(rmContext);
  scheduler.init(conf);
  scheduler.start();
  scheduler.reinitialize(conf, rmContext);
  QueueMetrics metrics = scheduler.getRootQueueMetrics();
  int beforeAppsSubmitted = metrics.getAppsSubmitted();

  ApplicationId appId = BuilderUtils.newApplicationId(200, 1);
  ApplicationAttemptId appAttemptId = BuilderUtils.newApplicationAttemptId(
      appId, 1);

  SchedulerEvent appEvent = new AppAddedSchedulerEvent(appId, "queue", "user");
  scheduler.handle(appEvent);
  SchedulerEvent attemptEvent =
      new AppAttemptAddedSchedulerEvent(appAttemptId, false);
  scheduler.handle(attemptEvent);

  appAttemptId = BuilderUtils.newApplicationAttemptId(appId, 2);
  SchedulerEvent attemptEvent2 =
      new AppAttemptAddedSchedulerEvent(appAttemptId, false);
  scheduler.handle(attemptEvent2);

  int afterAppsSubmitted = metrics.getAppsSubmitted();
  Assert.assertEquals(1, afterAppsSubmitted - beforeAppsSubmitted);
  scheduler.stop();
}
Example #27
Source File: TestQueueParsing.java    From hadoop with Apache License 2.0 5 votes vote down vote up 
/**
 * Test init a queue configuration, children's capacity for a given label
 * doesn't equals to 100%. This expect IllegalArgumentException thrown.
 */
@Test(expected = IllegalArgumentException.class)
public void testQueueParsingWithSumOfChildLabelCapacityNot100PercentWithWildCard()
    throws IOException {
  nodeLabelManager.addToCluserNodeLabels(ImmutableSet
      .of("red", "blue"));
  
  YarnConfiguration conf = new YarnConfiguration();
  CapacitySchedulerConfiguration csConf =
      new CapacitySchedulerConfiguration(conf);
  setupQueueConfigurationWithLabels(csConf);
  csConf.setCapacityByLabel(CapacitySchedulerConfiguration.ROOT + ".b.b3",
      "red", 24);
  csConf.setAccessibleNodeLabels(CapacitySchedulerConfiguration.ROOT,
      ImmutableSet.of(RMNodeLabelsManager.ANY));
  csConf.setAccessibleNodeLabels(CapacitySchedulerConfiguration.ROOT + ".b",
      ImmutableSet.of(RMNodeLabelsManager.ANY));

  CapacityScheduler capacityScheduler = new CapacityScheduler();
  RMContextImpl rmContext =
      new RMContextImpl(null, null, null, null, null, null,
          new RMContainerTokenSecretManager(csConf),
          new NMTokenSecretManagerInRM(csConf),
          new ClientToAMTokenSecretManagerInRM(), null);
  rmContext.setNodeLabelManager(nodeLabelManager);
  capacityScheduler.setConf(csConf);
  capacityScheduler.setRMContext(rmContext);
  capacityScheduler.init(csConf);
  capacityScheduler.start();
  ServiceOperations.stopQuietly(capacityScheduler);
}
Example #28
Source File: TestRMWebAppFairScheduler.java    From big-c with Apache License 2.0 5 votes vote down vote up 
private static RMContext mockRMContext(List<RMAppState> states) {
  final ConcurrentMap<ApplicationId, RMApp> applicationsMaps = Maps
      .newConcurrentMap();
  int i = 0;
  for (RMAppState state : states) {
    MockRMApp app = new MockRMApp(i, i, state) {
      @Override
      public RMAppMetrics getRMAppMetrics() {
        return new RMAppMetrics(Resource.newInstance(0, 0), 0, 0, 0, 0);
      }
      @Override
      public YarnApplicationState createApplicationState() {
        return YarnApplicationState.ACCEPTED;
      }
    };
    RMAppAttempt attempt = mock(RMAppAttempt.class);
    app.setCurrentAppAttempt(attempt);
    applicationsMaps.put(app.getApplicationId(), app);
    i++;
  }

  RMContextImpl rmContext =  new RMContextImpl(null, null, null, null,
      null, null, null, null, null, null) {
    @Override
    public ConcurrentMap<ApplicationId, RMApp> getRMApps() {
      return applicationsMaps;
    }
    @Override
    public ResourceScheduler getScheduler() {
      return mock(AbstractYarnScheduler.class);
    }
  };
  return rmContext;
}
Example #29
Source File: TestQueueParsing.java    From hadoop with Apache License 2.0 5 votes vote down vote up 
@Test(expected = IOException.class)
public void testQueueParsingWithMoveQueue()
    throws IOException {
  YarnConfiguration conf = new YarnConfiguration();
  CapacitySchedulerConfiguration csConf =
      new CapacitySchedulerConfiguration(conf);
  csConf.setQueues("root", new String[] { "a" });
  csConf.setQueues("root.a", new String[] { "x", "y" });
  csConf.setCapacity("root.a", 100);
  csConf.setCapacity("root.a.x", 50);
  csConf.setCapacity("root.a.y", 50);

  CapacityScheduler capacityScheduler = new CapacityScheduler();
  RMContextImpl rmContext =
      new RMContextImpl(null, null, null, null, null, null,
          new RMContainerTokenSecretManager(csConf),
          new NMTokenSecretManagerInRM(csConf),
          new ClientToAMTokenSecretManagerInRM(), null);
  rmContext.setNodeLabelManager(nodeLabelManager);
  capacityScheduler.setConf(csConf);
  capacityScheduler.setRMContext(rmContext);
  capacityScheduler.init(csConf);
  capacityScheduler.start();
  
  csConf.setQueues("root", new String[] { "a", "x" });
  csConf.setQueues("root.a", new String[] { "y" });
  csConf.setCapacity("root.x", 50);
  csConf.setCapacity("root.a", 50);
  csConf.setCapacity("root.a.y", 100);
  
  capacityScheduler.reinitialize(csConf, rmContext);
}
Example #30
Source File: TestRMWebServices.java    From big-c with Apache License 2.0 5 votes vote down vote up 
@Test
public void testAppsRace() throws Exception {
  // mock up an RM that returns app reports for apps that don't exist
  // in the RMApps list
  ApplicationId appId = ApplicationId.newInstance(1, 1);
  ApplicationReport mockReport = mock(ApplicationReport.class);
  when(mockReport.getApplicationId()).thenReturn(appId);
  GetApplicationsResponse mockAppsResponse =
      mock(GetApplicationsResponse.class);
  when(mockAppsResponse.getApplicationList())
    .thenReturn(Arrays.asList(new ApplicationReport[] { mockReport }));
  ClientRMService mockClientSvc = mock(ClientRMService.class);
  when(mockClientSvc.getApplications(isA(GetApplicationsRequest.class),
      anyBoolean())).thenReturn(mockAppsResponse);
  ResourceManager mockRM = mock(ResourceManager.class);
  RMContextImpl rmContext = new RMContextImpl(null, null, null, null, null,
      null, null, null, null, null);
  when(mockRM.getRMContext()).thenReturn(rmContext);
  when(mockRM.getClientRMService()).thenReturn(mockClientSvc);

  RMWebServices webSvc = new RMWebServices(mockRM, new Configuration(),
      mock(HttpServletResponse.class));

  final Set<String> emptySet =
      Collections.unmodifiableSet(Collections.<String>emptySet());

  // verify we don't get any apps when querying
  HttpServletRequest mockHsr = mock(HttpServletRequest.class);
  AppsInfo appsInfo = webSvc.getApps(mockHsr, null, emptySet, null,
      null, null, null, null, null, null, null, emptySet, emptySet);
  assertTrue(appsInfo.getApps().isEmpty());

  // verify we don't get an NPE when specifying a final status query
  appsInfo = webSvc.getApps(mockHsr, null, emptySet, "FAILED",
      null, null, null, null, null, null, null, emptySet, emptySet);
  assertTrue(appsInfo.getApps().isEmpty());
}