org.apache.hadoop.yarn.server.resourcemanager.security.NMTokenSecretManagerInRM Java Examples

The following examples show how to use org.apache.hadoop.yarn.server.resourcemanager.security.NMTokenSecretManagerInRM. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: TestQueueParsing.java    From big-c with Apache License 2.0 6 votes vote down vote up 
@Test
public void testQueueParsingWithLabels() throws IOException {
  nodeLabelManager.addToCluserNodeLabels(ImmutableSet.of("red", "blue"));
  
  YarnConfiguration conf = new YarnConfiguration();
  CapacitySchedulerConfiguration csConf =
      new CapacitySchedulerConfiguration(conf);
  setupQueueConfigurationWithLabels(csConf);

  CapacityScheduler capacityScheduler = new CapacityScheduler();
  RMContextImpl rmContext =
      new RMContextImpl(null, null, null, null, null, null,
          new RMContainerTokenSecretManager(csConf),
          new NMTokenSecretManagerInRM(csConf),
          new ClientToAMTokenSecretManagerInRM(), null);
  rmContext.setNodeLabelManager(nodeLabelManager);
  capacityScheduler.setConf(csConf);
  capacityScheduler.setRMContext(rmContext);
  capacityScheduler.init(csConf);
  capacityScheduler.start();
  checkQueueLabels(capacityScheduler);
  ServiceOperations.stopQuietly(capacityScheduler);
}
Example #2
Source File: TestCapacityScheduler.java    From hadoop with Apache License 2.0 6 votes vote down vote up 
/** Test that parseQueue throws an exception when two leaf queues have the
  *  same name
* @throws IOException
  */
 @Test(expected=IOException.class)
 public void testParseQueue() throws IOException {
   CapacityScheduler cs = new CapacityScheduler();
   cs.setConf(new YarnConfiguration());
   cs.setRMContext(resourceManager.getRMContext());
   CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration();
   setupQueueConfiguration(conf);
   cs.init(conf);
   cs.start();

   conf.setQueues(CapacitySchedulerConfiguration.ROOT + ".a.a1", new String[] {"b1"} );
   conf.setCapacity(CapacitySchedulerConfiguration.ROOT + ".a.a1.b1", 100.0f);
   conf.setUserLimitFactor(CapacitySchedulerConfiguration.ROOT + ".a.a1.b1", 100.0f);

   cs.reinitialize(conf, new RMContextImpl(null, null, null, null, null,
     null, new RMContainerTokenSecretManager(conf),
     new NMTokenSecretManagerInRM(conf),
     new ClientToAMTokenSecretManagerInRM(), null));
 }
Example #3
Source File: TestCapacityScheduler.java    From hadoop with Apache License 2.0 6 votes vote down vote up 
@Test
public void testRefreshQueues() throws Exception {
  CapacityScheduler cs = new CapacityScheduler();
  CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration();
  RMContextImpl rmContext =  new RMContextImpl(null, null, null, null, null,
      null, new RMContainerTokenSecretManager(conf),
      new NMTokenSecretManagerInRM(conf),
      new ClientToAMTokenSecretManagerInRM(), null);
  setupQueueConfiguration(conf);
  cs.setConf(new YarnConfiguration());
  cs.setRMContext(resourceManager.getRMContext());
  cs.init(conf);
  cs.start();
  cs.reinitialize(conf, rmContext);
  checkQueueCapacities(cs, A_CAPACITY, B_CAPACITY);

  conf.setCapacity(A, 80f);
  conf.setCapacity(B, 20f);
  cs.reinitialize(conf, mockContext);
  checkQueueCapacities(cs, 80f, 20f);
  cs.stop();
}
Example #4
Source File: TestQueueParsing.java    From big-c with Apache License 2.0 6 votes vote down vote up 
@Test
public void testQueueParsingWithLabelsInherit() throws IOException {
  nodeLabelManager.addToCluserNodeLabels(ImmutableSet.of("red", "blue"));

  YarnConfiguration conf = new YarnConfiguration();
  CapacitySchedulerConfiguration csConf =
      new CapacitySchedulerConfiguration(conf);
  setupQueueConfigurationWithLabelsInherit(csConf);

  CapacityScheduler capacityScheduler = new CapacityScheduler();
  RMContextImpl rmContext =
      new RMContextImpl(null, null, null, null, null, null,
          new RMContainerTokenSecretManager(csConf),
          new NMTokenSecretManagerInRM(csConf),
          new ClientToAMTokenSecretManagerInRM(), null);
  rmContext.setNodeLabelManager(nodeLabelManager);
  capacityScheduler.setConf(csConf);
  capacityScheduler.setRMContext(rmContext);
  capacityScheduler.init(csConf);
  capacityScheduler.start();
  checkQueueLabelsInheritConfig(capacityScheduler);
  ServiceOperations.stopQuietly(capacityScheduler);
}
Example #5
Source File: TestQueueParsing.java    From hadoop with Apache License 2.0 6 votes vote down vote up 
@Test
public void testQueueParsingWithLabels() throws IOException {
  nodeLabelManager.addToCluserNodeLabels(ImmutableSet.of("red", "blue"));
  
  YarnConfiguration conf = new YarnConfiguration();
  CapacitySchedulerConfiguration csConf =
      new CapacitySchedulerConfiguration(conf);
  setupQueueConfigurationWithLabels(csConf);

  CapacityScheduler capacityScheduler = new CapacityScheduler();
  RMContextImpl rmContext =
      new RMContextImpl(null, null, null, null, null, null,
          new RMContainerTokenSecretManager(csConf),
          new NMTokenSecretManagerInRM(csConf),
          new ClientToAMTokenSecretManagerInRM(), null);
  rmContext.setNodeLabelManager(nodeLabelManager);
  capacityScheduler.setConf(csConf);
  capacityScheduler.setRMContext(rmContext);
  capacityScheduler.init(csConf);
  capacityScheduler.start();
  checkQueueLabels(capacityScheduler);
  ServiceOperations.stopQuietly(capacityScheduler);
}
Example #6
Source File: TestQueueParsing.java    From hadoop with Apache License 2.0 6 votes vote down vote up 
@Test
public void testQueueParsingWithLabelsInherit() throws IOException {
  nodeLabelManager.addToCluserNodeLabels(ImmutableSet.of("red", "blue"));

  YarnConfiguration conf = new YarnConfiguration();
  CapacitySchedulerConfiguration csConf =
      new CapacitySchedulerConfiguration(conf);
  setupQueueConfigurationWithLabelsInherit(csConf);

  CapacityScheduler capacityScheduler = new CapacityScheduler();
  RMContextImpl rmContext =
      new RMContextImpl(null, null, null, null, null, null,
          new RMContainerTokenSecretManager(csConf),
          new NMTokenSecretManagerInRM(csConf),
          new ClientToAMTokenSecretManagerInRM(), null);
  rmContext.setNodeLabelManager(nodeLabelManager);
  capacityScheduler.setConf(csConf);
  capacityScheduler.setRMContext(rmContext);
  capacityScheduler.init(csConf);
  capacityScheduler.start();
  checkQueueLabelsInheritConfig(capacityScheduler);
  ServiceOperations.stopQuietly(capacityScheduler);
}
Example #7
Source File: TestQueueParsing.java    From big-c with Apache License 2.0 6 votes vote down vote up 
@Test
public void testQueueParsingWhenLabelsNotExist() throws IOException {
  YarnConfiguration conf = new YarnConfiguration();
  CapacitySchedulerConfiguration csConf =
      new CapacitySchedulerConfiguration(conf);
  setupQueueConfigurationWithLabels(csConf);

  CapacityScheduler capacityScheduler = new CapacityScheduler();
  RMContextImpl rmContext =
      new RMContextImpl(null, null, null, null, null, null,
          new RMContainerTokenSecretManager(csConf),
          new NMTokenSecretManagerInRM(csConf),
          new ClientToAMTokenSecretManagerInRM(), null);
  
  RMNodeLabelsManager nodeLabelsManager = new NullRMNodeLabelsManager();
  nodeLabelsManager.init(conf);
  nodeLabelsManager.start();
  
  rmContext.setNodeLabelManager(nodeLabelsManager);
  capacityScheduler.setConf(csConf);
  capacityScheduler.setRMContext(rmContext);
  capacityScheduler.init(csConf);
  capacityScheduler.start();
  ServiceOperations.stopQuietly(capacityScheduler);
  ServiceOperations.stopQuietly(nodeLabelsManager);
}
Example #8
Source File: MockRM.java    From hadoop with Apache License 2.0 6 votes vote down vote up 
@Override
protected ResourceTrackerService createResourceTrackerService() {

  RMContainerTokenSecretManager containerTokenSecretManager =
      getRMContext().getContainerTokenSecretManager();
  containerTokenSecretManager.rollMasterKey();
  NMTokenSecretManagerInRM nmTokenSecretManager =
      getRMContext().getNMTokenSecretManager();
  nmTokenSecretManager.rollMasterKey();
  return new ResourceTrackerService(getRMContext(), nodesListManager,
      this.nmLivelinessMonitor, containerTokenSecretManager,
      nmTokenSecretManager) {

    @Override
    protected void serviceStart() {
      // override to not start rpc handler
    }

    @Override
    protected void serviceStop() {
      // don't do anything
    }
  };
}
Example #9
Source File: TestCapacityScheduler.java    From big-c with Apache License 2.0 6 votes vote down vote up 
/** Test that parseQueue throws an exception when two leaf queues have the
  *  same name
* @throws IOException
  */
 @Test(expected=IOException.class)
 public void testParseQueue() throws IOException {
   CapacityScheduler cs = new CapacityScheduler();
   cs.setConf(new YarnConfiguration());
   cs.setRMContext(resourceManager.getRMContext());
   CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration();
   setupQueueConfiguration(conf);
   cs.init(conf);
   cs.start();

   conf.setQueues(CapacitySchedulerConfiguration.ROOT + ".a.a1", new String[] {"b1"} );
   conf.setCapacity(CapacitySchedulerConfiguration.ROOT + ".a.a1.b1", 100.0f);
   conf.setUserLimitFactor(CapacitySchedulerConfiguration.ROOT + ".a.a1.b1", 100.0f);

   cs.reinitialize(conf, new RMContextImpl(null, null, null, null, null,
     null, new RMContainerTokenSecretManager(conf),
     new NMTokenSecretManagerInRM(conf),
     new ClientToAMTokenSecretManagerInRM(), null));
 }
Example #10
Source File: TestRMWebApp.java    From hadoop with Apache License 2.0 6 votes vote down vote up 
public static CapacityScheduler mockCapacityScheduler() throws IOException {
  // stolen from TestCapacityScheduler
  CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration();
  setupQueueConfiguration(conf);

  CapacityScheduler cs = new CapacityScheduler();
  cs.setConf(new YarnConfiguration());
  RMContext rmContext = new RMContextImpl(null, null, null, null, null,
      null, new RMContainerTokenSecretManager(conf),
      new NMTokenSecretManagerInRM(conf),
      new ClientToAMTokenSecretManagerInRM(), null);
  rmContext.setNodeLabelManager(new NullRMNodeLabelsManager());
  cs.setRMContext(rmContext);
  cs.init(conf);
  return cs;
}
Example #11
Source File: TestCapacityScheduler.java    From big-c with Apache License 2.0 6 votes vote down vote up 
@Test
public void testRefreshQueues() throws Exception {
  CapacityScheduler cs = new CapacityScheduler();
  CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration();
  RMContextImpl rmContext =  new RMContextImpl(null, null, null, null, null,
      null, new RMContainerTokenSecretManager(conf),
      new NMTokenSecretManagerInRM(conf),
      new ClientToAMTokenSecretManagerInRM(), null);
  setupQueueConfiguration(conf);
  cs.setConf(new YarnConfiguration());
  cs.setRMContext(resourceManager.getRMContext());
  cs.init(conf);
  cs.start();
  cs.reinitialize(conf, rmContext);
  checkQueueCapacities(cs, A_CAPACITY, B_CAPACITY);

  conf.setCapacity(A, 80f);
  conf.setCapacity(B, 20f);
  cs.reinitialize(conf, mockContext);
  checkQueueCapacities(cs, 80f, 20f);
  cs.stop();
}
Example #12
Source File: RMContextImpl.java    From hadoop with Apache License 2.0 6 votes vote down vote up 
@VisibleForTesting
// helper constructor for tests
public RMContextImpl(Dispatcher rmDispatcher,
    ContainerAllocationExpirer containerAllocationExpirer,
    AMLivelinessMonitor amLivelinessMonitor,
    AMLivelinessMonitor amFinishingMonitor,
    DelegationTokenRenewer delegationTokenRenewer,
    AMRMTokenSecretManager appTokenSecretManager,
    RMContainerTokenSecretManager containerTokenSecretManager,
    NMTokenSecretManagerInRM nmTokenSecretManager,
    ClientToAMTokenSecretManagerInRM clientToAMTokenSecretManager,
    RMApplicationHistoryWriter rmApplicationHistoryWriter) {
  this(
    rmDispatcher,
    containerAllocationExpirer,
    amLivelinessMonitor,
    amFinishingMonitor,
    delegationTokenRenewer,
    appTokenSecretManager,
    containerTokenSecretManager,
    nmTokenSecretManager,
    clientToAMTokenSecretManager,
    rmApplicationHistoryWriter,
    null);
}
Example #13
Source File: TestQueueParsing.java    From hadoop with Apache License 2.0 6 votes vote down vote up 
@Test
public void testQueueParsingWhenLabelsNotExist() throws IOException {
  YarnConfiguration conf = new YarnConfiguration();
  CapacitySchedulerConfiguration csConf =
      new CapacitySchedulerConfiguration(conf);
  setupQueueConfigurationWithLabels(csConf);

  CapacityScheduler capacityScheduler = new CapacityScheduler();
  RMContextImpl rmContext =
      new RMContextImpl(null, null, null, null, null, null,
          new RMContainerTokenSecretManager(csConf),
          new NMTokenSecretManagerInRM(csConf),
          new ClientToAMTokenSecretManagerInRM(), null);
  
  RMNodeLabelsManager nodeLabelsManager = new NullRMNodeLabelsManager();
  nodeLabelsManager.init(conf);
  nodeLabelsManager.start();
  
  rmContext.setNodeLabelManager(nodeLabelsManager);
  capacityScheduler.setConf(csConf);
  capacityScheduler.setRMContext(rmContext);
  capacityScheduler.init(csConf);
  capacityScheduler.start();
  ServiceOperations.stopQuietly(capacityScheduler);
  ServiceOperations.stopQuietly(nodeLabelsManager);
}
Example #14
Source File: TestQueueParsing.java    From hadoop with Apache License 2.0 6 votes vote down vote up 
/**
 * Test init a queue configuration, children's capacity for a given label
 * doesn't equals to 100%. This expect IllegalArgumentException thrown.
 */
@Test(expected = IllegalArgumentException.class)
public void testQueueParsingFailWhenSumOfChildrenNonLabeledCapacityNot100Percent()
    throws IOException {
  nodeLabelManager.addToCluserNodeLabels(ImmutableSet
      .of("red", "blue"));
  
  YarnConfiguration conf = new YarnConfiguration();
  CapacitySchedulerConfiguration csConf =
      new CapacitySchedulerConfiguration(conf);
  setupQueueConfiguration(csConf);
  csConf.setCapacity(CapacitySchedulerConfiguration.ROOT + ".c.c2", 5);

  CapacityScheduler capacityScheduler = new CapacityScheduler();
  RMContextImpl rmContext =
      new RMContextImpl(null, null, null, null, null, null,
          new RMContainerTokenSecretManager(csConf),
          new NMTokenSecretManagerInRM(csConf),
          new ClientToAMTokenSecretManagerInRM(), null);
  rmContext.setNodeLabelManager(nodeLabelManager);
  capacityScheduler.setConf(csConf);
  capacityScheduler.setRMContext(rmContext);
  capacityScheduler.init(csConf);
  capacityScheduler.start();
  ServiceOperations.stopQuietly(capacityScheduler);
}
Example #15
Source File: MockRM.java    From big-c with Apache License 2.0 6 votes vote down vote up 
@Override
protected ResourceTrackerService createResourceTrackerService() {

  RMContainerTokenSecretManager containerTokenSecretManager =
      getRMContext().getContainerTokenSecretManager();
  containerTokenSecretManager.rollMasterKey();
  NMTokenSecretManagerInRM nmTokenSecretManager =
      getRMContext().getNMTokenSecretManager();
  nmTokenSecretManager.rollMasterKey();
  return new ResourceTrackerService(getRMContext(), nodesListManager,
      this.nmLivelinessMonitor, containerTokenSecretManager,
      nmTokenSecretManager) {

    @Override
    protected void serviceStart() {
      // override to not start rpc handler
    }

    @Override
    protected void serviceStop() {
      // don't do anything
    }
  };
}
Example #16
Source File: TestRMWebApp.java    From big-c with Apache License 2.0 6 votes vote down vote up 
public static CapacityScheduler mockCapacityScheduler() throws IOException {
  // stolen from TestCapacityScheduler
  CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration();
  setupQueueConfiguration(conf);

  CapacityScheduler cs = new CapacityScheduler();
  cs.setConf(new YarnConfiguration());
  RMContext rmContext = new RMContextImpl(null, null, null, null, null,
      null, new RMContainerTokenSecretManager(conf),
      new NMTokenSecretManagerInRM(conf),
      new ClientToAMTokenSecretManagerInRM(), null);
  rmContext.setNodeLabelManager(new NullRMNodeLabelsManager());
  cs.setRMContext(rmContext);
  cs.init(conf);
  return cs;
}
Example #17
Source File: RMContextImpl.java    From big-c with Apache License 2.0 6 votes vote down vote up 
@VisibleForTesting
// helper constructor for tests
public RMContextImpl(Dispatcher rmDispatcher,
    ContainerAllocationExpirer containerAllocationExpirer,
    AMLivelinessMonitor amLivelinessMonitor,
    AMLivelinessMonitor amFinishingMonitor,
    DelegationTokenRenewer delegationTokenRenewer,
    AMRMTokenSecretManager appTokenSecretManager,
    RMContainerTokenSecretManager containerTokenSecretManager,
    NMTokenSecretManagerInRM nmTokenSecretManager,
    ClientToAMTokenSecretManagerInRM clientToAMTokenSecretManager,
    RMApplicationHistoryWriter rmApplicationHistoryWriter) {
  this(
    rmDispatcher,
    containerAllocationExpirer,
    amLivelinessMonitor,
    amFinishingMonitor,
    delegationTokenRenewer,
    appTokenSecretManager,
    containerTokenSecretManager,
    nmTokenSecretManager,
    clientToAMTokenSecretManager,
    rmApplicationHistoryWriter,
    null);
}
Example #18
Source File: RMContextImpl.java    From big-c with Apache License 2.0 6 votes vote down vote up 
@VisibleForTesting
// helper constructor for tests
public RMContextImpl(Dispatcher rmDispatcher,
    ContainerAllocationExpirer containerAllocationExpirer,
    AMLivelinessMonitor amLivelinessMonitor,
    AMLivelinessMonitor amFinishingMonitor,
    DelegationTokenRenewer delegationTokenRenewer,
    AMRMTokenSecretManager appTokenSecretManager,
    RMContainerTokenSecretManager containerTokenSecretManager,
    NMTokenSecretManagerInRM nmTokenSecretManager,
    ClientToAMTokenSecretManagerInRM clientToAMTokenSecretManager,
    RMApplicationHistoryWriter rmApplicationHistoryWriter,
    ResourceScheduler scheduler) {
  this();
  this.setDispatcher(rmDispatcher);
  setActiveServiceContext(new RMActiveServiceContext(rmDispatcher,
      containerAllocationExpirer, amLivelinessMonitor, amFinishingMonitor,
      delegationTokenRenewer, appTokenSecretManager,
      containerTokenSecretManager, nmTokenSecretManager,
      clientToAMTokenSecretManager, rmApplicationHistoryWriter,
      scheduler));

  ConfigurationProvider provider = new LocalConfigurationProvider();
  setConfigurationProvider(provider);
}
Example #19
Source File: RMContextImpl.java    From hadoop with Apache License 2.0 6 votes vote down vote up 
@VisibleForTesting
// helper constructor for tests
public RMContextImpl(Dispatcher rmDispatcher,
    ContainerAllocationExpirer containerAllocationExpirer,
    AMLivelinessMonitor amLivelinessMonitor,
    AMLivelinessMonitor amFinishingMonitor,
    DelegationTokenRenewer delegationTokenRenewer,
    AMRMTokenSecretManager appTokenSecretManager,
    RMContainerTokenSecretManager containerTokenSecretManager,
    NMTokenSecretManagerInRM nmTokenSecretManager,
    ClientToAMTokenSecretManagerInRM clientToAMTokenSecretManager,
    RMApplicationHistoryWriter rmApplicationHistoryWriter,
    ResourceScheduler scheduler) {
  this();
  this.setDispatcher(rmDispatcher);
  setActiveServiceContext(new RMActiveServiceContext(rmDispatcher,
      containerAllocationExpirer, amLivelinessMonitor, amFinishingMonitor,
      delegationTokenRenewer, appTokenSecretManager,
      containerTokenSecretManager, nmTokenSecretManager,
      clientToAMTokenSecretManager, rmApplicationHistoryWriter,
      scheduler));

  ConfigurationProvider provider = new LocalConfigurationProvider();
  setConfigurationProvider(provider);
}
Example #20
Source File: TestQueueParsing.java    From big-c with Apache License 2.0 6 votes vote down vote up 
/**
 * Test init a queue configuration, children's capacity for a given label
 * doesn't equals to 100%. This expect IllegalArgumentException thrown.
 */
@Test(expected = IllegalArgumentException.class)
public void testQueueParsingFailWhenSumOfChildrenNonLabeledCapacityNot100Percent()
    throws IOException {
  nodeLabelManager.addToCluserNodeLabels(ImmutableSet
      .of("red", "blue"));
  
  YarnConfiguration conf = new YarnConfiguration();
  CapacitySchedulerConfiguration csConf =
      new CapacitySchedulerConfiguration(conf);
  setupQueueConfiguration(csConf);
  csConf.setCapacity(CapacitySchedulerConfiguration.ROOT + ".c.c2", 5);

  CapacityScheduler capacityScheduler = new CapacityScheduler();
  RMContextImpl rmContext =
      new RMContextImpl(null, null, null, null, null, null,
          new RMContainerTokenSecretManager(csConf),
          new NMTokenSecretManagerInRM(csConf),
          new ClientToAMTokenSecretManagerInRM(), null);
  rmContext.setNodeLabelManager(nodeLabelManager);
  capacityScheduler.setConf(csConf);
  capacityScheduler.setRMContext(rmContext);
  capacityScheduler.init(csConf);
  capacityScheduler.start();
  ServiceOperations.stopQuietly(capacityScheduler);
}
Example #21
Source File: TestQueueParsing.java    From hadoop with Apache License 2.0 5 votes vote down vote up 
/**
 * Test init a queue configuration, children's capacity for a given label
 * doesn't equals to 100%. This expect IllegalArgumentException thrown.
 */
@Test(expected = IllegalArgumentException.class)
public void testQueueParsingWithSumOfChildLabelCapacityNot100PercentWithWildCard()
    throws IOException {
  nodeLabelManager.addToCluserNodeLabels(ImmutableSet
      .of("red", "blue"));
  
  YarnConfiguration conf = new YarnConfiguration();
  CapacitySchedulerConfiguration csConf =
      new CapacitySchedulerConfiguration(conf);
  setupQueueConfigurationWithLabels(csConf);
  csConf.setCapacityByLabel(CapacitySchedulerConfiguration.ROOT + ".b.b3",
      "red", 24);
  csConf.setAccessibleNodeLabels(CapacitySchedulerConfiguration.ROOT,
      ImmutableSet.of(RMNodeLabelsManager.ANY));
  csConf.setAccessibleNodeLabels(CapacitySchedulerConfiguration.ROOT + ".b",
      ImmutableSet.of(RMNodeLabelsManager.ANY));

  CapacityScheduler capacityScheduler = new CapacityScheduler();
  RMContextImpl rmContext =
      new RMContextImpl(null, null, null, null, null, null,
          new RMContainerTokenSecretManager(csConf),
          new NMTokenSecretManagerInRM(csConf),
          new ClientToAMTokenSecretManagerInRM(), null);
  rmContext.setNodeLabelManager(nodeLabelManager);
  capacityScheduler.setConf(csConf);
  capacityScheduler.setRMContext(rmContext);
  capacityScheduler.init(csConf);
  capacityScheduler.start();
  ServiceOperations.stopQuietly(capacityScheduler);
}
Example #22
Source File: TestQueueParsing.java    From hadoop with Apache License 2.0 5 votes vote down vote up 
/**
 * Test init a queue configuration, children's capacity for a given label
 * doesn't equals to 100%. This expect IllegalArgumentException thrown.
 */
@Test(expected = IllegalArgumentException.class)
public void testQueueParsingFailWhenSumOfChildrenLabeledCapacityNot100Percent()
    throws IOException {
  nodeLabelManager.addToCluserNodeLabels(ImmutableSet
      .of("red", "blue"));
  
  YarnConfiguration conf = new YarnConfiguration();
  CapacitySchedulerConfiguration csConf =
      new CapacitySchedulerConfiguration(conf);
  setupQueueConfigurationWithLabels(csConf);
  csConf.setCapacityByLabel(CapacitySchedulerConfiguration.ROOT + ".b.b3",
      "red", 24);

  CapacityScheduler capacityScheduler = new CapacityScheduler();
  RMContextImpl rmContext =
      new RMContextImpl(null, null, null, null, null, null,
          new RMContainerTokenSecretManager(csConf),
          new NMTokenSecretManagerInRM(csConf),
          new ClientToAMTokenSecretManagerInRM(), null);
  rmContext.setNodeLabelManager(nodeLabelManager);
  capacityScheduler.setConf(csConf);
  capacityScheduler.setRMContext(rmContext);
  capacityScheduler.init(csConf);
  capacityScheduler.start();
  ServiceOperations.stopQuietly(capacityScheduler);
}
Example #23
Source File: TestQueueParsing.java    From hadoop with Apache License 2.0 5 votes vote down vote up 
@Test(expected = IOException.class)
public void testQueueParsingWithMoveQueue()
    throws IOException {
  YarnConfiguration conf = new YarnConfiguration();
  CapacitySchedulerConfiguration csConf =
      new CapacitySchedulerConfiguration(conf);
  csConf.setQueues("root", new String[] { "a" });
  csConf.setQueues("root.a", new String[] { "x", "y" });
  csConf.setCapacity("root.a", 100);
  csConf.setCapacity("root.a.x", 50);
  csConf.setCapacity("root.a.y", 50);

  CapacityScheduler capacityScheduler = new CapacityScheduler();
  RMContextImpl rmContext =
      new RMContextImpl(null, null, null, null, null, null,
          new RMContainerTokenSecretManager(csConf),
          new NMTokenSecretManagerInRM(csConf),
          new ClientToAMTokenSecretManagerInRM(), null);
  rmContext.setNodeLabelManager(nodeLabelManager);
  capacityScheduler.setConf(csConf);
  capacityScheduler.setRMContext(rmContext);
  capacityScheduler.init(csConf);
  capacityScheduler.start();
  
  csConf.setQueues("root", new String[] { "a", "x" });
  csConf.setQueues("root.a", new String[] { "y" });
  csConf.setCapacity("root.x", 50);
  csConf.setCapacity("root.a", 50);
  csConf.setCapacity("root.a.y", 100);
  
  capacityScheduler.reinitialize(csConf, rmContext);
}
Example #24
Source File: TestCapacityScheduler.java    From big-c with Apache License 2.0 5 votes vote down vote up 
@Test
public void testReconnectedNode() throws Exception {
  CapacitySchedulerConfiguration csConf =
      new CapacitySchedulerConfiguration();
  setupQueueConfiguration(csConf);
  CapacityScheduler cs = new CapacityScheduler();
  cs.setConf(new YarnConfiguration());
  cs.setRMContext(resourceManager.getRMContext());
  cs.init(csConf);
  cs.start();
  cs.reinitialize(csConf, new RMContextImpl(null, null, null, null,
    null, null, new RMContainerTokenSecretManager(csConf),
    new NMTokenSecretManagerInRM(csConf),
    new ClientToAMTokenSecretManagerInRM(), null));

  RMNode n1 = MockNodes.newNodeInfo(0, MockNodes.newResource(4 * GB), 1);
  RMNode n2 = MockNodes.newNodeInfo(0, MockNodes.newResource(2 * GB), 2);

  cs.handle(new NodeAddedSchedulerEvent(n1));
  cs.handle(new NodeAddedSchedulerEvent(n2));

  Assert.assertEquals(6 * GB, cs.getClusterResource().getMemory());

  // reconnect n1 with downgraded memory
  n1 = MockNodes.newNodeInfo(0, MockNodes.newResource(2 * GB), 1);
  cs.handle(new NodeRemovedSchedulerEvent(n1));
  cs.handle(new NodeAddedSchedulerEvent(n1));

  Assert.assertEquals(4 * GB, cs.getClusterResource().getMemory());
  cs.stop();
}
Example #25
Source File: TestQueueParsing.java    From hadoop with Apache License 2.0 5 votes vote down vote up 
@Test
public void testQueueParsingWhenLabelsNotExistedInNodeLabelManager()
    throws IOException {
  YarnConfiguration conf = new YarnConfiguration();
  CapacitySchedulerConfiguration csConf =
      new CapacitySchedulerConfiguration(conf);
  setupQueueConfigurationWithLabels(csConf);

  CapacityScheduler capacityScheduler = new CapacityScheduler();
  RMContextImpl rmContext =
      new RMContextImpl(null, null, null, null, null, null,
          new RMContainerTokenSecretManager(csConf),
          new NMTokenSecretManagerInRM(csConf),
          new ClientToAMTokenSecretManagerInRM(), null);
  
  RMNodeLabelsManager nodeLabelsManager = new NullRMNodeLabelsManager();
  nodeLabelsManager.init(conf);
  nodeLabelsManager.start();
  
  rmContext.setNodeLabelManager(nodeLabelsManager);
  capacityScheduler.setConf(csConf);
  capacityScheduler.setRMContext(rmContext);
  capacityScheduler.init(csConf);
  capacityScheduler.start();
  ServiceOperations.stopQuietly(capacityScheduler);
  ServiceOperations.stopQuietly(nodeLabelsManager);
}
Example #26
Source File: TestNMExpiry.java    From big-c with Apache License 2.0 5 votes vote down vote up 
@Before
public void setUp() {
  Configuration conf = new Configuration();
  // Dispatcher that processes events inline
  Dispatcher dispatcher = new InlineDispatcher();
  RMContext context = new RMContextImpl(dispatcher, null,
      null, null, null, null, null, null, null, null);
  dispatcher.register(SchedulerEventType.class,
      new InlineDispatcher.EmptyEventHandler());
  dispatcher.register(RMNodeEventType.class,
      new NodeEventDispatcher(context));
  NMLivelinessMonitor nmLivelinessMonitor = new TestNmLivelinessMonitor(
      dispatcher);
  nmLivelinessMonitor.init(conf);
  nmLivelinessMonitor.start();
  NodesListManager nodesListManager = new NodesListManager(context);
  nodesListManager.init(conf);
  RMContainerTokenSecretManager containerTokenSecretManager =
      new RMContainerTokenSecretManager(conf);
  containerTokenSecretManager.start();
  NMTokenSecretManagerInRM nmTokenSecretManager =
      new NMTokenSecretManagerInRM(conf);
  nmTokenSecretManager.start();
  resourceTrackerService = new ResourceTrackerService(context,
      nodesListManager, nmLivelinessMonitor, containerTokenSecretManager,
      nmTokenSecretManager);
  
  resourceTrackerService.init(conf);
  resourceTrackerService.start();
}
Example #27
Source File: TestRMNMRPCResponseId.java    From big-c with Apache License 2.0 5 votes vote down vote up 
@Before
public void setUp() {
  Configuration conf = new Configuration();
  // Dispatcher that processes events inline
  Dispatcher dispatcher = new InlineDispatcher();
  dispatcher.register(SchedulerEventType.class, new EventHandler<Event>() {
    @Override
    public void handle(Event event) {
      ; // ignore
    }
  });
  RMContext context =
      new RMContextImpl(dispatcher, null, null, null, null,
        null, new RMContainerTokenSecretManager(conf),
        new NMTokenSecretManagerInRM(conf), null, null);
  dispatcher.register(RMNodeEventType.class,
      new ResourceManager.NodeEventDispatcher(context));
  NodesListManager nodesListManager = new NodesListManager(context);
  nodesListManager.init(conf);
  
  context.getContainerTokenSecretManager().rollMasterKey();
  context.getNMTokenSecretManager().rollMasterKey();
  resourceTrackerService = new ResourceTrackerService(context,
      nodesListManager, new NMLivelinessMonitor(dispatcher),
      context.getContainerTokenSecretManager(),
      context.getNMTokenSecretManager());
  resourceTrackerService.init(conf);
}
Example #28
Source File: TestNMReconnect.java    From big-c with Apache License 2.0 5 votes vote down vote up 
@Before
public void setUp() {
  Configuration conf = new Configuration();
  // Dispatcher that processes events inline
  Dispatcher dispatcher = new InlineDispatcher();

  dispatcher.register(RMNodeEventType.class,
      new TestRMNodeEventDispatcher());

  RMContext context = new RMContextImpl(dispatcher, null,
      null, null, null, null, null, null, null, null);
  dispatcher.register(SchedulerEventType.class,
      new InlineDispatcher.EmptyEventHandler());
  dispatcher.register(RMNodeEventType.class,
      new NodeEventDispatcher(context));
  NMLivelinessMonitor nmLivelinessMonitor = new NMLivelinessMonitor(
      dispatcher);
  nmLivelinessMonitor.init(conf);
  nmLivelinessMonitor.start();
  NodesListManager nodesListManager = new NodesListManager(context);
  nodesListManager.init(conf);
  RMContainerTokenSecretManager containerTokenSecretManager =
      new RMContainerTokenSecretManager(conf);
  containerTokenSecretManager.start();
  NMTokenSecretManagerInRM nmTokenSecretManager =
      new NMTokenSecretManagerInRM(conf);
  nmTokenSecretManager.start();
  resourceTrackerService = new ResourceTrackerService(context,
      nodesListManager, nmLivelinessMonitor, containerTokenSecretManager,
      nmTokenSecretManager);
  
  resourceTrackerService.init(conf);
  resourceTrackerService.start();
}
Example #29
Source File: TestRMWebAppFairScheduler.java    From big-c with Apache License 2.0 5 votes vote down vote up 
private static FairScheduler mockFairScheduler() throws IOException {
  FairScheduler fs = new FairScheduler();
  FairSchedulerConfiguration conf = new FairSchedulerConfiguration();
  fs.setRMContext(new RMContextImpl(null, null, null, null, null,
      null, new RMContainerTokenSecretManager(conf),
      new NMTokenSecretManagerInRM(conf),
      new ClientToAMTokenSecretManagerInRM(), null));
  fs.init(conf);
  return fs;
}
Example #30
Source File: ResourceTrackerService.java    From big-c with Apache License 2.0 5 votes vote down vote up 
public ResourceTrackerService(RMContext rmContext,
    NodesListManager nodesListManager,
    NMLivelinessMonitor nmLivelinessMonitor,
    RMContainerTokenSecretManager containerTokenSecretManager,
    NMTokenSecretManagerInRM nmTokenSecretManager) {
  super(ResourceTrackerService.class.getName());
  this.rmContext = rmContext;
  this.nodesListManager = nodesListManager;
  this.nmLivelinessMonitor = nmLivelinessMonitor;
  this.containerTokenSecretManager = containerTokenSecretManager;
  this.nmTokenSecretManager = nmTokenSecretManager;

}