Java Code Examples for org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager#rollMasterKey()

The following examples show how to use org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager#rollMasterKey() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example 1

Source File: MockRM.java From hadoop with Apache License 2.0

6 votes

@Override
protected ResourceTrackerService createResourceTrackerService() {

  RMContainerTokenSecretManager containerTokenSecretManager =
      getRMContext().getContainerTokenSecretManager();
  containerTokenSecretManager.rollMasterKey();
  NMTokenSecretManagerInRM nmTokenSecretManager =
      getRMContext().getNMTokenSecretManager();
  nmTokenSecretManager.rollMasterKey();
  return new ResourceTrackerService(getRMContext(), nodesListManager,
      this.nmLivelinessMonitor, containerTokenSecretManager,
      nmTokenSecretManager) {

    @Override
    protected void serviceStart() {
      // override to not start rpc handler
    }

    @Override
    protected void serviceStop() {
      // don't do anything
    }
  };
}

Example 2

Source File: MockRM.java From big-c with Apache License 2.0

6 votes

@Override
protected ResourceTrackerService createResourceTrackerService() {

  RMContainerTokenSecretManager containerTokenSecretManager =
      getRMContext().getContainerTokenSecretManager();
  containerTokenSecretManager.rollMasterKey();
  NMTokenSecretManagerInRM nmTokenSecretManager =
      getRMContext().getNMTokenSecretManager();
  nmTokenSecretManager.rollMasterKey();
  return new ResourceTrackerService(getRMContext(), nodesListManager,
      this.nmLivelinessMonitor, containerTokenSecretManager,
      nmTokenSecretManager) {

    @Override
    protected void serviceStart() {
      // override to not start rpc handler
    }

    @Override
    protected void serviceStop() {
      // don't do anything
    }
  };
}

Example 3

Source File: TestContainerManagerSecurity.java From hadoop with Apache License 2.0

4 votes

/**
 * This tests a malice user getting a proper token but then messing with it by
 * tampering with containerID/Resource etc.. His/her containers should be
 * rejected.
 * 
 * @throws IOException
 * @throws InterruptedException
 * @throws YarnException
 */
private void testContainerToken(Configuration conf) throws IOException,
    InterruptedException, YarnException {

  LOG.info("Running test for malice user");
  /*
   * We need to check for containerToken (authorization).
   * Here we will be assuming that we have valid NMToken  
   * 1) ContainerToken used is expired.
   * 2) ContainerToken is tampered (resource is modified).
   */
  NMTokenSecretManagerInRM nmTokenSecretManagerInRM =
      yarnCluster.getResourceManager().getRMContext()
        .getNMTokenSecretManager();
  ApplicationId appId = ApplicationId.newInstance(1, 1);
  ApplicationAttemptId appAttemptId =
      ApplicationAttemptId.newInstance(appId, 0);
  ContainerId cId = ContainerId.newContainerId(appAttemptId, 0);
  NodeManager nm = yarnCluster.getNodeManager(0);
  NMTokenSecretManagerInNM nmTokenSecretManagerInNM =
      nm.getNMContext().getNMTokenSecretManager();
  String user = "test";
  
  waitForNMToReceiveNMTokenKey(nmTokenSecretManagerInNM, nm);

  NodeId nodeId = nm.getNMContext().getNodeId();
  
  // Both id should be equal.
  Assert.assertEquals(nmTokenSecretManagerInNM.getCurrentKey().getKeyId(),
      nmTokenSecretManagerInRM.getCurrentKey().getKeyId());
  
  
  RMContainerTokenSecretManager containerTokenSecretManager =
      yarnCluster.getResourceManager().getRMContext().
          getContainerTokenSecretManager();
  
  Resource r = Resource.newInstance(1230, 2, 2);
  
  Token containerToken = 
      containerTokenSecretManager.createContainerToken(
          cId, nodeId, user, r, Priority.newInstance(0), 0);
  
  ContainerTokenIdentifier containerTokenIdentifier = 
      getContainerTokenIdentifierFromToken(containerToken);
  
  // Verify new compatible version ContainerTokenIdentifier can work successfully.
  ContainerTokenIdentifierForTest newVersionTokenIdentifier = 
      new ContainerTokenIdentifierForTest(containerTokenIdentifier, "message");
  byte[] password = 
      containerTokenSecretManager.createPassword(newVersionTokenIdentifier);
  
  Token newContainerToken = BuilderUtils.newContainerToken(
      nodeId, password, newVersionTokenIdentifier);
  
  Token nmToken =
          nmTokenSecretManagerInRM.createNMToken(appAttemptId, nodeId, user);
  YarnRPC rpc = YarnRPC.create(conf);
  Assert.assertTrue(testStartContainer(rpc, appAttemptId, nodeId,
      newContainerToken, nmToken, false).isEmpty());
  
  // Creating a tampered Container Token
  RMContainerTokenSecretManager tamperedContainerTokenSecretManager =
      new RMContainerTokenSecretManager(conf);
  tamperedContainerTokenSecretManager.rollMasterKey();
  do {
    tamperedContainerTokenSecretManager.rollMasterKey();
    tamperedContainerTokenSecretManager.activateNextMasterKey();
  } while (containerTokenSecretManager.getCurrentKey().getKeyId()
      == tamperedContainerTokenSecretManager.getCurrentKey().getKeyId());
  
  ContainerId cId2 = ContainerId.newContainerId(appAttemptId, 1);
  // Creating modified containerToken
  Token containerToken2 =
      tamperedContainerTokenSecretManager.createContainerToken(cId2, nodeId,
          user, r, Priority.newInstance(0), 0);
  
  StringBuilder sb = new StringBuilder("Given Container ");
  sb.append(cId2);
  sb.append(" seems to have an illegally generated token.");
  Assert.assertTrue(testStartContainer(rpc, appAttemptId, nodeId,
      containerToken2, nmToken, true).contains(sb.toString()));
}

Example 4

Source File: TestCapacitySchedulerPlanFollower.java From hadoop with Apache License 2.0

4 votes

@Before
public void setUp() throws Exception {
  CapacityScheduler spyCs = new CapacityScheduler();
  cs = spy(spyCs);
  scheduler = cs;

  rmContext = TestUtils.getMockRMContext();
  spyRMContext = spy(rmContext);

  ConcurrentMap<ApplicationId, RMApp> spyApps =
      spy(new ConcurrentHashMap<ApplicationId, RMApp>());
  RMApp rmApp = mock(RMApp.class);
  when(rmApp.getRMAppAttempt((ApplicationAttemptId) Matchers.any()))
      .thenReturn(null);
  Mockito.doReturn(rmApp).when(spyApps).get((ApplicationId) Matchers.any());
  when(spyRMContext.getRMApps()).thenReturn(spyApps);
  when(spyRMContext.getScheduler()).thenReturn(scheduler);

  CapacitySchedulerConfiguration csConf =
      new CapacitySchedulerConfiguration();
  ReservationSystemTestUtil.setupQueueConfiguration(csConf);

  cs.setConf(csConf);

  csContext = mock(CapacitySchedulerContext.class);
  when(csContext.getConfiguration()).thenReturn(csConf);
  when(csContext.getConf()).thenReturn(csConf);
  when(csContext.getMinimumResourceCapability()).thenReturn(minAlloc);
  when(csContext.getMaximumResourceCapability()).thenReturn(maxAlloc);
  when(csContext.getClusterResource()).thenReturn(
      Resources.createResource(100 * 16 * GB, 100 * 32, 100 * 32));
  when(scheduler.getClusterResource()).thenReturn(
      Resources.createResource(125 * GB, 125, 125));
  when(csContext.getResourceCalculator()).thenReturn(
      new DefaultResourceCalculator());
  RMContainerTokenSecretManager containerTokenSecretManager =
      new RMContainerTokenSecretManager(csConf);
  containerTokenSecretManager.rollMasterKey();
  when(csContext.getContainerTokenSecretManager()).thenReturn(
      containerTokenSecretManager);

  cs.setRMContext(spyRMContext);
  cs.init(csConf);
  cs.start();

  setupPlanFollower();
}

Example 5

Source File: TestFifoScheduler.java From hadoop with Apache License 2.0

4 votes

@Test(timeout=2000)
public void testNodeLocalAssignment() throws Exception {
  AsyncDispatcher dispatcher = new InlineDispatcher();
  Configuration conf = new Configuration();
  RMContainerTokenSecretManager containerTokenSecretManager =
      new RMContainerTokenSecretManager(conf);
  containerTokenSecretManager.rollMasterKey();
  NMTokenSecretManagerInRM nmTokenSecretManager =
      new NMTokenSecretManagerInRM(conf);
  nmTokenSecretManager.rollMasterKey();
  RMApplicationHistoryWriter writer = mock(RMApplicationHistoryWriter.class);
  
  FifoScheduler scheduler = new FifoScheduler();
  RMContext rmContext = new RMContextImpl(dispatcher, null, null, null, null,
      null, containerTokenSecretManager, nmTokenSecretManager, null, writer,
      scheduler);
  ((RMContextImpl) rmContext).setSystemMetricsPublisher(
      mock(SystemMetricsPublisher.class));

  scheduler.setRMContext(rmContext);
  scheduler.init(conf);
  scheduler.start();
  scheduler.reinitialize(new Configuration(), rmContext);

  RMNode node0 = MockNodes.newNodeInfo(1,
      Resources.createResource(1024 * 64), 1, "127.0.0.1");
  NodeAddedSchedulerEvent nodeEvent1 = new NodeAddedSchedulerEvent(node0);
  scheduler.handle(nodeEvent1);

  int _appId = 1;
  int _appAttemptId = 1;
  ApplicationAttemptId appAttemptId = createAppAttemptId(_appId,
      _appAttemptId);

  createMockRMApp(appAttemptId, rmContext);

  AppAddedSchedulerEvent appEvent =
      new AppAddedSchedulerEvent(appAttemptId.getApplicationId(), "queue1",
          "user1");
  scheduler.handle(appEvent);
  AppAttemptAddedSchedulerEvent attemptEvent =
      new AppAttemptAddedSchedulerEvent(appAttemptId, false);
  scheduler.handle(attemptEvent);

  int memory = 64;
  int nConts = 3;
  int priority = 20;

  List<ResourceRequest> ask = new ArrayList<ResourceRequest>();
  ResourceRequest nodeLocal = createResourceRequest(memory,
      node0.getHostName(), priority, nConts);
  ResourceRequest rackLocal = createResourceRequest(memory,
      node0.getRackName(), priority, nConts);
  ResourceRequest any = createResourceRequest(memory, ResourceRequest.ANY, priority,
      nConts);
  ask.add(nodeLocal);
  ask.add(rackLocal);
  ask.add(any);
  scheduler.allocate(appAttemptId, ask, new ArrayList<ContainerId>(), null, null);

  NodeUpdateSchedulerEvent node0Update = new NodeUpdateSchedulerEvent(node0);

  // Before the node update event, there are 3 local requests outstanding
  Assert.assertEquals(3, nodeLocal.getNumContainers());

  scheduler.handle(node0Update);

  // After the node update event, check that there are no more local requests
  // outstanding
  Assert.assertEquals(0, nodeLocal.getNumContainers());
  //Also check that the containers were scheduled
  SchedulerAppReport info = scheduler.getSchedulerAppInfo(appAttemptId);
  Assert.assertEquals(3, info.getLiveContainers().size());
  scheduler.stop();
}

Example 6

Source File: TestFifoScheduler.java From hadoop with Apache License 2.0

4 votes

@Test(timeout=2000)
public void testUpdateResourceOnNode() throws Exception {
  AsyncDispatcher dispatcher = new InlineDispatcher();
  Configuration conf = new Configuration();
  RMContainerTokenSecretManager containerTokenSecretManager =
      new RMContainerTokenSecretManager(conf);
  containerTokenSecretManager.rollMasterKey();
  NMTokenSecretManagerInRM nmTokenSecretManager =
      new NMTokenSecretManagerInRM(conf);
  nmTokenSecretManager.rollMasterKey();
  RMApplicationHistoryWriter writer = mock(RMApplicationHistoryWriter.class);
  
  FifoScheduler scheduler = new FifoScheduler(){
    @SuppressWarnings("unused")
    public Map<NodeId, FiCaSchedulerNode> getNodes(){
      return nodes;
    }
  };
  RMContext rmContext = new RMContextImpl(dispatcher, null, null, null, null,
      null, containerTokenSecretManager, nmTokenSecretManager, null, writer,
      scheduler);
  ((RMContextImpl) rmContext).setSystemMetricsPublisher(
      mock(SystemMetricsPublisher.class));

  scheduler.setRMContext(rmContext);
  scheduler.init(conf);
  scheduler.start();
  scheduler.reinitialize(new Configuration(), rmContext);
  RMNode node0 = MockNodes.newNodeInfo(1,
      Resources.createResource(2048, 4, 4), 1, "127.0.0.1");
  NodeAddedSchedulerEvent nodeEvent1 = new NodeAddedSchedulerEvent(node0);
  scheduler.handle(nodeEvent1);
  
  Method method = scheduler.getClass().getDeclaredMethod("getNodes");
  @SuppressWarnings("unchecked")
  Map<NodeId, FiCaSchedulerNode> schedulerNodes = 
      (Map<NodeId, FiCaSchedulerNode>) method.invoke(scheduler);
  assertEquals(schedulerNodes.values().size(), 1);
  
  Resource newResource = Resources.createResource(1024, 4, 4);
  
  NodeResourceUpdateSchedulerEvent node0ResourceUpdate = new 
      NodeResourceUpdateSchedulerEvent(node0, ResourceOption.newInstance(
          newResource, RMNode.OVER_COMMIT_TIMEOUT_MILLIS_DEFAULT));
  scheduler.handle(node0ResourceUpdate);
  
  // SchedulerNode's total resource and available resource are changed.
  assertEquals(schedulerNodes.get(node0.getNodeID()).getTotalResource()
      .getMemory(), 1024);
  assertEquals(schedulerNodes.get(node0.getNodeID()).
      getAvailableResource().getMemory(), 1024);
  QueueInfo queueInfo = scheduler.getQueueInfo(null, false, false);
  Assert.assertEquals(0.0f, queueInfo.getCurrentCapacity(), 0.0f);
  
  int _appId = 1;
  int _appAttemptId = 1;
  ApplicationAttemptId appAttemptId = createAppAttemptId(_appId,
      _appAttemptId);
  createMockRMApp(appAttemptId, rmContext);

  AppAddedSchedulerEvent appEvent =
      new AppAddedSchedulerEvent(appAttemptId.getApplicationId(), "queue1",
        "user1");
  scheduler.handle(appEvent);
  AppAttemptAddedSchedulerEvent attemptEvent =
      new AppAttemptAddedSchedulerEvent(appAttemptId, false);
  scheduler.handle(attemptEvent);

  int memory = 1024;
  int priority = 1;

  List<ResourceRequest> ask = new ArrayList<ResourceRequest>();
  ResourceRequest nodeLocal = createResourceRequest(memory,
      node0.getHostName(), priority, 1);
  ResourceRequest rackLocal = createResourceRequest(memory,
      node0.getRackName(), priority, 1);
  ResourceRequest any = createResourceRequest(memory, ResourceRequest.ANY, priority,
      1);
  ask.add(nodeLocal);
  ask.add(rackLocal);
  ask.add(any);
  scheduler.allocate(appAttemptId, ask, new ArrayList<ContainerId>(), null, null);

  // Before the node update event, there are one local request
  Assert.assertEquals(1, nodeLocal.getNumContainers());

  NodeUpdateSchedulerEvent node0Update = new NodeUpdateSchedulerEvent(node0);
  // Now schedule.
  scheduler.handle(node0Update);

  // After the node update event, check no local request
  Assert.assertEquals(0, nodeLocal.getNumContainers());
  // Also check that one container was scheduled
  SchedulerAppReport info = scheduler.getSchedulerAppInfo(appAttemptId);
  Assert.assertEquals(1, info.getLiveContainers().size());
  // And check the default Queue now is full.
  queueInfo = scheduler.getQueueInfo(null, false, false);
  Assert.assertEquals(1.0f, queueInfo.getCurrentCapacity(), 0.0f);
}

Example 7

Source File: TestReservations.java From hadoop with Apache License 2.0

4 votes

private void setup(CapacitySchedulerConfiguration csConf,
    boolean addUserLimits) throws Exception {

  csConf.setBoolean("yarn.scheduler.capacity.user-metrics.enable", true);
  final String newRoot = "root" + System.currentTimeMillis();
  // final String newRoot = "root";

  setupQueueConfiguration(csConf, newRoot, addUserLimits);
  YarnConfiguration conf = new YarnConfiguration();
  cs.setConf(conf);

  csContext = mock(CapacitySchedulerContext.class);
  when(csContext.getConfiguration()).thenReturn(csConf);
  when(csContext.getConf()).thenReturn(conf);
  when(csContext.getMinimumResourceCapability()).thenReturn(
      Resources.createResource(GB, 1));
  when(csContext.getMaximumResourceCapability()).thenReturn(
      Resources.createResource(16 * GB, 12));
  when(csContext.getClusterResource()).thenReturn(
      Resources.createResource(100 * 16 * GB, 100 * 12));
  when(csContext.getApplicationComparator()).thenReturn(
      CapacityScheduler.applicationComparator);
  when(csContext.getQueueComparator()).thenReturn(
      CapacityScheduler.queueComparator);
  when(csContext.getResourceCalculator()).thenReturn(resourceCalculator);
  when(csContext.getRMContext()).thenReturn(rmContext);
  RMContainerTokenSecretManager containerTokenSecretManager = new RMContainerTokenSecretManager(
      conf);
  containerTokenSecretManager.rollMasterKey();
  when(csContext.getContainerTokenSecretManager()).thenReturn(
      containerTokenSecretManager);

  root = CapacityScheduler.parseQueue(csContext, csConf, null,
      CapacitySchedulerConfiguration.ROOT, queues, queues, TestUtils.spyHook);

  spyRMContext = spy(rmContext);
  when(spyRMContext.getScheduler()).thenReturn(cs);
  
  cs.setRMContext(spyRMContext);
  cs.init(csConf);
  cs.start();
}

Example 8

Source File: TestApplicationLimits.java From hadoop with Apache License 2.0

4 votes

@Before
public void setUp() throws IOException {
  CapacitySchedulerConfiguration csConf = 
      new CapacitySchedulerConfiguration();
  YarnConfiguration conf = new YarnConfiguration();
  setupQueueConfiguration(csConf);
  
  rmContext = TestUtils.getMockRMContext();


  CapacitySchedulerContext csContext = mock(CapacitySchedulerContext.class);
  when(csContext.getConfiguration()).thenReturn(csConf);
  when(csContext.getConf()).thenReturn(conf);
  when(csContext.getMinimumResourceCapability()).
      thenReturn(Resources.createResource(GB, 1));
  when(csContext.getMaximumResourceCapability()).
      thenReturn(Resources.createResource(16*GB, 32));
  when(csContext.getClusterResource()).
      thenReturn(Resources.createResource(10 * 16 * GB, 10 * 32));
  when(csContext.getApplicationComparator()).
      thenReturn(CapacityScheduler.applicationComparator);
  when(csContext.getQueueComparator()).
      thenReturn(CapacityScheduler.queueComparator);
  when(csContext.getResourceCalculator()).
      thenReturn(resourceCalculator);
  when(csContext.getRMContext()).thenReturn(rmContext);
  
  RMContainerTokenSecretManager containerTokenSecretManager =
      new RMContainerTokenSecretManager(conf);
  containerTokenSecretManager.rollMasterKey();
  when(csContext.getContainerTokenSecretManager()).thenReturn(
      containerTokenSecretManager);

  Map<String, CSQueue> queues = new HashMap<String, CSQueue>();
  CSQueue root = 
      CapacityScheduler.parseQueue(csContext, csConf, null, "root", 
          queues, queues, 
          TestUtils.spyHook);

  
  queue = spy(new LeafQueue(csContext, A, root, null));

  // Stub out ACL checks
  doReturn(true).
      when(queue).hasAccess(any(QueueACL.class), 
                            any(UserGroupInformation.class));
  
  // Some default values
  doReturn(100).when(queue).getMaxApplications();
  doReturn(25).when(queue).getMaxApplicationsPerUser();
}

Example 9

Source File: TestLeafQueue.java From hadoop with Apache License 2.0

4 votes

@Before
public void setUp() throws Exception {
  CapacityScheduler spyCs = new CapacityScheduler();
  cs = spy(spyCs);
  rmContext = TestUtils.getMockRMContext();
  spyRMContext = spy(rmContext);

  ConcurrentMap<ApplicationId, RMApp> spyApps = 
      spy(new ConcurrentHashMap<ApplicationId, RMApp>());
  RMApp rmApp = mock(RMApp.class);
  when(rmApp.getRMAppAttempt((ApplicationAttemptId)Matchers.any())).thenReturn(null);
  amResourceRequest = mock(ResourceRequest.class);
  when(amResourceRequest.getCapability()).thenReturn(
    Resources.createResource(0, 0));
  when(rmApp.getAMResourceRequest()).thenReturn(amResourceRequest);
  Mockito.doReturn(rmApp).when(spyApps).get((ApplicationId)Matchers.any());
  when(spyRMContext.getRMApps()).thenReturn(spyApps);
  
  csConf = 
      new CapacitySchedulerConfiguration();
  csConf.setBoolean("yarn.scheduler.capacity.user-metrics.enable", true);
  final String newRoot = "root" + System.currentTimeMillis();
  setupQueueConfiguration(csConf, newRoot);
  YarnConfiguration conf = new YarnConfiguration();
  cs.setConf(conf);

  csContext = mock(CapacitySchedulerContext.class);
  when(csContext.getConfiguration()).thenReturn(csConf);
  when(csContext.getConf()).thenReturn(conf);
  when(csContext.getMinimumResourceCapability()).
      thenReturn(Resources.createResource(GB, 1));
  when(csContext.getMaximumResourceCapability()).
      thenReturn(Resources.createResource(16*GB, 32));
  when(csContext.getClusterResource()).
      thenReturn(Resources.createResource(100 * 16 * GB, 100 * 32));
  when(csContext.getApplicationComparator()).
  thenReturn(CapacityScheduler.applicationComparator);
  when(csContext.getQueueComparator()).
      thenReturn(CapacityScheduler.queueComparator);
  when(csContext.getResourceCalculator()).
      thenReturn(resourceCalculator);
  when(csContext.getRMContext()).thenReturn(rmContext);
  RMContainerTokenSecretManager containerTokenSecretManager =
      new RMContainerTokenSecretManager(conf);
  containerTokenSecretManager.rollMasterKey();
  when(csContext.getContainerTokenSecretManager()).thenReturn(
      containerTokenSecretManager);

  root = 
      CapacityScheduler.parseQueue(csContext, csConf, null, 
          CapacitySchedulerConfiguration.ROOT, 
          queues, queues, 
          TestUtils.spyHook);

  cs.setRMContext(spyRMContext);
  cs.init(csConf);
  cs.start();
}

Example 10

Source File: TestContainerManagerSecurity.java From big-c with Apache License 2.0

4 votes

/**
 * This tests a malice user getting a proper token but then messing with it by
 * tampering with containerID/Resource etc.. His/her containers should be
 * rejected.
 * 
 * @throws IOException
 * @throws InterruptedException
 * @throws YarnException
 */
private void testContainerToken(Configuration conf) throws IOException,
    InterruptedException, YarnException {

  LOG.info("Running test for malice user");
  /*
   * We need to check for containerToken (authorization).
   * Here we will be assuming that we have valid NMToken  
   * 1) ContainerToken used is expired.
   * 2) ContainerToken is tampered (resource is modified).
   */
  NMTokenSecretManagerInRM nmTokenSecretManagerInRM =
      yarnCluster.getResourceManager().getRMContext()
        .getNMTokenSecretManager();
  ApplicationId appId = ApplicationId.newInstance(1, 1);
  ApplicationAttemptId appAttemptId =
      ApplicationAttemptId.newInstance(appId, 0);
  ContainerId cId = ContainerId.newContainerId(appAttemptId, 0);
  NodeManager nm = yarnCluster.getNodeManager(0);
  NMTokenSecretManagerInNM nmTokenSecretManagerInNM =
      nm.getNMContext().getNMTokenSecretManager();
  String user = "test";
  
  waitForNMToReceiveNMTokenKey(nmTokenSecretManagerInNM, nm);

  NodeId nodeId = nm.getNMContext().getNodeId();
  
  // Both id should be equal.
  Assert.assertEquals(nmTokenSecretManagerInNM.getCurrentKey().getKeyId(),
      nmTokenSecretManagerInRM.getCurrentKey().getKeyId());
  
  
  RMContainerTokenSecretManager containerTokenSecretManager =
      yarnCluster.getResourceManager().getRMContext().
          getContainerTokenSecretManager();
  
  Resource r = Resource.newInstance(1230, 2);
  
  Token containerToken = 
      containerTokenSecretManager.createContainerToken(
          cId, nodeId, user, r, Priority.newInstance(0), 0);
  
  ContainerTokenIdentifier containerTokenIdentifier = 
      getContainerTokenIdentifierFromToken(containerToken);
  
  // Verify new compatible version ContainerTokenIdentifier can work successfully.
  ContainerTokenIdentifierForTest newVersionTokenIdentifier = 
      new ContainerTokenIdentifierForTest(containerTokenIdentifier, "message");
  byte[] password = 
      containerTokenSecretManager.createPassword(newVersionTokenIdentifier);
  
  Token newContainerToken = BuilderUtils.newContainerToken(
      nodeId, password, newVersionTokenIdentifier);
  
  Token nmToken =
          nmTokenSecretManagerInRM.createNMToken(appAttemptId, nodeId, user);
  YarnRPC rpc = YarnRPC.create(conf);
  Assert.assertTrue(testStartContainer(rpc, appAttemptId, nodeId,
      newContainerToken, nmToken, false).isEmpty());
  
  // Creating a tampered Container Token
  RMContainerTokenSecretManager tamperedContainerTokenSecretManager =
      new RMContainerTokenSecretManager(conf);
  tamperedContainerTokenSecretManager.rollMasterKey();
  do {
    tamperedContainerTokenSecretManager.rollMasterKey();
    tamperedContainerTokenSecretManager.activateNextMasterKey();
  } while (containerTokenSecretManager.getCurrentKey().getKeyId()
      == tamperedContainerTokenSecretManager.getCurrentKey().getKeyId());
  
  ContainerId cId2 = ContainerId.newContainerId(appAttemptId, 1);
  // Creating modified containerToken
  Token containerToken2 =
      tamperedContainerTokenSecretManager.createContainerToken(cId2, nodeId,
          user, r, Priority.newInstance(0), 0);
  
  StringBuilder sb = new StringBuilder("Given Container ");
  sb.append(cId2);
  sb.append(" seems to have an illegally generated token.");
  Assert.assertTrue(testStartContainer(rpc, appAttemptId, nodeId,
      containerToken2, nmToken, true).contains(sb.toString()));
}

Example 11

Source File: TestCapacitySchedulerPlanFollower.java From big-c with Apache License 2.0

4 votes

@Before
public void setUp() throws Exception {
  CapacityScheduler spyCs = new CapacityScheduler();
  cs = spy(spyCs);
  scheduler = cs;

  rmContext = TestUtils.getMockRMContext();
  spyRMContext = spy(rmContext);

  ConcurrentMap<ApplicationId, RMApp> spyApps =
      spy(new ConcurrentHashMap<ApplicationId, RMApp>());
  RMApp rmApp = mock(RMApp.class);
  when(rmApp.getRMAppAttempt((ApplicationAttemptId) Matchers.any()))
      .thenReturn(null);
  Mockito.doReturn(rmApp).when(spyApps).get((ApplicationId) Matchers.any());
  when(spyRMContext.getRMApps()).thenReturn(spyApps);
  when(spyRMContext.getScheduler()).thenReturn(scheduler);

  CapacitySchedulerConfiguration csConf =
      new CapacitySchedulerConfiguration();
  ReservationSystemTestUtil.setupQueueConfiguration(csConf);

  cs.setConf(csConf);

  csContext = mock(CapacitySchedulerContext.class);
  when(csContext.getConfiguration()).thenReturn(csConf);
  when(csContext.getConf()).thenReturn(csConf);
  when(csContext.getMinimumResourceCapability()).thenReturn(minAlloc);
  when(csContext.getMaximumResourceCapability()).thenReturn(maxAlloc);
  when(csContext.getClusterResource()).thenReturn(
      Resources.createResource(100 * 16 * GB, 100 * 32));
  when(scheduler.getClusterResource()).thenReturn(
      Resources.createResource(125 * GB, 125));
  when(csContext.getResourceCalculator()).thenReturn(
      new DefaultResourceCalculator());
  RMContainerTokenSecretManager containerTokenSecretManager =
      new RMContainerTokenSecretManager(csConf);
  containerTokenSecretManager.rollMasterKey();
  when(csContext.getContainerTokenSecretManager()).thenReturn(
      containerTokenSecretManager);

  cs.setRMContext(spyRMContext);
  cs.init(csConf);
  cs.start();

  setupPlanFollower();
}

Example 12

Source File: TestFifoScheduler.java From big-c with Apache License 2.0

4 votes

@Test(timeout=2000)
public void testNodeLocalAssignment() throws Exception {
  AsyncDispatcher dispatcher = new InlineDispatcher();
  Configuration conf = new Configuration();
  RMContainerTokenSecretManager containerTokenSecretManager =
      new RMContainerTokenSecretManager(conf);
  containerTokenSecretManager.rollMasterKey();
  NMTokenSecretManagerInRM nmTokenSecretManager =
      new NMTokenSecretManagerInRM(conf);
  nmTokenSecretManager.rollMasterKey();
  RMApplicationHistoryWriter writer = mock(RMApplicationHistoryWriter.class);
  
  FifoScheduler scheduler = new FifoScheduler();
  RMContext rmContext = new RMContextImpl(dispatcher, null, null, null, null,
      null, containerTokenSecretManager, nmTokenSecretManager, null, writer,
      scheduler);
  ((RMContextImpl) rmContext).setSystemMetricsPublisher(
      mock(SystemMetricsPublisher.class));

  scheduler.setRMContext(rmContext);
  scheduler.init(conf);
  scheduler.start();
  scheduler.reinitialize(new Configuration(), rmContext);

  RMNode node0 = MockNodes.newNodeInfo(1,
      Resources.createResource(1024 * 64), 1, "127.0.0.1");
  NodeAddedSchedulerEvent nodeEvent1 = new NodeAddedSchedulerEvent(node0);
  scheduler.handle(nodeEvent1);

  int _appId = 1;
  int _appAttemptId = 1;
  ApplicationAttemptId appAttemptId = createAppAttemptId(_appId,
      _appAttemptId);

  createMockRMApp(appAttemptId, rmContext);

  AppAddedSchedulerEvent appEvent =
      new AppAddedSchedulerEvent(appAttemptId.getApplicationId(), "queue1",
          "user1");
  scheduler.handle(appEvent);
  AppAttemptAddedSchedulerEvent attemptEvent =
      new AppAttemptAddedSchedulerEvent(appAttemptId, false);
  scheduler.handle(attemptEvent);

  int memory = 64;
  int nConts = 3;
  int priority = 20;

  List<ResourceRequest> ask = new ArrayList<ResourceRequest>();
  ResourceRequest nodeLocal = createResourceRequest(memory,
      node0.getHostName(), priority, nConts);
  ResourceRequest rackLocal = createResourceRequest(memory,
      node0.getRackName(), priority, nConts);
  ResourceRequest any = createResourceRequest(memory, ResourceRequest.ANY, priority,
      nConts);
  ask.add(nodeLocal);
  ask.add(rackLocal);
  ask.add(any);
  scheduler.allocate(appAttemptId, ask, new ArrayList<ContainerId>(), null, null);

  NodeUpdateSchedulerEvent node0Update = new NodeUpdateSchedulerEvent(node0);

  // Before the node update event, there are 3 local requests outstanding
  Assert.assertEquals(3, nodeLocal.getNumContainers());

  scheduler.handle(node0Update);

  // After the node update event, check that there are no more local requests
  // outstanding
  Assert.assertEquals(0, nodeLocal.getNumContainers());
  //Also check that the containers were scheduled
  SchedulerAppReport info = scheduler.getSchedulerAppInfo(appAttemptId);
  Assert.assertEquals(3, info.getLiveContainers().size());
  scheduler.stop();
}

Example 13

Source File: TestFifoScheduler.java From big-c with Apache License 2.0

4 votes

@Test(timeout=2000)
public void testUpdateResourceOnNode() throws Exception {
  AsyncDispatcher dispatcher = new InlineDispatcher();
  Configuration conf = new Configuration();
  RMContainerTokenSecretManager containerTokenSecretManager =
      new RMContainerTokenSecretManager(conf);
  containerTokenSecretManager.rollMasterKey();
  NMTokenSecretManagerInRM nmTokenSecretManager =
      new NMTokenSecretManagerInRM(conf);
  nmTokenSecretManager.rollMasterKey();
  RMApplicationHistoryWriter writer = mock(RMApplicationHistoryWriter.class);
  
  FifoScheduler scheduler = new FifoScheduler(){
    @SuppressWarnings("unused")
    public Map<NodeId, FiCaSchedulerNode> getNodes(){
      return nodes;
    }
  };
  RMContext rmContext = new RMContextImpl(dispatcher, null, null, null, null,
      null, containerTokenSecretManager, nmTokenSecretManager, null, writer,
      scheduler);
  ((RMContextImpl) rmContext).setSystemMetricsPublisher(
      mock(SystemMetricsPublisher.class));

  scheduler.setRMContext(rmContext);
  scheduler.init(conf);
  scheduler.start();
  scheduler.reinitialize(new Configuration(), rmContext);
  RMNode node0 = MockNodes.newNodeInfo(1,
      Resources.createResource(2048, 4), 1, "127.0.0.1");
  NodeAddedSchedulerEvent nodeEvent1 = new NodeAddedSchedulerEvent(node0);
  scheduler.handle(nodeEvent1);
  
  Method method = scheduler.getClass().getDeclaredMethod("getNodes");
  @SuppressWarnings("unchecked")
  Map<NodeId, FiCaSchedulerNode> schedulerNodes = 
      (Map<NodeId, FiCaSchedulerNode>) method.invoke(scheduler);
  assertEquals(schedulerNodes.values().size(), 1);
  
  Resource newResource = Resources.createResource(1024, 4);
  
  NodeResourceUpdateSchedulerEvent node0ResourceUpdate = new 
      NodeResourceUpdateSchedulerEvent(node0, ResourceOption.newInstance(
          newResource, RMNode.OVER_COMMIT_TIMEOUT_MILLIS_DEFAULT));
  scheduler.handle(node0ResourceUpdate);
  
  // SchedulerNode's total resource and available resource are changed.
  assertEquals(schedulerNodes.get(node0.getNodeID()).getTotalResource()
      .getMemory(), 1024);
  assertEquals(schedulerNodes.get(node0.getNodeID()).
      getAvailableResource().getMemory(), 1024);
  QueueInfo queueInfo = scheduler.getQueueInfo(null, false, false);
  Assert.assertEquals(0.0f, queueInfo.getCurrentCapacity(), 0.0f);
  
  int _appId = 1;
  int _appAttemptId = 1;
  ApplicationAttemptId appAttemptId = createAppAttemptId(_appId,
      _appAttemptId);
  createMockRMApp(appAttemptId, rmContext);

  AppAddedSchedulerEvent appEvent =
      new AppAddedSchedulerEvent(appAttemptId.getApplicationId(), "queue1",
        "user1");
  scheduler.handle(appEvent);
  AppAttemptAddedSchedulerEvent attemptEvent =
      new AppAttemptAddedSchedulerEvent(appAttemptId, false);
  scheduler.handle(attemptEvent);

  int memory = 1024;
  int priority = 1;

  List<ResourceRequest> ask = new ArrayList<ResourceRequest>();
  ResourceRequest nodeLocal = createResourceRequest(memory,
      node0.getHostName(), priority, 1);
  ResourceRequest rackLocal = createResourceRequest(memory,
      node0.getRackName(), priority, 1);
  ResourceRequest any = createResourceRequest(memory, ResourceRequest.ANY, priority,
      1);
  ask.add(nodeLocal);
  ask.add(rackLocal);
  ask.add(any);
  scheduler.allocate(appAttemptId, ask, new ArrayList<ContainerId>(), null, null);

  // Before the node update event, there are one local request
  Assert.assertEquals(1, nodeLocal.getNumContainers());

  NodeUpdateSchedulerEvent node0Update = new NodeUpdateSchedulerEvent(node0);
  // Now schedule.
  scheduler.handle(node0Update);

  // After the node update event, check no local request
  Assert.assertEquals(0, nodeLocal.getNumContainers());
  // Also check that one container was scheduled
  SchedulerAppReport info = scheduler.getSchedulerAppInfo(appAttemptId);
  Assert.assertEquals(1, info.getLiveContainers().size());
  // And check the default Queue now is full.
  queueInfo = scheduler.getQueueInfo(null, false, false);
  Assert.assertEquals(1.0f, queueInfo.getCurrentCapacity(), 0.0f);
}

Example 14

Source File: TestReservations.java From big-c with Apache License 2.0

4 votes

private void setup(CapacitySchedulerConfiguration csConf) throws Exception {

    csConf.setBoolean("yarn.scheduler.capacity.user-metrics.enable", true);
    final String newRoot = "root" + System.currentTimeMillis();
    // final String newRoot = "root";

    setupQueueConfiguration(csConf, newRoot);
    YarnConfiguration conf = new YarnConfiguration();
    cs.setConf(conf);

    csContext = mock(CapacitySchedulerContext.class);
    when(csContext.getConfiguration()).thenReturn(csConf);
    when(csContext.getConf()).thenReturn(conf);
    when(csContext.getMinimumResourceCapability()).thenReturn(
        Resources.createResource(GB, 1));
    when(csContext.getMaximumResourceCapability()).thenReturn(
        Resources.createResource(16 * GB, 12));
    when(csContext.getClusterResource()).thenReturn(
        Resources.createResource(100 * 16 * GB, 100 * 12));
    when(csContext.getApplicationComparator()).thenReturn(
        CapacityScheduler.applicationComparator);
    when(csContext.getQueueComparator()).thenReturn(
        CapacityScheduler.queueComparator);
    when(csContext.getResourceCalculator()).thenReturn(resourceCalculator);
    when(csContext.getRMContext()).thenReturn(rmContext);
    RMContainerTokenSecretManager containerTokenSecretManager = new RMContainerTokenSecretManager(
        conf);
    containerTokenSecretManager.rollMasterKey();
    when(csContext.getContainerTokenSecretManager()).thenReturn(
        containerTokenSecretManager);

    root = CapacityScheduler.parseQueue(csContext, csConf, null,
        CapacitySchedulerConfiguration.ROOT, queues, queues, TestUtils.spyHook);

    spyRMContext = spy(rmContext);
    when(spyRMContext.getScheduler()).thenReturn(cs);
    
    cs.setRMContext(spyRMContext);
    cs.init(csConf);
    cs.start();
  }

Example 15

Source File: TestApplicationLimits.java From big-c with Apache License 2.0

4 votes

@Before
public void setUp() throws IOException {
  CapacitySchedulerConfiguration csConf = 
      new CapacitySchedulerConfiguration();
  YarnConfiguration conf = new YarnConfiguration();
  setupQueueConfiguration(csConf);
  
  rmContext = TestUtils.getMockRMContext();


  CapacitySchedulerContext csContext = mock(CapacitySchedulerContext.class);
  when(csContext.getConfiguration()).thenReturn(csConf);
  when(csContext.getConf()).thenReturn(conf);
  when(csContext.getMinimumResourceCapability()).
      thenReturn(Resources.createResource(GB, 1));
  when(csContext.getMaximumResourceCapability()).
      thenReturn(Resources.createResource(16*GB, 32));
  when(csContext.getClusterResource()).
      thenReturn(Resources.createResource(10 * 16 * GB, 10 * 32));
  when(csContext.getApplicationComparator()).
      thenReturn(CapacityScheduler.applicationComparator);
  when(csContext.getQueueComparator()).
      thenReturn(CapacityScheduler.queueComparator);
  when(csContext.getResourceCalculator()).
      thenReturn(resourceCalculator);
  when(csContext.getRMContext()).thenReturn(rmContext);
  
  RMContainerTokenSecretManager containerTokenSecretManager =
      new RMContainerTokenSecretManager(conf);
  containerTokenSecretManager.rollMasterKey();
  when(csContext.getContainerTokenSecretManager()).thenReturn(
      containerTokenSecretManager);

  Map<String, CSQueue> queues = new HashMap<String, CSQueue>();
  CSQueue root = 
      CapacityScheduler.parseQueue(csContext, csConf, null, "root", 
          queues, queues, 
          TestUtils.spyHook);

  
  queue = spy(new LeafQueue(csContext, A, root, null));

  // Stub out ACL checks
  doReturn(true).
      when(queue).hasAccess(any(QueueACL.class), 
                            any(UserGroupInformation.class));
  
  // Some default values
  doReturn(100).when(queue).getMaxApplications();
  doReturn(25).when(queue).getMaxApplicationsPerUser();
}

Example 16

Source File: TestLeafQueue.java From big-c with Apache License 2.0

4 votes

@Before
public void setUp() throws Exception {
  CapacityScheduler spyCs = new CapacityScheduler();
  cs = spy(spyCs);
  rmContext = TestUtils.getMockRMContext();
  spyRMContext = spy(rmContext);

  ConcurrentMap<ApplicationId, RMApp> spyApps = 
      spy(new ConcurrentHashMap<ApplicationId, RMApp>());
  RMApp rmApp = mock(RMApp.class);
  when(rmApp.getRMAppAttempt((ApplicationAttemptId)Matchers.any())).thenReturn(null);
  amResourceRequest = mock(ResourceRequest.class);
  when(amResourceRequest.getCapability()).thenReturn(
    Resources.createResource(0, 0));
  when(rmApp.getAMResourceRequest()).thenReturn(amResourceRequest);
  Mockito.doReturn(rmApp).when(spyApps).get((ApplicationId)Matchers.any());
  when(spyRMContext.getRMApps()).thenReturn(spyApps);
  
  csConf = 
      new CapacitySchedulerConfiguration();
  csConf.setBoolean("yarn.scheduler.capacity.user-metrics.enable", true);
  final String newRoot = "root" + System.currentTimeMillis();
  setupQueueConfiguration(csConf, newRoot);
  YarnConfiguration conf = new YarnConfiguration();
  cs.setConf(conf);

  csContext = mock(CapacitySchedulerContext.class);
  when(csContext.getConfiguration()).thenReturn(csConf);
  when(csContext.getConf()).thenReturn(conf);
  when(csContext.getMinimumResourceCapability()).
      thenReturn(Resources.createResource(GB, 1));
  when(csContext.getMaximumResourceCapability()).
      thenReturn(Resources.createResource(16*GB, 32));
  when(csContext.getClusterResource()).
      thenReturn(Resources.createResource(100 * 16 * GB, 100 * 32));
  when(csContext.getApplicationComparator()).
  thenReturn(CapacityScheduler.applicationComparator);
  when(csContext.getQueueComparator()).
      thenReturn(CapacityScheduler.queueComparator);
  when(csContext.getResourceCalculator()).
      thenReturn(resourceCalculator);
  when(csContext.getRMContext()).thenReturn(rmContext);
  RMContainerTokenSecretManager containerTokenSecretManager =
      new RMContainerTokenSecretManager(conf);
  containerTokenSecretManager.rollMasterKey();
  when(csContext.getContainerTokenSecretManager()).thenReturn(
      containerTokenSecretManager);

  root = 
      CapacityScheduler.parseQueue(csContext, csConf, null, 
          CapacitySchedulerConfiguration.ROOT, 
          queues, queues, 
          TestUtils.spyHook);

  cs.setRMContext(spyRMContext);
  cs.init(csConf);
  cs.start();
}