Java Code Examples for org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager#init()
The following examples show how to use
org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager#init() .
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: TestQueueParsing.java From big-c with Apache License 2.0 | 6 votes |
@Test public void testQueueParsingWhenLabelsNotExist() throws IOException { YarnConfiguration conf = new YarnConfiguration(); CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration(conf); setupQueueConfigurationWithLabels(csConf); CapacityScheduler capacityScheduler = new CapacityScheduler(); RMContextImpl rmContext = new RMContextImpl(null, null, null, null, null, null, new RMContainerTokenSecretManager(csConf), new NMTokenSecretManagerInRM(csConf), new ClientToAMTokenSecretManagerInRM(), null); RMNodeLabelsManager nodeLabelsManager = new NullRMNodeLabelsManager(); nodeLabelsManager.init(conf); nodeLabelsManager.start(); rmContext.setNodeLabelManager(nodeLabelsManager); capacityScheduler.setConf(csConf); capacityScheduler.setRMContext(rmContext); capacityScheduler.init(csConf); capacityScheduler.start(); ServiceOperations.stopQuietly(capacityScheduler); ServiceOperations.stopQuietly(nodeLabelsManager); }
Example 2
Source File: TestCapacityScheduler.java From hadoop with Apache License 2.0 | 6 votes |
@Before public void setUp() throws Exception { resourceManager = new ResourceManager() { @Override protected RMNodeLabelsManager createNodeLabelManager() { RMNodeLabelsManager mgr = new NullRMNodeLabelsManager(); mgr.init(getConfig()); return mgr; } }; CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration(); setupQueueConfiguration(csConf); YarnConfiguration conf = new YarnConfiguration(csConf); conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class, ResourceScheduler.class); resourceManager.init(conf); resourceManager.getRMContext().getContainerTokenSecretManager().rollMasterKey(); resourceManager.getRMContext().getNMTokenSecretManager().rollMasterKey(); ((AsyncDispatcher)resourceManager.getRMContext().getDispatcher()).start(); mockContext = mock(RMContext.class); when(mockContext.getConfigurationProvider()).thenReturn( new LocalConfigurationProvider()); }
Example 3
Source File: TestCapacityScheduler.java From big-c with Apache License 2.0 | 6 votes |
@Before public void setUp() throws Exception { resourceManager = new ResourceManager() { @Override protected RMNodeLabelsManager createNodeLabelManager() { RMNodeLabelsManager mgr = new NullRMNodeLabelsManager(); mgr.init(getConfig()); return mgr; } }; CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration(); setupQueueConfiguration(csConf); YarnConfiguration conf = new YarnConfiguration(csConf); conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class, ResourceScheduler.class); resourceManager.init(conf); resourceManager.getRMContext().getContainerTokenSecretManager().rollMasterKey(); resourceManager.getRMContext().getNMTokenSecretManager().rollMasterKey(); ((AsyncDispatcher)resourceManager.getRMContext().getDispatcher()).start(); mockContext = mock(RMContext.class); when(mockContext.getConfigurationProvider()).thenReturn( new LocalConfigurationProvider()); }
Example 4
Source File: TestQueueParsing.java From hadoop with Apache License 2.0 | 6 votes |
@Test public void testQueueParsingWhenLabelsNotExist() throws IOException { YarnConfiguration conf = new YarnConfiguration(); CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration(conf); setupQueueConfigurationWithLabels(csConf); CapacityScheduler capacityScheduler = new CapacityScheduler(); RMContextImpl rmContext = new RMContextImpl(null, null, null, null, null, null, new RMContainerTokenSecretManager(csConf), new NMTokenSecretManagerInRM(csConf), new ClientToAMTokenSecretManagerInRM(), null); RMNodeLabelsManager nodeLabelsManager = new NullRMNodeLabelsManager(); nodeLabelsManager.init(conf); nodeLabelsManager.start(); rmContext.setNodeLabelManager(nodeLabelsManager); capacityScheduler.setConf(csConf); capacityScheduler.setRMContext(rmContext); capacityScheduler.init(csConf); capacityScheduler.start(); ServiceOperations.stopQuietly(capacityScheduler); ServiceOperations.stopQuietly(nodeLabelsManager); }
Example 5
Source File: MockRM.java From big-c with Apache License 2.0 | 5 votes |
@Override protected RMNodeLabelsManager createNodeLabelManager() throws InstantiationException, IllegalAccessException { if (useNullRMNodeLabelsManager) { RMNodeLabelsManager mgr = new NullRMNodeLabelsManager(); mgr.init(getConfig()); return mgr; } else { return super.createNodeLabelManager(); } }
Example 6
Source File: TestQueueParsing.java From big-c with Apache License 2.0 | 5 votes |
@Test public void testSingleLevelQueueParsingWhenLabelsNotExistedInNodeLabelManager() throws IOException { YarnConfiguration conf = new YarnConfiguration(); CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration(conf); setupQueueConfigurationWithSingleLevel(csConf); CapacityScheduler capacityScheduler = new CapacityScheduler(); RMContextImpl rmContext = new RMContextImpl(null, null, null, null, null, null, new RMContainerTokenSecretManager(csConf), new NMTokenSecretManagerInRM(csConf), new ClientToAMTokenSecretManagerInRM(), null); RMNodeLabelsManager nodeLabelsManager = new NullRMNodeLabelsManager(); nodeLabelsManager.init(conf); nodeLabelsManager.start(); rmContext.setNodeLabelManager(nodeLabelsManager); capacityScheduler.setConf(csConf); capacityScheduler.setRMContext(rmContext); capacityScheduler.init(csConf); capacityScheduler.start(); ServiceOperations.stopQuietly(capacityScheduler); ServiceOperations.stopQuietly(nodeLabelsManager); }
Example 7
Source File: TestQueueParsing.java From big-c with Apache License 2.0 | 5 votes |
@Test public void testQueueParsingWhenLabelsInheritedNotExistedInNodeLabelManager() throws IOException { YarnConfiguration conf = new YarnConfiguration(); CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration(conf); setupQueueConfigurationWithLabelsInherit(csConf); CapacityScheduler capacityScheduler = new CapacityScheduler(); RMContextImpl rmContext = new RMContextImpl(null, null, null, null, null, null, new RMContainerTokenSecretManager(csConf), new NMTokenSecretManagerInRM(csConf), new ClientToAMTokenSecretManagerInRM(), null); RMNodeLabelsManager nodeLabelsManager = new NullRMNodeLabelsManager(); nodeLabelsManager.init(conf); nodeLabelsManager.start(); rmContext.setNodeLabelManager(nodeLabelsManager); capacityScheduler.setConf(csConf); capacityScheduler.setRMContext(rmContext); capacityScheduler.init(csConf); capacityScheduler.start(); ServiceOperations.stopQuietly(capacityScheduler); ServiceOperations.stopQuietly(nodeLabelsManager); }
Example 8
Source File: TestQueueParsing.java From big-c with Apache License 2.0 | 5 votes |
@Test public void testQueueParsingWhenLabelsNotExistedInNodeLabelManager() throws IOException { YarnConfiguration conf = new YarnConfiguration(); CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration(conf); setupQueueConfigurationWithLabels(csConf); CapacityScheduler capacityScheduler = new CapacityScheduler(); RMContextImpl rmContext = new RMContextImpl(null, null, null, null, null, null, new RMContainerTokenSecretManager(csConf), new NMTokenSecretManagerInRM(csConf), new ClientToAMTokenSecretManagerInRM(), null); RMNodeLabelsManager nodeLabelsManager = new NullRMNodeLabelsManager(); nodeLabelsManager.init(conf); nodeLabelsManager.start(); rmContext.setNodeLabelManager(nodeLabelsManager); capacityScheduler.setConf(csConf); capacityScheduler.setRMContext(rmContext); capacityScheduler.init(csConf); capacityScheduler.start(); ServiceOperations.stopQuietly(capacityScheduler); ServiceOperations.stopQuietly(nodeLabelsManager); }
Example 9
Source File: TestSchedulerUtils.java From big-c with Apache License 2.0 | 5 votes |
private static RMContext getMockRMContext() { RMContext rmContext = mock(RMContext.class); RMNodeLabelsManager nlm = new NullRMNodeLabelsManager(); nlm.init(new Configuration(false)); when(rmContext.getNodeLabelManager()).thenReturn(nlm); return rmContext; }
Example 10
Source File: MockRM.java From hadoop with Apache License 2.0 | 5 votes |
@Override protected RMNodeLabelsManager createNodeLabelManager() throws InstantiationException, IllegalAccessException { if (useNullRMNodeLabelsManager) { RMNodeLabelsManager mgr = new NullRMNodeLabelsManager(); mgr.init(getConfig()); return mgr; } else { return super.createNodeLabelManager(); } }
Example 11
Source File: TestQueueParsing.java From hadoop with Apache License 2.0 | 5 votes |
@Test public void testSingleLevelQueueParsingWhenLabelsNotExistedInNodeLabelManager() throws IOException { YarnConfiguration conf = new YarnConfiguration(); CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration(conf); setupQueueConfigurationWithSingleLevel(csConf); CapacityScheduler capacityScheduler = new CapacityScheduler(); RMContextImpl rmContext = new RMContextImpl(null, null, null, null, null, null, new RMContainerTokenSecretManager(csConf), new NMTokenSecretManagerInRM(csConf), new ClientToAMTokenSecretManagerInRM(), null); RMNodeLabelsManager nodeLabelsManager = new NullRMNodeLabelsManager(); nodeLabelsManager.init(conf); nodeLabelsManager.start(); rmContext.setNodeLabelManager(nodeLabelsManager); capacityScheduler.setConf(csConf); capacityScheduler.setRMContext(rmContext); capacityScheduler.init(csConf); capacityScheduler.start(); ServiceOperations.stopQuietly(capacityScheduler); ServiceOperations.stopQuietly(nodeLabelsManager); }
Example 12
Source File: TestQueueParsing.java From hadoop with Apache License 2.0 | 5 votes |
@Test public void testQueueParsingWhenLabelsInheritedNotExistedInNodeLabelManager() throws IOException { YarnConfiguration conf = new YarnConfiguration(); CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration(conf); setupQueueConfigurationWithLabelsInherit(csConf); CapacityScheduler capacityScheduler = new CapacityScheduler(); RMContextImpl rmContext = new RMContextImpl(null, null, null, null, null, null, new RMContainerTokenSecretManager(csConf), new NMTokenSecretManagerInRM(csConf), new ClientToAMTokenSecretManagerInRM(), null); RMNodeLabelsManager nodeLabelsManager = new NullRMNodeLabelsManager(); nodeLabelsManager.init(conf); nodeLabelsManager.start(); rmContext.setNodeLabelManager(nodeLabelsManager); capacityScheduler.setConf(csConf); capacityScheduler.setRMContext(rmContext); capacityScheduler.init(csConf); capacityScheduler.start(); ServiceOperations.stopQuietly(capacityScheduler); ServiceOperations.stopQuietly(nodeLabelsManager); }
Example 13
Source File: TestQueueParsing.java From hadoop with Apache License 2.0 | 5 votes |
@Test public void testQueueParsingWhenLabelsNotExistedInNodeLabelManager() throws IOException { YarnConfiguration conf = new YarnConfiguration(); CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration(conf); setupQueueConfigurationWithLabels(csConf); CapacityScheduler capacityScheduler = new CapacityScheduler(); RMContextImpl rmContext = new RMContextImpl(null, null, null, null, null, null, new RMContainerTokenSecretManager(csConf), new NMTokenSecretManagerInRM(csConf), new ClientToAMTokenSecretManagerInRM(), null); RMNodeLabelsManager nodeLabelsManager = new NullRMNodeLabelsManager(); nodeLabelsManager.init(conf); nodeLabelsManager.start(); rmContext.setNodeLabelManager(nodeLabelsManager); capacityScheduler.setConf(csConf); capacityScheduler.setRMContext(rmContext); capacityScheduler.init(csConf); capacityScheduler.start(); ServiceOperations.stopQuietly(capacityScheduler); ServiceOperations.stopQuietly(nodeLabelsManager); }
Example 14
Source File: TestSchedulerUtils.java From hadoop with Apache License 2.0 | 5 votes |
private static RMContext getMockRMContext() { RMContext rmContext = mock(RMContext.class); RMNodeLabelsManager nlm = new NullRMNodeLabelsManager(); nlm.init(new Configuration(false)); when(rmContext.getNodeLabelManager()).thenReturn(nlm); return rmContext; }
Example 15
Source File: TestCapacityScheduler.java From big-c with Apache License 2.0 | 4 votes |
@Test(expected = YarnException.class) public void testMoveAppViolateQueueState() throws Exception { resourceManager = new ResourceManager() { @Override protected RMNodeLabelsManager createNodeLabelManager() { RMNodeLabelsManager mgr = new NullRMNodeLabelsManager(); mgr.init(getConfig()); return mgr; } }; CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration(); setupQueueConfiguration(csConf); StringBuilder qState = new StringBuilder(); qState.append(CapacitySchedulerConfiguration.PREFIX).append(B) .append(CapacitySchedulerConfiguration.DOT) .append(CapacitySchedulerConfiguration.STATE); csConf.set(qState.toString(), QueueState.STOPPED.name()); YarnConfiguration conf = new YarnConfiguration(csConf); conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class, ResourceScheduler.class); resourceManager.init(conf); resourceManager.getRMContext().getContainerTokenSecretManager() .rollMasterKey(); resourceManager.getRMContext().getNMTokenSecretManager().rollMasterKey(); ((AsyncDispatcher) resourceManager.getRMContext().getDispatcher()).start(); mockContext = mock(RMContext.class); when(mockContext.getConfigurationProvider()).thenReturn( new LocalConfigurationProvider()); ResourceScheduler scheduler = resourceManager.getResourceScheduler(); // Register node1 String host_0 = "host_0"; NodeManager nm_0 = registerNode(host_0, 1234, 2345, NetworkTopology.DEFAULT_RACK, Resources.createResource(6 * GB, 1)); // ResourceRequest priorities Priority priority_0 = org.apache.hadoop.yarn.server.resourcemanager.resource.Priority .create(0); Priority priority_1 = org.apache.hadoop.yarn.server.resourcemanager.resource.Priority .create(1); // Submit application_0 Application application_0 = new Application("user_0", "a1", resourceManager); application_0.submit(); // app + app attempt event sent to scheduler application_0.addNodeManager(host_0, 1234, nm_0); Resource capability_0_0 = Resources.createResource(3 * GB, 1); application_0.addResourceRequestSpec(priority_1, capability_0_0); Resource capability_0_1 = Resources.createResource(2 * GB, 1); application_0.addResourceRequestSpec(priority_0, capability_0_1); Task task_0_0 = new Task(application_0, priority_1, new String[] { host_0 }); application_0.addTask(task_0_0); // Send resource requests to the scheduler application_0.schedule(); // allocate // task_0_0 allocated nodeUpdate(nm_0); // Get allocations from the scheduler application_0.schedule(); // task_0_0 checkApplicationResourceUsage(3 * GB, application_0); checkNodeResourceUsage(3 * GB, nm_0); // b2 queue contains 3GB consumption app, // add another 3GB will hit max capacity limit on queue b scheduler.moveApplication(application_0.getApplicationId(), "b1"); }
Example 16
Source File: TestContainerAllocation.java From big-c with Apache License 2.0 | 4 votes |
@Test (timeout = 300000) public void testContainerAllocationWithSingleUserLimits() throws Exception { final RMNodeLabelsManager mgr = new NullRMNodeLabelsManager(); mgr.init(conf); // set node -> label mgr.addToCluserNodeLabels(ImmutableSet.of("x", "y")); mgr.addLabelsToNode(ImmutableMap.of(NodeId.newInstance("h1", 0), toSet("x"), NodeId.newInstance("h2", 0), toSet("y"))); // inject node label manager MockRM rm1 = new MockRM(getConfigurationWithDefaultQueueLabels(conf)) { @Override public RMNodeLabelsManager createNodeLabelManager() { return mgr; } }; rm1.getRMContext().setNodeLabelManager(mgr); rm1.start(); MockNM nm1 = rm1.registerNode("h1:1234", 8000); // label = x rm1.registerNode("h2:1234", 8000); // label = y MockNM nm3 = rm1.registerNode("h3:1234", 8000); // label = <empty> // launch an app to queue a1 (label = x), and check all container will // be allocated in h1 RMApp app1 = rm1.submitApp(200, "app", "user", null, "a1"); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1); // A has only 10% of x, so it can only allocate one container in label=empty ContainerId containerId = ContainerId.newContainerId(am1.getApplicationAttemptId(), 2); am1.allocate("*", 1024, 1, new ArrayList<ContainerId>(), ""); Assert.assertTrue(rm1.waitForState(nm3, containerId, RMContainerState.ALLOCATED, 10 * 1000)); // Cannot allocate 2nd label=empty container containerId = ContainerId.newContainerId(am1.getApplicationAttemptId(), 3); am1.allocate("*", 1024, 1, new ArrayList<ContainerId>(), ""); Assert.assertFalse(rm1.waitForState(nm3, containerId, RMContainerState.ALLOCATED, 10 * 1000)); // A has default user limit = 100, so it can use all resource in label = x // We can allocate floor(8000 / 1024) = 7 containers for (int id = 3; id <= 8; id++) { containerId = ContainerId.newContainerId(am1.getApplicationAttemptId(), id); am1.allocate("*", 1024, 1, new ArrayList<ContainerId>(), "x"); Assert.assertTrue(rm1.waitForState(nm1, containerId, RMContainerState.ALLOCATED, 10 * 1000)); } rm1.close(); }
Example 17
Source File: TestContainerAllocation.java From hadoop with Apache License 2.0 | 4 votes |
@Test (timeout = 300000) public void testContainerAllocationWithSingleUserLimits() throws Exception { final RMNodeLabelsManager mgr = new NullRMNodeLabelsManager(); mgr.init(conf); // set node -> label mgr.addToCluserNodeLabels(ImmutableSet.of("x", "y")); mgr.addLabelsToNode(ImmutableMap.of(NodeId.newInstance("h1", 0), toSet("x"), NodeId.newInstance("h2", 0), toSet("y"))); // inject node label manager MockRM rm1 = new MockRM(TestUtils.getConfigurationWithDefaultQueueLabels(conf)) { @Override public RMNodeLabelsManager createNodeLabelManager() { return mgr; } }; rm1.getRMContext().setNodeLabelManager(mgr); rm1.start(); MockNM nm1 = rm1.registerNode("h1:1234", 8000); // label = x rm1.registerNode("h2:1234", 8000); // label = y MockNM nm3 = rm1.registerNode("h3:1234", 8000); // label = <empty> // launch an app to queue a1 (label = x), and check all container will // be allocated in h1 RMApp app1 = rm1.submitApp(200, "app", "user", null, "a1"); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1); // A has only 10% of x, so it can only allocate one container in label=empty ContainerId containerId = ContainerId.newContainerId(am1.getApplicationAttemptId(), 2); am1.allocate("*", 1024, 1, new ArrayList<ContainerId>(), ""); Assert.assertTrue(rm1.waitForState(nm3, containerId, RMContainerState.ALLOCATED, 10 * 1000)); // Cannot allocate 2nd label=empty container containerId = ContainerId.newContainerId(am1.getApplicationAttemptId(), 3); am1.allocate("*", 1024, 1, new ArrayList<ContainerId>(), ""); Assert.assertFalse(rm1.waitForState(nm3, containerId, RMContainerState.ALLOCATED, 10 * 1000)); // A has default user limit = 100, so it can use all resource in label = x // We can allocate floor(8000 / 1024) = 7 containers for (int id = 3; id <= 8; id++) { containerId = ContainerId.newContainerId(am1.getApplicationAttemptId(), id); am1.allocate("*", 1024, 1, new ArrayList<ContainerId>(), "x"); Assert.assertTrue(rm1.waitForState(nm1, containerId, RMContainerState.ALLOCATED, 10 * 1000)); } rm1.close(); }
Example 18
Source File: TestCapacityScheduler.java From hadoop with Apache License 2.0 | 4 votes |
@Test(expected = YarnException.class) public void testMoveAppViolateQueueState() throws Exception { resourceManager = new ResourceManager() { @Override protected RMNodeLabelsManager createNodeLabelManager() { RMNodeLabelsManager mgr = new NullRMNodeLabelsManager(); mgr.init(getConfig()); return mgr; } }; CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration(); setupQueueConfiguration(csConf); StringBuilder qState = new StringBuilder(); qState.append(CapacitySchedulerConfiguration.PREFIX).append(B) .append(CapacitySchedulerConfiguration.DOT) .append(CapacitySchedulerConfiguration.STATE); csConf.set(qState.toString(), QueueState.STOPPED.name()); YarnConfiguration conf = new YarnConfiguration(csConf); conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class, ResourceScheduler.class); resourceManager.init(conf); resourceManager.getRMContext().getContainerTokenSecretManager() .rollMasterKey(); resourceManager.getRMContext().getNMTokenSecretManager().rollMasterKey(); ((AsyncDispatcher) resourceManager.getRMContext().getDispatcher()).start(); mockContext = mock(RMContext.class); when(mockContext.getConfigurationProvider()).thenReturn( new LocalConfigurationProvider()); ResourceScheduler scheduler = resourceManager.getResourceScheduler(); // Register node1 String host_0 = "host_0"; NodeManager nm_0 = registerNode(host_0, 1234, 2345, NetworkTopology.DEFAULT_RACK, Resources.createResource(6 * GB, 1)); // ResourceRequest priorities Priority priority_0 = org.apache.hadoop.yarn.server.resourcemanager.resource.Priority .create(0); Priority priority_1 = org.apache.hadoop.yarn.server.resourcemanager.resource.Priority .create(1); // Submit application_0 Application application_0 = new Application("user_0", "a1", resourceManager); application_0.submit(); // app + app attempt event sent to scheduler application_0.addNodeManager(host_0, 1234, nm_0); Resource capability_0_0 = Resources.createResource(3 * GB, 1); application_0.addResourceRequestSpec(priority_1, capability_0_0); Resource capability_0_1 = Resources.createResource(2 * GB, 1); application_0.addResourceRequestSpec(priority_0, capability_0_1); Task task_0_0 = new Task(application_0, priority_1, new String[] { host_0 }); application_0.addTask(task_0_0); // Send resource requests to the scheduler application_0.schedule(); // allocate // task_0_0 allocated nodeUpdate(nm_0); // Get allocations from the scheduler application_0.schedule(); // task_0_0 checkApplicationResourceUsage(3 * GB, application_0); checkNodeResourceUsage(3 * GB, nm_0); // b2 queue contains 3GB consumption app, // add another 3GB will hit max capacity limit on queue b scheduler.moveApplication(application_0.getApplicationId(), "b1"); }