Java Code Examples for org.apache.hadoop.yarn.server.resourcemanager.ResourceManager#init()

The following examples show how to use org.apache.hadoop.yarn.server.resourcemanager.ResourceManager#init() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: TestFairSchedulerEventLog.java    From hadoop with Apache License 2.0 6 votes vote down vote up 
@Before
public void setUp() throws IOException {
  scheduler = new FairScheduler();
  
  Configuration conf = new YarnConfiguration();
  conf.setClass(YarnConfiguration.RM_SCHEDULER, FairScheduler.class,
      ResourceScheduler.class);
  conf.set("yarn.scheduler.fair.event-log-enabled", "true");

  // All tests assume only one assignment per node update
  conf.set(FairSchedulerConfiguration.ASSIGN_MULTIPLE, "false");
  resourceManager = new ResourceManager();
  resourceManager.init(conf);
  ((AsyncDispatcher)resourceManager.getRMContext().getDispatcher()).start();
  scheduler.init(conf);
  scheduler.start();
  scheduler.reinitialize(conf, resourceManager.getRMContext());
}
Example 2
Source File: TestFairScheduler.java    From hadoop with Apache License 2.0 6 votes vote down vote up 
@Before
public void setUp() throws IOException {
  scheduler = new FairScheduler();
  conf = createConfiguration();
  resourceManager = new ResourceManager();
  resourceManager.init(conf);

  // TODO: This test should really be using MockRM. For now starting stuff
  // that is needed at a bare minimum.
  ((AsyncDispatcher)resourceManager.getRMContext().getDispatcher()).start();
  resourceManager.getRMContext().getStateStore().start();

  // to initialize the master key
  resourceManager.getRMContext().getContainerTokenSecretManager().rollMasterKey();

  scheduler.setRMContext(resourceManager.getRMContext());
}
Example 3
Source File: TestCapacityScheduler.java    From hadoop with Apache License 2.0 6 votes vote down vote up 
@Before
public void setUp() throws Exception {
  resourceManager = new ResourceManager() {
    @Override
    protected RMNodeLabelsManager createNodeLabelManager() {
      RMNodeLabelsManager mgr = new NullRMNodeLabelsManager();
      mgr.init(getConfig());
      return mgr;
    }
  };
  CapacitySchedulerConfiguration csConf 
     = new CapacitySchedulerConfiguration();
  setupQueueConfiguration(csConf);
  YarnConfiguration conf = new YarnConfiguration(csConf);
  conf.setClass(YarnConfiguration.RM_SCHEDULER, 
      CapacityScheduler.class, ResourceScheduler.class);
  resourceManager.init(conf);
  resourceManager.getRMContext().getContainerTokenSecretManager().rollMasterKey();
  resourceManager.getRMContext().getNMTokenSecretManager().rollMasterKey();
  ((AsyncDispatcher)resourceManager.getRMContext().getDispatcher()).start();
  mockContext = mock(RMContext.class);
  when(mockContext.getConfigurationProvider()).thenReturn(
      new LocalConfigurationProvider());
}
Example 4
Source File: TestFairScheduler.java    From big-c with Apache License 2.0 6 votes vote down vote up 
@Before
public void setUp() throws IOException {
  scheduler = new FairScheduler();
  conf = createConfiguration();
  resourceManager = new ResourceManager();
  resourceManager.init(conf);

  // TODO: This test should really be using MockRM. For now starting stuff
  // that is needed at a bare minimum.
  ((AsyncDispatcher)resourceManager.getRMContext().getDispatcher()).start();
  resourceManager.getRMContext().getStateStore().start();

  // to initialize the master key
  resourceManager.getRMContext().getContainerTokenSecretManager().rollMasterKey();

  scheduler.setRMContext(resourceManager.getRMContext());
}
Example 5
Source File: TestFairSchedulerEventLog.java    From big-c with Apache License 2.0 6 votes vote down vote up 
@Before
public void setUp() throws IOException {
  scheduler = new FairScheduler();
  
  Configuration conf = new YarnConfiguration();
  conf.setClass(YarnConfiguration.RM_SCHEDULER, FairScheduler.class,
      ResourceScheduler.class);
  conf.set("yarn.scheduler.fair.event-log-enabled", "true");

  // All tests assume only one assignment per node update
  conf.set(FairSchedulerConfiguration.ASSIGN_MULTIPLE, "false");
  resourceManager = new ResourceManager();
  resourceManager.init(conf);
  ((AsyncDispatcher)resourceManager.getRMContext().getDispatcher()).start();
  scheduler.init(conf);
  scheduler.start();
  scheduler.reinitialize(conf, resourceManager.getRMContext());
}
Example 6
Source File: SLSRunner.java    From hadoop with Apache License 2.0 5 votes vote down vote up 
private void startRM() throws IOException, ClassNotFoundException {
  Configuration rmConf = new YarnConfiguration();
  String schedulerClass = rmConf.get(YarnConfiguration.RM_SCHEDULER);
  rmConf.set(SLSConfiguration.RM_SCHEDULER, schedulerClass);
  rmConf.set(YarnConfiguration.RM_SCHEDULER,
          ResourceSchedulerWrapper.class.getName());
  rmConf.set(SLSConfiguration.METRICS_OUTPUT_DIR, metricsOutputDir);
  rm = new ResourceManager();
  rm.init(rmConf);
  rm.start();
}
Example 7
Source File: TestSchedulingMonitor.java    From big-c with Apache License 2.0 5 votes vote down vote up 
@Test(timeout = 10000)
public void testRMStarts() {
  Configuration conf = new YarnConfiguration();
  conf.setBoolean(YarnConfiguration.RM_SCHEDULER_ENABLE_MONITORS, true);
  conf.set(YarnConfiguration.RM_SCHEDULER_MONITOR_POLICIES,
      ProportionalCapacityPreemptionPolicy.class.getCanonicalName());

  ResourceManager rm = new ResourceManager();
  try {
    rm.init(conf);
  } catch (Exception e) {
    fail("ResourceManager does not start when " +
        YarnConfiguration.RM_SCHEDULER_ENABLE_MONITORS + " is set to true");
  }
}
Example 8
Source File: TestYarnClient.java    From big-c with Apache License 2.0 5 votes vote down vote up 
@Test
public void testClientStop() {
  Configuration conf = new Configuration();
  ResourceManager rm = new ResourceManager();
  rm.init(conf);
  rm.start();

  YarnClient client = YarnClient.createYarnClient();
  client.init(conf);
  client.start();
  client.stop();
  rm.stop();
}
Example 9
Source File: TestNMSimulator.java    From big-c with Apache License 2.0 5 votes vote down vote up 
@Before
public void setup() {
  conf = new YarnConfiguration();
  conf.set(YarnConfiguration.RM_SCHEDULER,
      "org.apache.hadoop.yarn.sls.scheduler.ResourceSchedulerWrapper");
  conf.set(SLSConfiguration.RM_SCHEDULER,
      "org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler");
  conf.setBoolean(SLSConfiguration.METRICS_SWITCH, false);
  rm = new ResourceManager();
  rm.init(conf);
  rm.start();
}
Example 10
Source File: TestFifoScheduler.java    From big-c with Apache License 2.0 5 votes vote down vote up 
@Before
public void setUp() throws Exception {
  resourceManager = new ResourceManager();
  Configuration conf = new Configuration();
  conf.setClass(YarnConfiguration.RM_SCHEDULER, 
      FifoScheduler.class, ResourceScheduler.class);
  resourceManager.init(conf);
}
Example 11
Source File: TestAMSimulator.java    From hadoop with Apache License 2.0 5 votes vote down vote up 
@Before
public void setup() {
  conf = new YarnConfiguration();
  conf.set(YarnConfiguration.RM_SCHEDULER,
      "org.apache.hadoop.yarn.sls.scheduler.ResourceSchedulerWrapper");
  conf.set(SLSConfiguration.RM_SCHEDULER,
      "org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler");
  conf.setBoolean(SLSConfiguration.METRICS_SWITCH, false);
  rm = new ResourceManager();
  rm.init(conf);
  rm.start();
}
Example 12
Source File: TestResourceManagerAdministrationProtocolPBClientImpl.java    From big-c with Apache License 2.0 5 votes vote down vote up 
/**
 * Start resource manager server
 */

@BeforeClass
public static void setUpResourceManager() throws IOException,
        InterruptedException {
  Configuration.addDefaultResource("config-with-security.xml");
  Configuration configuration = new YarnConfiguration();
  resourceManager = new ResourceManager() {
    @Override
    protected void doSecureLogin() throws IOException {
    }
  };
  resourceManager.init(configuration);
  new Thread() {
    public void run() {
      resourceManager.start();
    }
  }.start();
  int waitCount = 0;
  while (resourceManager.getServiceState() == STATE.INITED
          && waitCount++ < 10) {
    LOG.info("Waiting for RM to start...");
    Thread.sleep(1000);
  }
  if (resourceManager.getServiceState() != STATE.STARTED) {
    throw new IOException("ResourceManager failed to start. Final state is "
            + resourceManager.getServiceState());
  }
  LOG.info("ResourceManager RMAdmin address: "
          + configuration.get(YarnConfiguration.RM_ADMIN_ADDRESS));

  client = new ResourceManagerAdministrationProtocolPBClientImpl(1L,
          getProtocolAddress(configuration), configuration);

}
Example 13
Source File: TestFifoScheduler.java    From hadoop with Apache License 2.0 5 votes vote down vote up 
@Before
public void setUp() throws Exception {
  resourceManager = new ResourceManager();
  Configuration conf = new Configuration();
  conf.setClass(YarnConfiguration.RM_SCHEDULER, 
      FifoScheduler.class, ResourceScheduler.class);
  resourceManager.init(conf);
}
Example 14
Source File: TestAMSimulator.java    From big-c with Apache License 2.0 5 votes vote down vote up 
@Before
public void setup() {
  conf = new YarnConfiguration();
  conf.set(YarnConfiguration.RM_SCHEDULER,
      "org.apache.hadoop.yarn.sls.scheduler.ResourceSchedulerWrapper");
  conf.set(SLSConfiguration.RM_SCHEDULER,
      "org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler");
  conf.setBoolean(SLSConfiguration.METRICS_SWITCH, false);
  rm = new ResourceManager();
  rm.init(conf);
  rm.start();
}
Example 15
Source File: TestGetGroups.java    From hadoop with Apache License 2.0 5 votes vote down vote up 
@BeforeClass
public static void setUpResourceManager() throws IOException, InterruptedException {
  conf = new YarnConfiguration();
  resourceManager = new ResourceManager() {
    @Override
    protected void doSecureLogin() throws IOException {
    };
  };
  resourceManager.init(conf);
  new Thread() {
    public void run() {
      resourceManager.start();
    };
  }.start();
  int waitCount = 0;
  while (resourceManager.getServiceState() == STATE.INITED
      && waitCount++ < 10) {
    LOG.info("Waiting for RM to start...");
    Thread.sleep(1000);
  }
  if (resourceManager.getServiceState() != STATE.STARTED) {
    throw new IOException(
        "ResourceManager failed to start. Final state is "
            + resourceManager.getServiceState());
  }
  LOG.info("ResourceManager RMAdmin address: " +
      conf.get(YarnConfiguration.RM_ADMIN_ADDRESS));
}
Example 16
Source File: TestGetGroups.java    From big-c with Apache License 2.0 5 votes vote down vote up 
@BeforeClass
public static void setUpResourceManager() throws IOException, InterruptedException {
  conf = new YarnConfiguration();
  resourceManager = new ResourceManager() {
    @Override
    protected void doSecureLogin() throws IOException {
    };
  };
  resourceManager.init(conf);
  new Thread() {
    public void run() {
      resourceManager.start();
    };
  }.start();
  int waitCount = 0;
  while (resourceManager.getServiceState() == STATE.INITED
      && waitCount++ < 10) {
    LOG.info("Waiting for RM to start...");
    Thread.sleep(1000);
  }
  if (resourceManager.getServiceState() != STATE.STARTED) {
    throw new IOException(
        "ResourceManager failed to start. Final state is "
            + resourceManager.getServiceState());
  }
  LOG.info("ResourceManager RMAdmin address: " +
      conf.get(YarnConfiguration.RM_ADMIN_ADDRESS));
}
Example 17
Source File: TestSchedulingMonitor.java    From hadoop with Apache License 2.0 5 votes vote down vote up 
@Test(timeout = 10000)
public void testRMStarts() {
  Configuration conf = new YarnConfiguration();
  conf.setBoolean(YarnConfiguration.RM_SCHEDULER_ENABLE_MONITORS, true);
  conf.set(YarnConfiguration.RM_SCHEDULER_MONITOR_POLICIES,
      ProportionalCapacityPreemptionPolicy.class.getCanonicalName());

  ResourceManager rm = new ResourceManager();
  try {
    rm.init(conf);
  } catch (Exception e) {
    fail("ResourceManager does not start when " +
        YarnConfiguration.RM_SCHEDULER_ENABLE_MONITORS + " is set to true");
  }
}
Example 18
Source File: TestZKRMStateStore.java    From hadoop with Apache License 2.0 4 votes vote down vote up 
@SuppressWarnings("unchecked")
@Test
public void testFencing() throws Exception {
  StateChangeRequestInfo req = new StateChangeRequestInfo(
      HAServiceProtocol.RequestSource.REQUEST_BY_USER);

  Configuration conf1 = createHARMConf("rm1,rm2", "rm1", 1234);
  conf1.setBoolean(YarnConfiguration.AUTO_FAILOVER_ENABLED, false);
  ResourceManager rm1 = new ResourceManager();
  rm1.init(conf1);
  rm1.start();
  rm1.getRMContext().getRMAdminService().transitionToActive(req);
  assertEquals("RM with ZKStore didn't start",
      Service.STATE.STARTED, rm1.getServiceState());
  assertEquals("RM should be Active",
      HAServiceProtocol.HAServiceState.ACTIVE,
      rm1.getRMContext().getRMAdminService().getServiceStatus().getState());

  Configuration conf2 = createHARMConf("rm1,rm2", "rm2", 5678);
  conf2.setBoolean(YarnConfiguration.AUTO_FAILOVER_ENABLED, false);
  ResourceManager rm2 = new ResourceManager();
  rm2.init(conf2);
  rm2.start();
  rm2.getRMContext().getRMAdminService().transitionToActive(req);
  assertEquals("RM with ZKStore didn't start",
      Service.STATE.STARTED, rm2.getServiceState());
  assertEquals("RM should be Active",
      HAServiceProtocol.HAServiceState.ACTIVE,
      rm2.getRMContext().getRMAdminService().getServiceStatus().getState());

  for (int i = 0; i < ZK_TIMEOUT_MS / 50; i++) {
    if (HAServiceProtocol.HAServiceState.ACTIVE ==
        rm1.getRMContext().getRMAdminService().getServiceStatus().getState()) {
      Thread.sleep(100);
    }
  }
  assertEquals("RM should have been fenced",
      HAServiceProtocol.HAServiceState.STANDBY,
      rm1.getRMContext().getRMAdminService().getServiceStatus().getState());
  assertEquals("RM should be Active",
      HAServiceProtocol.HAServiceState.ACTIVE,
      rm2.getRMContext().getRMAdminService().getServiceStatus().getState());
}
Example 19
Source File: TestCapacityScheduler.java    From hadoop with Apache License 2.0 4 votes vote down vote up 
@Test(expected = YarnException.class)
public void testMoveAppViolateQueueState() throws Exception {
  resourceManager = new ResourceManager() {
     @Override
      protected RMNodeLabelsManager createNodeLabelManager() {
        RMNodeLabelsManager mgr = new NullRMNodeLabelsManager();
        mgr.init(getConfig());
        return mgr;
      }
  };
  CapacitySchedulerConfiguration csConf =
      new CapacitySchedulerConfiguration();
  setupQueueConfiguration(csConf);
  StringBuilder qState = new StringBuilder();
  qState.append(CapacitySchedulerConfiguration.PREFIX).append(B)
      .append(CapacitySchedulerConfiguration.DOT)
      .append(CapacitySchedulerConfiguration.STATE);
  csConf.set(qState.toString(), QueueState.STOPPED.name());
  YarnConfiguration conf = new YarnConfiguration(csConf);
  conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class,
      ResourceScheduler.class);
  resourceManager.init(conf);
  resourceManager.getRMContext().getContainerTokenSecretManager()
      .rollMasterKey();
  resourceManager.getRMContext().getNMTokenSecretManager().rollMasterKey();
  ((AsyncDispatcher) resourceManager.getRMContext().getDispatcher()).start();
  mockContext = mock(RMContext.class);
  when(mockContext.getConfigurationProvider()).thenReturn(
      new LocalConfigurationProvider());

  ResourceScheduler scheduler = resourceManager.getResourceScheduler();

  // Register node1
  String host_0 = "host_0";
  NodeManager nm_0 =
      registerNode(host_0, 1234, 2345, NetworkTopology.DEFAULT_RACK,
          Resources.createResource(6 * GB, 1));

  // ResourceRequest priorities
  Priority priority_0 =
      org.apache.hadoop.yarn.server.resourcemanager.resource.Priority
          .create(0);
  Priority priority_1 =
      org.apache.hadoop.yarn.server.resourcemanager.resource.Priority
          .create(1);

  // Submit application_0
  Application application_0 =
      new Application("user_0", "a1", resourceManager);
  application_0.submit(); // app + app attempt event sent to scheduler

  application_0.addNodeManager(host_0, 1234, nm_0);

  Resource capability_0_0 = Resources.createResource(3 * GB, 1);
  application_0.addResourceRequestSpec(priority_1, capability_0_0);

  Resource capability_0_1 = Resources.createResource(2 * GB, 1);
  application_0.addResourceRequestSpec(priority_0, capability_0_1);

  Task task_0_0 =
      new Task(application_0, priority_1, new String[] { host_0 });
  application_0.addTask(task_0_0);

  // Send resource requests to the scheduler
  application_0.schedule(); // allocate

  // task_0_0 allocated
  nodeUpdate(nm_0);

  // Get allocations from the scheduler
  application_0.schedule(); // task_0_0
  checkApplicationResourceUsage(3 * GB, application_0);

  checkNodeResourceUsage(3 * GB, nm_0);
  // b2 queue contains 3GB consumption app,
  // add another 3GB will hit max capacity limit on queue b
  scheduler.moveApplication(application_0.getApplicationId(), "b1");

}
Example 20
Source File: TestZKRMStateStore.java    From big-c with Apache License 2.0 4 votes vote down vote up 
@SuppressWarnings("unchecked")
@Test
public void testFencing() throws Exception {
  StateChangeRequestInfo req = new StateChangeRequestInfo(
      HAServiceProtocol.RequestSource.REQUEST_BY_USER);

  Configuration conf1 = createHARMConf("rm1,rm2", "rm1", 1234);
  conf1.setBoolean(YarnConfiguration.AUTO_FAILOVER_ENABLED, false);
  ResourceManager rm1 = new ResourceManager();
  rm1.init(conf1);
  rm1.start();
  rm1.getRMContext().getRMAdminService().transitionToActive(req);
  assertEquals("RM with ZKStore didn't start",
      Service.STATE.STARTED, rm1.getServiceState());
  assertEquals("RM should be Active",
      HAServiceProtocol.HAServiceState.ACTIVE,
      rm1.getRMContext().getRMAdminService().getServiceStatus().getState());

  Configuration conf2 = createHARMConf("rm1,rm2", "rm2", 5678);
  conf2.setBoolean(YarnConfiguration.AUTO_FAILOVER_ENABLED, false);
  ResourceManager rm2 = new ResourceManager();
  rm2.init(conf2);
  rm2.start();
  rm2.getRMContext().getRMAdminService().transitionToActive(req);
  assertEquals("RM with ZKStore didn't start",
      Service.STATE.STARTED, rm2.getServiceState());
  assertEquals("RM should be Active",
      HAServiceProtocol.HAServiceState.ACTIVE,
      rm2.getRMContext().getRMAdminService().getServiceStatus().getState());

  for (int i = 0; i < ZK_TIMEOUT_MS / 50; i++) {
    if (HAServiceProtocol.HAServiceState.ACTIVE ==
        rm1.getRMContext().getRMAdminService().getServiceStatus().getState()) {
      Thread.sleep(100);
    }
  }
  assertEquals("RM should have been fenced",
      HAServiceProtocol.HAServiceState.STANDBY,
      rm1.getRMContext().getRMAdminService().getServiceStatus().getState());
  assertEquals("RM should be Active",
      HAServiceProtocol.HAServiceState.ACTIVE,
      rm2.getRMContext().getRMAdminService().getServiceStatus().getState());
}