Java Code Examples for org.apache.hadoop.yarn.conf.YarnConfiguration#setBoolean()

@Test
public void testZKRetryInterval() throws Exception {
  TestZKClient zkClientTester = new TestZKClient();
  YarnConfiguration conf = new YarnConfiguration();

  ZKRMStateStore store =
      (ZKRMStateStore) zkClientTester.getRMStateStore(conf);
  assertEquals(YarnConfiguration.DEFAULT_RM_ZK_RETRY_INTERVAL_MS,
      store.zkRetryInterval);
  store.stop();

  conf.setBoolean(YarnConfiguration.RM_HA_ENABLED, true);
  store =
      (ZKRMStateStore) zkClientTester.getRMStateStore(conf);
  assertEquals(YarnConfiguration.DEFAULT_RM_ZK_TIMEOUT_MS /
          YarnConfiguration.DEFAULT_ZK_RM_NUM_RETRIES,
      store.zkRetryInterval);
  store.stop();
}
 

@Before
public void setup() throws IOException {
  conf = new YarnConfiguration();
  conf.setBoolean(YarnConfiguration.RM_HA_ENABLED, true);
  conf.setBoolean(YarnConfiguration.AUTO_FAILOVER_ENABLED, true);
  conf.setBoolean(YarnConfiguration.AUTO_FAILOVER_EMBEDDED, true);
  conf.set(YarnConfiguration.RM_CLUSTER_ID, "yarn-test-cluster");
  conf.set(YarnConfiguration.RM_ZK_ADDRESS, hostPort);
  conf.setInt(YarnConfiguration.RM_ZK_TIMEOUT_MS, 2000);

  conf.set(YarnConfiguration.RM_HA_IDS, RM1_NODE_ID + "," + RM2_NODE_ID);
  conf.set(YarnConfiguration.RM_HA_ID, RM1_NODE_ID);
  setRpcAddressForRM(RM1_NODE_ID, RM1_PORT_BASE);
  setRpcAddressForRM(RM2_NODE_ID, RM2_PORT_BASE);

  conf.setLong(YarnConfiguration.CLIENT_FAILOVER_SLEEPTIME_BASE_MS, 100L);

  callbackCalled = new AtomicBoolean(false);
}
 

@Before
public void setup() throws IOException {
  conf = new YarnConfiguration();
  conf.setBoolean(YarnConfiguration.RM_HA_ENABLED, true);
  conf.setBoolean(YarnConfiguration.AUTO_FAILOVER_ENABLED, true);
  conf.setBoolean(YarnConfiguration.AUTO_FAILOVER_EMBEDDED, true);
  conf.set(YarnConfiguration.RM_CLUSTER_ID, "yarn-test-cluster");
  conf.set(YarnConfiguration.RM_ZK_ADDRESS, hostPort);
  conf.setInt(YarnConfiguration.RM_ZK_TIMEOUT_MS, 2000);

  conf.set(YarnConfiguration.RM_HA_IDS, RM1_NODE_ID + "," + RM2_NODE_ID);
  conf.set(YarnConfiguration.RM_HA_ID, RM1_NODE_ID);
  setRpcAddressForRM(RM1_NODE_ID, RM1_PORT_BASE);
  setRpcAddressForRM(RM2_NODE_ID, RM2_PORT_BASE);

  conf.setLong(YarnConfiguration.CLIENT_FAILOVER_SLEEPTIME_BASE_MS, 100L);

  callbackCalled = new AtomicBoolean(false);
}
 

@Before
public void setup() throws IOException {
  failoverThread = null;
  keepRunning = true;
  conf = new YarnConfiguration();
  conf.setBoolean(YarnConfiguration.RM_HA_ENABLED, true);
  conf.setInt(YarnConfiguration.CLIENT_FAILOVER_MAX_ATTEMPTS, 5);
  conf.set(YarnConfiguration.RM_HA_IDS, RM1_NODE_ID + "," + RM2_NODE_ID);
  setRpcAddressForRM(RM1_NODE_ID, RM1_PORT_BASE);
  setRpcAddressForRM(RM2_NODE_ID, RM2_PORT_BASE);

  conf.setLong(YarnConfiguration.CLIENT_FAILOVER_SLEEPTIME_BASE_MS, 100L);

  conf.setBoolean(YarnConfiguration.YARN_MINICLUSTER_FIXED_PORTS, true);
  conf.setBoolean(YarnConfiguration.YARN_MINICLUSTER_USE_RPC, true);
}
 

@BeforeClass
public static void setup() throws Exception {
  YarnConfiguration conf = new YarnConfiguration();
  conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true);
  conf.setBoolean(YarnConfiguration.RM_SYSTEM_METRICS_PUBLISHER_ENABLED, true);
  conf.setClass(YarnConfiguration.TIMELINE_SERVICE_STORE,
      MemoryTimelineStore.class, TimelineStore.class);
  conf.setClass(YarnConfiguration.TIMELINE_SERVICE_STATE_STORE_CLASS,
      MemoryTimelineStateStore.class, TimelineStateStore.class);
  conf.setInt(
      YarnConfiguration.RM_SYSTEM_METRICS_PUBLISHER_DISPATCHER_POOL_SIZE,
      2);

  timelineServer = new ApplicationHistoryServer();
  timelineServer.init(conf);
  timelineServer.start();
  store = timelineServer.getTimelineStore();

  metricsPublisher = new SystemMetricsPublisher();
  metricsPublisher.init(conf);
  metricsPublisher.start();
}
 

@Before
public void setup() {
  privilegedOperationExecutorMock = mock(PrivilegedOperationExecutor.class);
  conf = new YarnConfiguration();
  tmpPath = System.getProperty("test.build.data") + "/cgroups";
  //no leading or trailing slashes here
  hierarchy = "test-hadoop-yarn";

  conf.set(YarnConfiguration.NM_LINUX_CONTAINER_CGROUPS_HIERARCHY, hierarchy);
  conf.setBoolean(YarnConfiguration.NM_LINUX_CONTAINER_CGROUPS_MOUNT, true);
  conf.set(YarnConfiguration.NM_LINUX_CONTAINER_CGROUPS_MOUNT_PATH, tmpPath);
  controller = CGroupsHandler.CGroupController.NET_CLS;
  controllerPath = new StringBuffer(tmpPath).append('/')
      .append(controller.getName()).append('/').append(hierarchy).toString();
}
 

@Before
public void setUp() throws Exception {
  conf = new YarnConfiguration();
  conf.setClass(YarnConfiguration.RM_SCHEDULER, FifoSchedulerWithMove.class,
      FifoSchedulerWithMove.class);
  conf.set(YarnConfiguration.YARN_ADMIN_ACL, " ");
  conf.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, true);
  resourceManager = new ResourceManager();
  resourceManager.init(conf);
  resourceManager.getRMContext().getContainerTokenSecretManager().rollMasterKey();
  resourceManager.getRMContext().getNMTokenSecretManager().rollMasterKey();
  resourceManager.start();
  failMove = false;
}
 

public TestApplicationHistoryManagerOnTimelineStore(String caller) {
  conf = new YarnConfiguration();
  if (!caller.equals("")) {
    callerUGI = UserGroupInformation.createRemoteUser(caller, AuthMethod.SIMPLE);
    conf.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, true);
    conf.set(YarnConfiguration.YARN_ADMIN_ACL, "admin");
  }
}
 

@Before
public void setup() throws Exception {
  fsPath = new File("target", getClass().getSimpleName() +
      "-tmpDir").getAbsoluteFile();
  fsContext = FileContext.getLocalFSFileContext();
  fsContext.delete(new Path(fsPath.getAbsolutePath()), true);
  conf = new YarnConfiguration();
  conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_RECOVERY_ENABLED, true);
  conf.setClass(YarnConfiguration.TIMELINE_SERVICE_STATE_STORE_CLASS,
      LeveldbTimelineStateStore.class,
      TimelineStateStore.class);
  conf.set(YarnConfiguration.TIMELINE_SERVICE_LEVELDB_STATE_STORE_PATH,
      fsPath.getAbsolutePath());
}
 

@Before
public void setup() {
  conf = new YarnConfiguration();
  conf.set(YarnConfiguration.RM_SCHEDULER,
      "org.apache.hadoop.yarn.sls.scheduler.ResourceSchedulerWrapper");
  conf.set(SLSConfiguration.RM_SCHEDULER,
      "org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler");
  conf.setBoolean(SLSConfiguration.METRICS_SWITCH, false);
  rm = new ResourceManager();
  rm.init(conf);
  rm.start();
}
 

@SuppressWarnings("unchecked")
protected void testContainerPreservationOnResyncImpl(TestNodeManager1 nm,
    boolean isWorkPreservingRestartEnabled)
    throws IOException, YarnException, InterruptedException {
  YarnConfiguration conf = createNMConfig();
  conf.setBoolean(YarnConfiguration.RM_WORK_PRESERVING_RECOVERY_ENABLED,
      isWorkPreservingRestartEnabled);

  try {
    nm.init(conf);
    nm.start();
    ContainerId cId = TestNodeManagerShutdown.createContainerId();
    TestNodeManagerShutdown.startContainer(nm, cId, localFS, tmpDir,
        processStartFile);

    nm.setExistingContainerId(cId);
    Assert.assertEquals(1, ((TestNodeManager1) nm).getNMRegistrationCount());
    nm.getNMDispatcher().getEventHandler().handle(resyncEvent);
    try {
      syncBarrier.await();
    } catch (BrokenBarrierException e) {
    }
    Assert.assertEquals(2, ((TestNodeManager1) nm).getNMRegistrationCount());
    // Only containers should be killed on resync, apps should lie around.
    // That way local resources for apps can be used beyond resync without
    // relocalization
    Assert.assertTrue(nm.getNMContext().getApplications()
        .containsKey(cId.getApplicationAttemptId().getApplicationId()));
    Assert.assertFalse(assertionFailedInThread.get());
  }
  finally {
    nm.stop();
  }
}
 

@Test
public void testApplicationKeepAlive() throws Exception {
  MyNodeManager nm = new MyNodeManager();
  try {
    YarnConfiguration conf = createNMConfig();
    conf.setBoolean(YarnConfiguration.LOG_AGGREGATION_ENABLED, true);
    conf.setLong(YarnConfiguration.RM_NM_EXPIRY_INTERVAL_MS,
        4000l);
    nm.init(conf);
    nm.start();
    // HB 2 -> app cancelled by RM.
    while (heartBeatID < 12) {
      Thread.sleep(1000l);
    }
    MyResourceTracker3 rt =
        (MyResourceTracker3) nm.getNodeStatusUpdater().getRMClient();
    rt.context.getApplications().remove(rt.appId);
    Assert.assertEquals(1, rt.keepAliveRequests.size());
    int numKeepAliveRequests = rt.keepAliveRequests.get(rt.appId).size();
    LOG.info("Number of Keep Alive Requests: [" + numKeepAliveRequests + "]");
    Assert.assertTrue(numKeepAliveRequests == 2 || numKeepAliveRequests == 3);
    while (heartBeatID < 20) {
      Thread.sleep(1000l);
    }
    int numKeepAliveRequests2 = rt.keepAliveRequests.get(rt.appId).size();
    Assert.assertEquals(numKeepAliveRequests, numKeepAliveRequests2);
  } finally {
    if (nm.getServiceState() == STATE.STARTED)
      nm.stop();
  }
}
 

@Before
public void setup() throws Exception {
  fsPath = new File("target", getClass().getSimpleName() +
      "-tmpDir").getAbsoluteFile();
  fsContext = FileContext.getLocalFSFileContext();
  fsContext.delete(new Path(fsPath.getAbsolutePath()), true);
  conf = new YarnConfiguration();
  conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_RECOVERY_ENABLED, true);
  conf.setClass(YarnConfiguration.TIMELINE_SERVICE_STATE_STORE_CLASS,
      LeveldbTimelineStateStore.class,
      TimelineStateStore.class);
  conf.set(YarnConfiguration.TIMELINE_SERVICE_LEVELDB_STATE_STORE_PATH,
      fsPath.getAbsolutePath());
}
 

@Before
public void setUp() throws Exception {
  conf = new YarnConfiguration();
  conf.setClass(YarnConfiguration.RM_SCHEDULER, FifoSchedulerWithMove.class,
      FifoSchedulerWithMove.class);
  conf.set(YarnConfiguration.YARN_ADMIN_ACL, " ");
  conf.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, true);
  resourceManager = new ResourceManager();
  resourceManager.init(conf);
  resourceManager.getRMContext().getContainerTokenSecretManager().rollMasterKey();
  resourceManager.getRMContext().getNMTokenSecretManager().rollMasterKey();
  resourceManager.start();
  failMove = false;
}
 

@Before
public void setup() {
  YarnConfiguration conf = new YarnConfiguration();
  conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true);
  client = createTimelineClient(conf);
}
 

public static synchronized Configuration getInstanceConfig() throws Exception {
    if (conf == null) {
        File zooRoot = File.createTempFile("hbase-zookeeper", "");
        zooRoot.delete();
        ZooKeeperServer zookeper = new ZooKeeperServer(zooRoot, zooRoot, 2000);
        ServerCnxnFactory factory = ServerCnxnFactory.createFactory(new InetSocketAddress("localhost", 0), 5000);
        factory.startup(zookeper);

        YarnConfiguration yconf = new YarnConfiguration();
        String argLine = System.getProperty("argLine");
        if (argLine != null) {
            yconf.set("yarn.app.mapreduce.am.command-opts", argLine.replace("jacoco.exec", "jacocoMR.exec"));
        }
        yconf.setBoolean(MRConfig.MAPREDUCE_MINICLUSTER_CONTROL_RESOURCE_MONITORING, false);
        yconf.setClass(YarnConfiguration.RM_SCHEDULER, FifoScheduler.class, ResourceScheduler.class);
        MiniMRYarnCluster miniCluster = new MiniMRYarnCluster("testCluster");
        miniCluster.init(yconf);
        String resourceManagerLink = yconf.get(YarnConfiguration.RM_ADDRESS);
        yconf.setBoolean(MRJobConfig.JOB_UBERTASK_ENABLE, true);
        miniCluster.start();
        miniCluster.waitForNodeManagersToConnect(10000);
        // following condition set in MiniYarnCluster:273
        while (resourceManagerLink.endsWith(":0")) {
            Thread.sleep(100);
            resourceManagerLink = yconf.get(YarnConfiguration.RM_ADDRESS);
        }

        File hbaseRoot = File.createTempFile("hbase-root", "");
        hbaseRoot.delete();
        conf = HBaseConfiguration.create(miniCluster.getConfig());
        conf.set(HConstants.HBASE_DIR, hbaseRoot.toURI().toURL().toString());
        conf.setInt(HConstants.ZOOKEEPER_CLIENT_PORT, factory.getLocalPort());
        conf.set("hbase.master.hostname", "localhost");
        conf.set("hbase.regionserver.hostname", "localhost");
        conf.setInt("hbase.master.info.port", -1);
        conf.set("hbase.fs.tmp.dir", new File(System.getProperty("java.io.tmpdir")).toURI().toURL().toString());
        LocalHBaseCluster cluster = new LocalHBaseCluster(conf);
        cluster.startup();
    }
    return new Configuration(conf);
}
 

@Test(timeout = 20000)
public void testPreemptedAMRestartOnRMRestart() throws Exception {
  YarnConfiguration conf = new YarnConfiguration();
  conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class,
    ResourceScheduler.class);
  conf.setBoolean(YarnConfiguration.RECOVERY_ENABLED, true);
  conf.setBoolean(YarnConfiguration.RM_WORK_PRESERVING_RECOVERY_ENABLED, false);

  conf.set(YarnConfiguration.RM_STORE, MemoryRMStateStore.class.getName());
  // explicitly set max-am-retry count as 1.
  conf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, 1);
  MemoryRMStateStore memStore = new MemoryRMStateStore();
  memStore.init(conf);

  MockRM rm1 = new MockRM(conf, memStore);
  rm1.start();
  MockNM nm1 =
      new MockNM("127.0.0.1:1234", 8000, rm1.getResourceTrackerService());
  nm1.registerNode();
  RMApp app1 = rm1.submitApp(200);
  RMAppAttempt attempt1 = app1.getCurrentAppAttempt();
  MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1);
  CapacityScheduler scheduler =
      (CapacityScheduler) rm1.getResourceScheduler();
  ContainerId amContainer =
      ContainerId.newContainerId(am1.getApplicationAttemptId(), 1);

  // Forcibly preempt the am container;
  scheduler.killContainer(scheduler.getRMContainer(amContainer));

  am1.waitForState(RMAppAttemptState.FAILED);
  Assert.assertTrue(! attempt1.shouldCountTowardsMaxAttemptRetry());
  rm1.waitForState(app1.getApplicationId(), RMAppState.ACCEPTED);

  // state store has 1 attempt stored.
  ApplicationStateData appState =
      memStore.getState().getApplicationState().get(app1.getApplicationId());
  Assert.assertEquals(1, appState.getAttemptCount());
  // attempt stored has the preempted container exit status.
  Assert.assertEquals(ContainerExitStatus.PREEMPTED,
    appState.getAttempt(am1.getApplicationAttemptId())
      .getAMContainerExitStatus());
  // Restart rm.
  MockRM rm2 = new MockRM(conf, memStore);
  nm1.setResourceTrackerService(rm2.getResourceTrackerService());
  nm1.registerNode();
  rm2.start();

  // Restarted RM should re-launch the am.
  MockAM am2 =
      rm2.waitForNewAMToLaunchAndRegister(app1.getApplicationId(), 2, nm1);
  MockRM.finishAMAndVerifyAppState(app1, rm2, nm1, am2);
  RMAppAttempt attempt2 =
      rm2.getRMContext().getRMApps().get(app1.getApplicationId())
        .getCurrentAppAttempt();
  Assert.assertTrue(attempt2.shouldCountTowardsMaxAttemptRetry());
  Assert.assertEquals(ContainerExitStatus.INVALID,
    appState.getAttempt(am2.getApplicationAttemptId())
      .getAMContainerExitStatus());
  rm1.stop();
  rm2.stop();
}
 

@Test
public void testParentQueueMaxCapsAreRespected() throws Exception {
  /*
   * Queue tree:
   *          Root
   *        /     \
   *       A       B
   *      / \
   *     A1 A2 
   */
  CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration();
  csConf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[] {"a", "b"});
  csConf.setCapacity(A, 50);
  csConf.setMaximumCapacity(A, 50);
  csConf.setCapacity(B, 50);
  
  // Define 2nd-level queues
  csConf.setQueues(A, new String[] {"a1", "a2"});
  csConf.setCapacity(A1, 50);
  csConf.setUserLimitFactor(A1, 100.0f);
  csConf.setCapacity(A2, 50);
  csConf.setUserLimitFactor(A2, 100.0f);
  csConf.setCapacity(B1, B1_CAPACITY);
  csConf.setUserLimitFactor(B1, 100.0f);
  
  YarnConfiguration conf = new YarnConfiguration(csConf);
  conf.setBoolean(CapacitySchedulerConfiguration.ENABLE_USER_METRICS, true);

  MemoryRMStateStore memStore = new MemoryRMStateStore();
  memStore.init(conf);
  MockRM rm1 = new MockRM(conf, memStore);
  rm1.start();
  MockNM nm1 =
      new MockNM("127.0.0.1:1234", 24 * GB, rm1.getResourceTrackerService());
  nm1.registerNode();
  
  // Launch app1 in a1, resource usage is 1GB (am) + 4GB * 2 = 9GB 
  RMApp app1 = rm1.submitApp(1 * GB, "app", "user", null, "a1");
  MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1);
  waitContainerAllocated(am1, 4 * GB, 2, 2, rm1, nm1);
  
  // Try to launch app2 in a2, asked 2GB, should success 
  RMApp app2 = rm1.submitApp(2 * GB, "app", "user", null, "a2");
  MockAM am2 = MockRM.launchAndRegisterAM(app2, rm1, nm1);
  try {
    // Try to allocate a container, a's usage=11G/max=12
    // a1's usage=9G/max=12
    // a2's usage=2G/max=12
    // In this case, if a2 asked 2G, should fail.
    waitContainerAllocated(am2, 2 * GB, 1, 2, rm1, nm1);
  } catch (AssertionError failure) {
    // Expected, return;
    return;
  }
  Assert.fail("Shouldn't successfully allocate containers for am2, "
      + "queue-a's max capacity will be violated if container allocated");
}
 

@SuppressWarnings("static-access")
@Before
public void configure() throws IOException, YarnException {
  remoteAdminServiceAccessed = false;
  admin = mock(ResourceManagerAdministrationProtocol.class);
  when(admin.addToClusterNodeLabels(any(AddToClusterNodeLabelsRequest.class)))
      .thenAnswer(new Answer<AddToClusterNodeLabelsResponse>() {

        @Override
        public AddToClusterNodeLabelsResponse answer(
            InvocationOnMock invocation) throws Throwable {
          remoteAdminServiceAccessed = true;
          return AddToClusterNodeLabelsResponse.newInstance();
        }
      });

  haadmin = mock(HAServiceProtocol.class);
  when(haadmin.getServiceStatus()).thenReturn(new HAServiceStatus(
      HAServiceProtocol.HAServiceState.INITIALIZING));

  final HAServiceTarget haServiceTarget = mock(HAServiceTarget.class);
  when(haServiceTarget.getProxy(any(Configuration.class), anyInt()))
      .thenReturn(haadmin);
  rmAdminCLI = new RMAdminCLI(new Configuration()) {
    @Override
    protected ResourceManagerAdministrationProtocol createAdminProtocol()
        throws IOException {
      return admin;
    }

    @Override
    protected HAServiceTarget resolveTarget(String rmId) {
      return haServiceTarget;
    }
  };
  initDummyNodeLabelsManager();
  rmAdminCLI.localNodeLabelsManager = dummyNodeLabelsManager;

  YarnConfiguration conf = new YarnConfiguration();
  conf.setBoolean(YarnConfiguration.RM_HA_ENABLED, true);
  rmAdminCLIWithHAEnabled = new RMAdminCLI(conf) {

    @Override
    protected ResourceManagerAdministrationProtocol createAdminProtocol()
        throws IOException {
      return admin;
    }

    @Override
    protected HAServiceTarget resolveTarget(String rmId) {
      return haServiceTarget;
    }
  };
}
 

@Before
public void setup() {
  YarnConfiguration conf = new YarnConfiguration();
  conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true);
  client = createTimelineClient(conf);
}