Java Code Examples for org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt#getAppAttemptId()

The following examples show how to use org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt#getAppAttemptId() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example 1

Source File: TestRM.java From hadoop with Apache License 2.0

4 votes

@Test (timeout = 300000)
public void testActivatingApplicationAfterAddingNM() throws Exception {
  MockRM rm1 = new MockRM(conf);

  // start like normal because state is empty
  rm1.start();

  // app that gets launched
  RMApp app1 = rm1.submitApp(200);

  // app that does not get launched
  RMApp app2 = rm1.submitApp(200);

  // app1 and app2 should be scheduled, but because no resource is available,
  // they are not activated.
  RMAppAttempt attempt1 = app1.getCurrentAppAttempt();
  ApplicationAttemptId attemptId1 = attempt1.getAppAttemptId();
  rm1.waitForState(attemptId1, RMAppAttemptState.SCHEDULED);
  RMAppAttempt attempt2 = app2.getCurrentAppAttempt();
  ApplicationAttemptId attemptId2 = attempt2.getAppAttemptId();
  rm1.waitForState(attemptId2, RMAppAttemptState.SCHEDULED);

  MockNM nm1 = new MockNM("h1:1234", 15120, rm1.getResourceTrackerService());
  MockNM nm2 = new MockNM("h2:5678", 15120, rm1.getResourceTrackerService());
  nm1.registerNode();
  nm2.registerNode();

  //kick the scheduling
  nm1.nodeHeartbeat(true);

  // app1 should be allocated now
  rm1.waitForState(attemptId1, RMAppAttemptState.ALLOCATED);
  rm1.waitForState(attemptId2, RMAppAttemptState.SCHEDULED);

  nm2.nodeHeartbeat(true);

  // app2 should be allocated now
  rm1.waitForState(attemptId1, RMAppAttemptState.ALLOCATED);
  rm1.waitForState(attemptId2, RMAppAttemptState.ALLOCATED);

  rm1.stop();
}

Example 2

Source File: TestRMRestart.java From hadoop with Apache License 2.0

4 votes

@Test (timeout = 60000)
public void testRMRestartOnMaxAppAttempts() throws Exception {
  conf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS,
      YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS);

  MemoryRMStateStore memStore = new MemoryRMStateStore();
  memStore.init(conf);
  RMState rmState = memStore.getState();

  Map<ApplicationId, ApplicationStateData> rmAppState =
      rmState.getApplicationState();  
  MockRM rm1 = createMockRM(conf, memStore);
  rm1.start();
  MockNM nm1 =
      new MockNM("127.0.0.1:1234", 15120, rm1.getResourceTrackerService());
  nm1.registerNode();

  // submit an app with maxAppAttempts equals to 1
  RMApp app1 = rm1.submitApp(200, "name", "user",
        new HashMap<ApplicationAccessType, String>(), false, "default", 1,
        null);
  // submit an app with maxAppAttempts equals to -1
  RMApp app2 = rm1.submitApp(200, "name", "user",
        new HashMap<ApplicationAccessType, String>(), false, "default", -1,
        null);

  // assert app1 info is saved
  ApplicationStateData appState = rmAppState.get(app1.getApplicationId());
  Assert.assertNotNull(appState);
  Assert.assertEquals(0, appState.getAttemptCount());
  Assert.assertEquals(appState.getApplicationSubmissionContext()
      .getApplicationId(), app1.getApplicationSubmissionContext()
      .getApplicationId());

  // Allocate the AM
  nm1.nodeHeartbeat(true);
  RMAppAttempt attempt = app1.getCurrentAppAttempt();
  ApplicationAttemptId attemptId1 = attempt.getAppAttemptId();
  rm1.waitForState(attemptId1, RMAppAttemptState.ALLOCATED);
  Assert.assertEquals(1, appState.getAttemptCount());
  ApplicationAttemptStateData attemptState =
                              appState.getAttempt(attemptId1);
  Assert.assertNotNull(attemptState);
  Assert.assertEquals(BuilderUtils.newContainerId(attemptId1, 1), 
                      attemptState.getMasterContainer().getId());

  // Setting AMLivelinessMonitor interval to be 3 Secs.
  conf.setInt(YarnConfiguration.RM_AM_EXPIRY_INTERVAL_MS, 3000);
  // start new RM   
  MockRM rm2 = createMockRM(conf, memStore);
  rm2.start();

  // verify that maxAppAttempts is set to global value
  Assert.assertEquals(2, 
      rm2.getRMContext().getRMApps().get(app2.getApplicationId())
      .getMaxAppAttempts());

  // app1 and app2 are loaded back, but app1 failed because it's
  // hitting max-retry.
  Assert.assertEquals(2, rm2.getRMContext().getRMApps().size());
  rm2.waitForState(app1.getApplicationId(), RMAppState.FAILED);
  rm2.waitForState(app2.getApplicationId(), RMAppState.ACCEPTED);

  // app1 failed state is saved in state store. app2 final saved state is not
  // determined yet.
  Assert.assertEquals(RMAppState.FAILED,
    rmAppState.get(app1.getApplicationId()).getState());
  Assert.assertNull(rmAppState.get(app2.getApplicationId()).getState());
}

Example 3

Source File: TestRMRestart.java From hadoop with Apache License 2.0

4 votes

@Test (timeout = 60000)
public void testAppAttemptTokensRestoredOnRMRestart() throws Exception {
  conf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, 2);
  conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION,
    "kerberos");
  UserGroupInformation.setConfiguration(conf);

  MemoryRMStateStore memStore = new MemoryRMStateStore();
  memStore.init(conf);
  RMState rmState = memStore.getState();

  Map<ApplicationId, ApplicationStateData> rmAppState =
      rmState.getApplicationState();
  MockRM rm1 = new TestSecurityMockRM(conf, memStore);
  rm1.start();
  MockNM nm1 =
      new MockNM("0.0.0.0:4321", 15120, rm1.getResourceTrackerService());
  nm1.registerNode();

  // submit an app
  RMApp app1 =
      rm1.submitApp(200, "name", "user",
        new HashMap<ApplicationAccessType, String>(), "default");

  // assert app info is saved
  ApplicationStateData appState = rmAppState.get(app1.getApplicationId());
  Assert.assertNotNull(appState);

  // Allocate the AM
  nm1.nodeHeartbeat(true);
  RMAppAttempt attempt1 = app1.getCurrentAppAttempt();
  ApplicationAttemptId attemptId1 = attempt1.getAppAttemptId();
  rm1.waitForState(attemptId1, RMAppAttemptState.ALLOCATED);

  // assert attempt info is saved
  ApplicationAttemptStateData attemptState = appState.getAttempt(attemptId1);
  Assert.assertNotNull(attemptState);
  Assert.assertEquals(BuilderUtils.newContainerId(attemptId1, 1),
    attemptState.getMasterContainer().getId());

  // the clientTokenMasterKey that are generated when
  // RMAppAttempt is created,
  byte[] clientTokenMasterKey =
      attempt1.getClientTokenMasterKey().getEncoded();

  // assert application credentials are saved
  Credentials savedCredentials = attemptState.getAppAttemptTokens();
  Assert.assertArrayEquals("client token master key not saved",
      clientTokenMasterKey, savedCredentials.getSecretKey(
          RMStateStore.AM_CLIENT_TOKEN_MASTER_KEY_NAME));

  // start new RM
  MockRM rm2 = new TestSecurityMockRM(conf, memStore);
  rm2.start();

  RMApp loadedApp1 =
      rm2.getRMContext().getRMApps().get(app1.getApplicationId());
  RMAppAttempt loadedAttempt1 = loadedApp1.getRMAppAttempt(attemptId1);

  // assert loaded attempt recovered
  Assert.assertNotNull(loadedAttempt1);

  // assert client token master key is recovered back to api-versioned
  // client token master key
  Assert.assertEquals("client token master key not restored",
      attempt1.getClientTokenMasterKey(),
      loadedAttempt1.getClientTokenMasterKey());

  // assert ClientTokenSecretManager also knows about the key
  Assert.assertArrayEquals(clientTokenMasterKey,
      rm2.getClientToAMTokenSecretManager().getMasterKey(attemptId1)
          .getEncoded());

  // assert AMRMTokenSecretManager also knows about the AMRMToken password
  Token<AMRMTokenIdentifier> amrmToken = loadedAttempt1.getAMRMToken();
  Assert.assertArrayEquals(amrmToken.getPassword(),
    rm2.getRMContext().getAMRMTokenSecretManager().retrievePassword(
      amrmToken.decodeIdentifier()));
}

Example 4

Source File: TestRMRestart.java From hadoop with Apache License 2.0

4 votes

@SuppressWarnings("resource")
@Test (timeout = 60000)
public void testQueueMetricsOnRMRestart() throws Exception {
  conf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS,
      YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS);
  MemoryRMStateStore memStore = new MemoryRMStateStore();
  memStore.init(conf);

  // PHASE 1: create state in an RM
  // start RM
  MockRM rm1 = createMockRM(conf, memStore);
  rm1.start();
  MockNM nm1 =
      new MockNM("127.0.0.1:1234", 15120, rm1.getResourceTrackerService());
  nm1.registerNode();
  QueueMetrics qm1 = rm1.getResourceScheduler().getRootQueueMetrics();
  resetQueueMetrics(qm1);
  assertQueueMetrics(qm1, 0, 0, 0, 0);

  // create app that gets launched and does allocate before RM restart
  RMApp app1 = rm1.submitApp(200);
  // Need to wait first for AppAttempt to be started (RMAppState.ACCEPTED)
  // and then for it to reach RMAppAttemptState.SCHEDULED
  // inorder to ensure appsPending metric is incremented
  rm1.waitForState(app1.getApplicationId(), RMAppState.ACCEPTED);
  RMAppAttempt attempt1 = app1.getCurrentAppAttempt();
  ApplicationAttemptId attemptId1 = attempt1.getAppAttemptId();
  rm1.waitForState(attemptId1, RMAppAttemptState.SCHEDULED);
  assertQueueMetrics(qm1, 1, 1, 0, 0);

  nm1.nodeHeartbeat(true);
  rm1.waitForState(attemptId1, RMAppAttemptState.ALLOCATED);
  MockAM am1 = rm1.sendAMLaunched(attempt1.getAppAttemptId());
  am1.registerAppAttempt();
  am1.allocate("127.0.0.1" , 1000, 1, new ArrayList<ContainerId>());
  nm1.nodeHeartbeat(true);
  List<Container> conts = am1.allocate(new ArrayList<ResourceRequest>(),
      new ArrayList<ContainerId>()).getAllocatedContainers();
  while (conts.size() == 0) {
    nm1.nodeHeartbeat(true);
    conts.addAll(am1.allocate(new ArrayList<ResourceRequest>(),
        new ArrayList<ContainerId>()).getAllocatedContainers());
    Thread.sleep(500);
  }
  assertQueueMetrics(qm1, 1, 0, 1, 0);

  // PHASE 2: create new RM and start from old state
  // create new RM to represent restart and recover state
  MockRM rm2 = createMockRM(conf, memStore);
  QueueMetrics qm2 = rm2.getResourceScheduler().getRootQueueMetrics();
  resetQueueMetrics(qm2);
  assertQueueMetrics(qm2, 0, 0, 0, 0);

  rm2.start();
  nm1.setResourceTrackerService(rm2.getResourceTrackerService());
  // recover app
  RMApp loadedApp1 = rm2.getRMContext().getRMApps().get(app1.getApplicationId());

  nm1.nodeHeartbeat(true);
  nm1 = new MockNM("127.0.0.1:1234", 15120, rm2.getResourceTrackerService());

  NMContainerStatus status =
      TestRMRestart
        .createNMContainerStatus(loadedApp1.getCurrentAppAttempt()
            .getAppAttemptId(), 1, ContainerState.COMPLETE);
  nm1.registerNode(Arrays.asList(status), null);

  while (loadedApp1.getAppAttempts().size() != 2) {
    Thread.sleep(200);
  }
  attempt1 = loadedApp1.getCurrentAppAttempt();
  attemptId1 = attempt1.getAppAttemptId();
  rm2.waitForState(attemptId1, RMAppAttemptState.SCHEDULED);
  assertQueueMetrics(qm2, 1, 1, 0, 0);
  nm1.nodeHeartbeat(true);
  rm2.waitForState(attemptId1, RMAppAttemptState.ALLOCATED);
  assertQueueMetrics(qm2, 1, 0, 1, 0);
  am1 = rm2.sendAMLaunched(attempt1.getAppAttemptId());
  am1.registerAppAttempt();
  am1.allocate("127.0.0.1" , 1000, 3, new ArrayList<ContainerId>());
  nm1.nodeHeartbeat(true);
  conts = am1.allocate(new ArrayList<ResourceRequest>(),
      new ArrayList<ContainerId>()).getAllocatedContainers();
  while (conts.size() == 0) {
    nm1.nodeHeartbeat(true);
    conts.addAll(am1.allocate(new ArrayList<ResourceRequest>(),
        new ArrayList<ContainerId>()).getAllocatedContainers());
    Thread.sleep(500);
  }

  // finish the AMs
  finishApplicationMaster(loadedApp1, rm2, nm1, am1);
  assertQueueMetrics(qm2, 1, 0, 0, 1);
}

Example 5

Source File: TestAMAuthorization.java From hadoop with Apache License 2.0

4 votes

@Test
public void testAuthorizedAccess() throws Exception {
  MyContainerManager containerManager = new MyContainerManager();
  rm =
      new MockRMWithAMS(conf, containerManager);
  rm.start();

  MockNM nm1 = rm.registerNode("localhost:1234", 5120);

  Map<ApplicationAccessType, String> acls =
      new HashMap<ApplicationAccessType, String>(2);
  acls.put(ApplicationAccessType.VIEW_APP, "*");
  RMApp app = rm.submitApp(1024, "appname", "appuser", acls);

  nm1.nodeHeartbeat(true);

  int waitCount = 0;
  while (containerManager.containerTokens == null && waitCount++ < 20) {
    LOG.info("Waiting for AM Launch to happen..");
    Thread.sleep(1000);
  }
  Assert.assertNotNull(containerManager.containerTokens);

  RMAppAttempt attempt = app.getCurrentAppAttempt();
  ApplicationAttemptId applicationAttemptId = attempt.getAppAttemptId();
  waitForLaunchedState(attempt);

  // Create a client to the RM.
  final Configuration conf = rm.getConfig();
  final YarnRPC rpc = YarnRPC.create(conf);

  UserGroupInformation currentUser = UserGroupInformation
      .createRemoteUser(applicationAttemptId.toString());
  Credentials credentials = containerManager.getContainerCredentials();
  final InetSocketAddress rmBindAddress =
      rm.getApplicationMasterService().getBindAddress();
  Token<? extends TokenIdentifier> amRMToken =
      MockRMWithAMS.setupAndReturnAMRMToken(rmBindAddress,
        credentials.getAllTokens());
  currentUser.addToken(amRMToken);
  ApplicationMasterProtocol client = currentUser
      .doAs(new PrivilegedAction<ApplicationMasterProtocol>() {
        @Override
        public ApplicationMasterProtocol run() {
          return (ApplicationMasterProtocol) rpc.getProxy(ApplicationMasterProtocol.class, rm
            .getApplicationMasterService().getBindAddress(), conf);
        }
      });

  RegisterApplicationMasterRequest request = Records
      .newRecord(RegisterApplicationMasterRequest.class);
  RegisterApplicationMasterResponse response =
      client.registerApplicationMaster(request);
  Assert.assertNotNull(response.getClientToAMTokenMasterKey());
  if (UserGroupInformation.isSecurityEnabled()) {
    Assert
      .assertTrue(response.getClientToAMTokenMasterKey().array().length > 0);
  }
  Assert.assertEquals("Register response has bad ACLs", "*",
      response.getApplicationACLs().get(ApplicationAccessType.VIEW_APP));
}

Example 6

Source File: TestAMAuthorization.java From hadoop with Apache License 2.0

4 votes

@Test
public void testUnauthorizedAccess() throws Exception {
  MyContainerManager containerManager = new MyContainerManager();
  rm = new MockRMWithAMS(conf, containerManager);
  rm.start();

  MockNM nm1 = rm.registerNode("localhost:1234", 5120);

  RMApp app = rm.submitApp(1024);

  nm1.nodeHeartbeat(true);

  int waitCount = 0;
  while (containerManager.containerTokens == null && waitCount++ < 40) {
    LOG.info("Waiting for AM Launch to happen..");
    Thread.sleep(1000);
  }
  Assert.assertNotNull(containerManager.containerTokens);

  RMAppAttempt attempt = app.getCurrentAppAttempt();
  ApplicationAttemptId applicationAttemptId = attempt.getAppAttemptId();
  waitForLaunchedState(attempt);

  final Configuration conf = rm.getConfig();
  final YarnRPC rpc = YarnRPC.create(conf);
  final InetSocketAddress serviceAddr = conf.getSocketAddr(
      YarnConfiguration.RM_SCHEDULER_ADDRESS,
      YarnConfiguration.DEFAULT_RM_SCHEDULER_ADDRESS,
      YarnConfiguration.DEFAULT_RM_SCHEDULER_PORT);

  UserGroupInformation currentUser = UserGroupInformation
      .createRemoteUser(applicationAttemptId.toString());

  // First try contacting NM without tokens
  ApplicationMasterProtocol client = currentUser
      .doAs(new PrivilegedAction<ApplicationMasterProtocol>() {
        @Override
        public ApplicationMasterProtocol run() {
          return (ApplicationMasterProtocol) rpc.getProxy(ApplicationMasterProtocol.class,
              serviceAddr, conf);
        }
      });
  
  RegisterApplicationMasterRequest request = Records
      .newRecord(RegisterApplicationMasterRequest.class);
  try {
    client.registerApplicationMaster(request);
    Assert.fail("Should fail with authorization error");
  } catch (Exception e) {
    if (isCause(AccessControlException.class, e)) {
      // Because there are no tokens, the request should be rejected as the
      // server side will assume we are trying simple auth.
      String expectedMessage = "";
      if (UserGroupInformation.isSecurityEnabled()) {
        expectedMessage = "Client cannot authenticate via:[TOKEN]";
      } else {
        expectedMessage =
            "SIMPLE authentication is not enabled.  Available:[TOKEN]";
      }
      Assert.assertTrue(e.getCause().getMessage().contains(expectedMessage)); 
    } else {
      throw e;
    }
  }

  // TODO: Add validation of invalid authorization when there's more data in
  // the AMRMToken
}

Example 7

Source File: TestSchedulerUtils.java From hadoop with Apache License 2.0

4 votes

@Test
public void testValidateResourceBlacklistRequest() throws Exception {

  MyContainerManager containerManager = new MyContainerManager();
  final MockRMWithAMS rm =
      new MockRMWithAMS(new YarnConfiguration(), containerManager);
  rm.start();

  MockNM nm1 = rm.registerNode("localhost:1234", 5120);

  Map<ApplicationAccessType, String> acls =
      new HashMap<ApplicationAccessType, String>(2);
  acls.put(ApplicationAccessType.VIEW_APP, "*");
  RMApp app = rm.submitApp(1024, "appname", "appuser", acls);

  nm1.nodeHeartbeat(true);

  RMAppAttempt attempt = app.getCurrentAppAttempt();
  ApplicationAttemptId applicationAttemptId = attempt.getAppAttemptId();
  waitForLaunchedState(attempt);

  // Create a client to the RM.
  final Configuration conf = rm.getConfig();
  final YarnRPC rpc = YarnRPC.create(conf);

  UserGroupInformation currentUser = 
      UserGroupInformation.createRemoteUser(applicationAttemptId.toString());
  Credentials credentials = containerManager.getContainerCredentials();
  final InetSocketAddress rmBindAddress =
      rm.getApplicationMasterService().getBindAddress();
  Token<? extends TokenIdentifier> amRMToken =
      MockRMWithAMS.setupAndReturnAMRMToken(rmBindAddress,
        credentials.getAllTokens());
  currentUser.addToken(amRMToken);
  ApplicationMasterProtocol client =
      currentUser.doAs(new PrivilegedAction<ApplicationMasterProtocol>() {
        @Override
        public ApplicationMasterProtocol run() {
          return (ApplicationMasterProtocol) rpc.getProxy(
            ApplicationMasterProtocol.class, rmBindAddress, conf);
        }
      });

  RegisterApplicationMasterRequest request = Records
      .newRecord(RegisterApplicationMasterRequest.class);
  client.registerApplicationMaster(request);

  ResourceBlacklistRequest blacklistRequest =
      ResourceBlacklistRequest.newInstance(
          Collections.singletonList(ResourceRequest.ANY), null);

  AllocateRequest allocateRequest =
      AllocateRequest.newInstance(0, 0.0f, null, null, blacklistRequest);
  boolean error = false;
  try {
    client.allocate(allocateRequest);
  } catch (InvalidResourceBlacklistRequestException e) {
    error = true;
  }

  rm.stop();
  
  Assert.assertTrue(
      "Didn't not catch InvalidResourceBlacklistRequestException", error);
}

Example 8

Source File: TestApplicationMasterLauncher.java From hadoop with Apache License 2.0

4 votes

@Test
public void testAMLaunchAndCleanup() throws Exception {
  Logger rootLogger = LogManager.getRootLogger();
  rootLogger.setLevel(Level.DEBUG);
  MyContainerManagerImpl containerManager = new MyContainerManagerImpl();
  MockRMWithCustomAMLauncher rm = new MockRMWithCustomAMLauncher(
      containerManager);
  rm.start();
  MockNM nm1 = rm.registerNode("127.0.0.1:1234", 5120);

  RMApp app = rm.submitApp(2000);

  // kick the scheduling
  nm1.nodeHeartbeat(true);

  int waitCount = 0;
  while (containerManager.launched == false && waitCount++ < 20) {
    LOG.info("Waiting for AM Launch to happen..");
    Thread.sleep(1000);
  }
  Assert.assertTrue(containerManager.launched);

  RMAppAttempt attempt = app.getCurrentAppAttempt();
  ApplicationAttemptId appAttemptId = attempt.getAppAttemptId();
  Assert.assertEquals(appAttemptId.toString(),
      containerManager.attemptIdAtContainerManager);
  Assert.assertEquals(app.getSubmitTime(),
      containerManager.submitTimeAtContainerManager);
  Assert.assertEquals(app.getRMAppAttempt(appAttemptId)
      .getMasterContainer().getId()
      .toString(), containerManager.containerIdAtContainerManager);
  Assert.assertEquals(nm1.getNodeId().toString(),
    containerManager.nmHostAtContainerManager);
  Assert.assertEquals(YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS,
      containerManager.maxAppAttempts);

  MockAM am = new MockAM(rm.getRMContext(), rm
      .getApplicationMasterService(), appAttemptId);
  am.registerAppAttempt();
  am.unregisterAppAttempt();

  //complete the AM container to finish the app normally
  nm1.nodeHeartbeat(attempt.getAppAttemptId(), 1, ContainerState.COMPLETE);
  am.waitForState(RMAppAttemptState.FINISHED);

  waitCount = 0;
  while (containerManager.cleanedup == false && waitCount++ < 20) {
    LOG.info("Waiting for AM Cleanup to happen..");
    Thread.sleep(1000);
  }
  Assert.assertTrue(containerManager.cleanedup);

  am.waitForState(RMAppAttemptState.FINISHED);
  rm.stop();
}

Example 9

Source File: TestRM.java From big-c with Apache License 2.0

4 votes

@Test (timeout = 300000)
public void testActivatingApplicationAfterAddingNM() throws Exception {
  MockRM rm1 = new MockRM(conf);

  // start like normal because state is empty
  rm1.start();

  // app that gets launched
  RMApp app1 = rm1.submitApp(200);

  // app that does not get launched
  RMApp app2 = rm1.submitApp(200);

  // app1 and app2 should be scheduled, but because no resource is available,
  // they are not activated.
  RMAppAttempt attempt1 = app1.getCurrentAppAttempt();
  ApplicationAttemptId attemptId1 = attempt1.getAppAttemptId();
  rm1.waitForState(attemptId1, RMAppAttemptState.SCHEDULED);
  RMAppAttempt attempt2 = app2.getCurrentAppAttempt();
  ApplicationAttemptId attemptId2 = attempt2.getAppAttemptId();
  rm1.waitForState(attemptId2, RMAppAttemptState.SCHEDULED);

  MockNM nm1 = new MockNM("h1:1234", 15120, rm1.getResourceTrackerService());
  MockNM nm2 = new MockNM("h2:5678", 15120, rm1.getResourceTrackerService());
  nm1.registerNode();
  nm2.registerNode();

  //kick the scheduling
  nm1.nodeHeartbeat(true);

  // app1 should be allocated now
  rm1.waitForState(attemptId1, RMAppAttemptState.ALLOCATED);
  rm1.waitForState(attemptId2, RMAppAttemptState.SCHEDULED);

  nm2.nodeHeartbeat(true);

  // app2 should be allocated now
  rm1.waitForState(attemptId1, RMAppAttemptState.ALLOCATED);
  rm1.waitForState(attemptId2, RMAppAttemptState.ALLOCATED);

  rm1.stop();
}

Example 10

Source File: TestRMRestart.java From big-c with Apache License 2.0

4 votes

@Test (timeout = 60000)
public void testRMRestartOnMaxAppAttempts() throws Exception {
  conf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS,
      YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS);

  MemoryRMStateStore memStore = new MemoryRMStateStore();
  memStore.init(conf);
  RMState rmState = memStore.getState();

  Map<ApplicationId, ApplicationStateData> rmAppState =
      rmState.getApplicationState();  
  MockRM rm1 = createMockRM(conf, memStore);
  rm1.start();
  MockNM nm1 =
      new MockNM("127.0.0.1:1234", 15120, rm1.getResourceTrackerService());
  nm1.registerNode();

  // submit an app with maxAppAttempts equals to 1
  RMApp app1 = rm1.submitApp(200, "name", "user",
        new HashMap<ApplicationAccessType, String>(), false, "default", 1,
        null);
  // submit an app with maxAppAttempts equals to -1
  RMApp app2 = rm1.submitApp(200, "name", "user",
        new HashMap<ApplicationAccessType, String>(), false, "default", -1,
        null);

  // assert app1 info is saved
  ApplicationStateData appState = rmAppState.get(app1.getApplicationId());
  Assert.assertNotNull(appState);
  Assert.assertEquals(0, appState.getAttemptCount());
  Assert.assertEquals(appState.getApplicationSubmissionContext()
      .getApplicationId(), app1.getApplicationSubmissionContext()
      .getApplicationId());

  // Allocate the AM
  nm1.nodeHeartbeat(true);
  RMAppAttempt attempt = app1.getCurrentAppAttempt();
  ApplicationAttemptId attemptId1 = attempt.getAppAttemptId();
  rm1.waitForState(attemptId1, RMAppAttemptState.ALLOCATED);
  Assert.assertEquals(1, appState.getAttemptCount());
  ApplicationAttemptStateData attemptState =
                              appState.getAttempt(attemptId1);
  Assert.assertNotNull(attemptState);
  Assert.assertEquals(BuilderUtils.newContainerId(attemptId1, 1), 
                      attemptState.getMasterContainer().getId());

  // Setting AMLivelinessMonitor interval to be 3 Secs.
  conf.setInt(YarnConfiguration.RM_AM_EXPIRY_INTERVAL_MS, 3000);
  // start new RM   
  MockRM rm2 = createMockRM(conf, memStore);
  rm2.start();

  // verify that maxAppAttempts is set to global value
  Assert.assertEquals(2, 
      rm2.getRMContext().getRMApps().get(app2.getApplicationId())
      .getMaxAppAttempts());

  // app1 and app2 are loaded back, but app1 failed because it's
  // hitting max-retry.
  Assert.assertEquals(2, rm2.getRMContext().getRMApps().size());
  rm2.waitForState(app1.getApplicationId(), RMAppState.FAILED);
  rm2.waitForState(app2.getApplicationId(), RMAppState.ACCEPTED);

  // app1 failed state is saved in state store. app2 final saved state is not
  // determined yet.
  Assert.assertEquals(RMAppState.FAILED,
    rmAppState.get(app1.getApplicationId()).getState());
  Assert.assertNull(rmAppState.get(app2.getApplicationId()).getState());
}

Example 11

Source File: TestRMRestart.java From big-c with Apache License 2.0

4 votes

@Test (timeout = 60000)
public void testAppAttemptTokensRestoredOnRMRestart() throws Exception {
  conf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, 2);
  conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION,
    "kerberos");
  UserGroupInformation.setConfiguration(conf);

  MemoryRMStateStore memStore = new MemoryRMStateStore();
  memStore.init(conf);
  RMState rmState = memStore.getState();

  Map<ApplicationId, ApplicationStateData> rmAppState =
      rmState.getApplicationState();
  MockRM rm1 = new TestSecurityMockRM(conf, memStore);
  rm1.start();
  MockNM nm1 =
      new MockNM("0.0.0.0:4321", 15120, rm1.getResourceTrackerService());
  nm1.registerNode();

  // submit an app
  RMApp app1 =
      rm1.submitApp(200, "name", "user",
        new HashMap<ApplicationAccessType, String>(), "default");

  // assert app info is saved
  ApplicationStateData appState = rmAppState.get(app1.getApplicationId());
  Assert.assertNotNull(appState);

  // Allocate the AM
  nm1.nodeHeartbeat(true);
  RMAppAttempt attempt1 = app1.getCurrentAppAttempt();
  ApplicationAttemptId attemptId1 = attempt1.getAppAttemptId();
  rm1.waitForState(attemptId1, RMAppAttemptState.ALLOCATED);

  // assert attempt info is saved
  ApplicationAttemptStateData attemptState = appState.getAttempt(attemptId1);
  Assert.assertNotNull(attemptState);
  Assert.assertEquals(BuilderUtils.newContainerId(attemptId1, 1),
    attemptState.getMasterContainer().getId());

  // the clientTokenMasterKey that are generated when
  // RMAppAttempt is created,
  byte[] clientTokenMasterKey =
      attempt1.getClientTokenMasterKey().getEncoded();

  // assert application credentials are saved
  Credentials savedCredentials = attemptState.getAppAttemptTokens();
  Assert.assertArrayEquals("client token master key not saved",
      clientTokenMasterKey, savedCredentials.getSecretKey(
          RMStateStore.AM_CLIENT_TOKEN_MASTER_KEY_NAME));

  // start new RM
  MockRM rm2 = new TestSecurityMockRM(conf, memStore);
  rm2.start();

  RMApp loadedApp1 =
      rm2.getRMContext().getRMApps().get(app1.getApplicationId());
  RMAppAttempt loadedAttempt1 = loadedApp1.getRMAppAttempt(attemptId1);

  // assert loaded attempt recovered
  Assert.assertNotNull(loadedAttempt1);

  // assert client token master key is recovered back to api-versioned
  // client token master key
  Assert.assertEquals("client token master key not restored",
      attempt1.getClientTokenMasterKey(),
      loadedAttempt1.getClientTokenMasterKey());

  // assert ClientTokenSecretManager also knows about the key
  Assert.assertArrayEquals(clientTokenMasterKey,
      rm2.getClientToAMTokenSecretManager().getMasterKey(attemptId1)
          .getEncoded());

  // assert AMRMTokenSecretManager also knows about the AMRMToken password
  Token<AMRMTokenIdentifier> amrmToken = loadedAttempt1.getAMRMToken();
  Assert.assertArrayEquals(amrmToken.getPassword(),
    rm2.getRMContext().getAMRMTokenSecretManager().retrievePassword(
      amrmToken.decodeIdentifier()));
}

Example 12

Source File: TestRMRestart.java From big-c with Apache License 2.0

4 votes

@SuppressWarnings("resource")
@Test (timeout = 60000)
public void testQueueMetricsOnRMRestart() throws Exception {
  conf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS,
      YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS);
  MemoryRMStateStore memStore = new MemoryRMStateStore();
  memStore.init(conf);

  // PHASE 1: create state in an RM
  // start RM
  MockRM rm1 = createMockRM(conf, memStore);
  rm1.start();
  MockNM nm1 =
      new MockNM("127.0.0.1:1234", 15120, rm1.getResourceTrackerService());
  nm1.registerNode();
  QueueMetrics qm1 = rm1.getResourceScheduler().getRootQueueMetrics();
  resetQueueMetrics(qm1);
  assertQueueMetrics(qm1, 0, 0, 0, 0);

  // create app that gets launched and does allocate before RM restart
  RMApp app1 = rm1.submitApp(200);
  // Need to wait first for AppAttempt to be started (RMAppState.ACCEPTED)
  // and then for it to reach RMAppAttemptState.SCHEDULED
  // inorder to ensure appsPending metric is incremented
  rm1.waitForState(app1.getApplicationId(), RMAppState.ACCEPTED);
  RMAppAttempt attempt1 = app1.getCurrentAppAttempt();
  ApplicationAttemptId attemptId1 = attempt1.getAppAttemptId();
  rm1.waitForState(attemptId1, RMAppAttemptState.SCHEDULED);
  assertQueueMetrics(qm1, 1, 1, 0, 0);

  nm1.nodeHeartbeat(true);
  rm1.waitForState(attemptId1, RMAppAttemptState.ALLOCATED);
  MockAM am1 = rm1.sendAMLaunched(attempt1.getAppAttemptId());
  am1.registerAppAttempt();
  am1.allocate("127.0.0.1" , 1000, 1, new ArrayList<ContainerId>());
  nm1.nodeHeartbeat(true);
  List<Container> conts = am1.allocate(new ArrayList<ResourceRequest>(),
      new ArrayList<ContainerId>()).getAllocatedContainers();
  while (conts.size() == 0) {
    nm1.nodeHeartbeat(true);
    conts.addAll(am1.allocate(new ArrayList<ResourceRequest>(),
        new ArrayList<ContainerId>()).getAllocatedContainers());
    Thread.sleep(500);
  }
  assertQueueMetrics(qm1, 1, 0, 1, 0);

  // PHASE 2: create new RM and start from old state
  // create new RM to represent restart and recover state
  MockRM rm2 = createMockRM(conf, memStore);
  QueueMetrics qm2 = rm2.getResourceScheduler().getRootQueueMetrics();
  resetQueueMetrics(qm2);
  assertQueueMetrics(qm2, 0, 0, 0, 0);

  rm2.start();
  nm1.setResourceTrackerService(rm2.getResourceTrackerService());
  // recover app
  RMApp loadedApp1 = rm2.getRMContext().getRMApps().get(app1.getApplicationId());

  nm1.nodeHeartbeat(true);
  nm1 = new MockNM("127.0.0.1:1234", 15120, rm2.getResourceTrackerService());

  NMContainerStatus status =
      TestRMRestart
        .createNMContainerStatus(loadedApp1.getCurrentAppAttempt()
            .getAppAttemptId(), 1, ContainerState.COMPLETE);
  nm1.registerNode(Arrays.asList(status), null);

  while (loadedApp1.getAppAttempts().size() != 2) {
    Thread.sleep(200);
  }
  attempt1 = loadedApp1.getCurrentAppAttempt();
  attemptId1 = attempt1.getAppAttemptId();
  rm2.waitForState(attemptId1, RMAppAttemptState.SCHEDULED);
  assertQueueMetrics(qm2, 1, 1, 0, 0);
  nm1.nodeHeartbeat(true);
  rm2.waitForState(attemptId1, RMAppAttemptState.ALLOCATED);
  assertQueueMetrics(qm2, 1, 0, 1, 0);
  am1 = rm2.sendAMLaunched(attempt1.getAppAttemptId());
  am1.registerAppAttempt();
  am1.allocate("127.0.0.1" , 1000, 3, new ArrayList<ContainerId>());
  nm1.nodeHeartbeat(true);
  conts = am1.allocate(new ArrayList<ResourceRequest>(),
      new ArrayList<ContainerId>()).getAllocatedContainers();
  while (conts.size() == 0) {
    nm1.nodeHeartbeat(true);
    conts.addAll(am1.allocate(new ArrayList<ResourceRequest>(),
        new ArrayList<ContainerId>()).getAllocatedContainers());
    Thread.sleep(500);
  }

  // finish the AMs
  finishApplicationMaster(loadedApp1, rm2, nm1, am1);
  assertQueueMetrics(qm2, 1, 0, 0, 1);
}

Example 13

Source File: TestAMAuthorization.java From big-c with Apache License 2.0

4 votes

@Test
public void testAuthorizedAccess() throws Exception {
  MyContainerManager containerManager = new MyContainerManager();
  rm =
      new MockRMWithAMS(conf, containerManager);
  rm.start();

  MockNM nm1 = rm.registerNode("localhost:1234", 5120);

  Map<ApplicationAccessType, String> acls =
      new HashMap<ApplicationAccessType, String>(2);
  acls.put(ApplicationAccessType.VIEW_APP, "*");
  RMApp app = rm.submitApp(1024, "appname", "appuser", acls);

  nm1.nodeHeartbeat(true);

  int waitCount = 0;
  while (containerManager.containerTokens == null && waitCount++ < 20) {
    LOG.info("Waiting for AM Launch to happen..");
    Thread.sleep(1000);
  }
  Assert.assertNotNull(containerManager.containerTokens);

  RMAppAttempt attempt = app.getCurrentAppAttempt();
  ApplicationAttemptId applicationAttemptId = attempt.getAppAttemptId();
  waitForLaunchedState(attempt);

  // Create a client to the RM.
  final Configuration conf = rm.getConfig();
  final YarnRPC rpc = YarnRPC.create(conf);

  UserGroupInformation currentUser = UserGroupInformation
      .createRemoteUser(applicationAttemptId.toString());
  Credentials credentials = containerManager.getContainerCredentials();
  final InetSocketAddress rmBindAddress =
      rm.getApplicationMasterService().getBindAddress();
  Token<? extends TokenIdentifier> amRMToken =
      MockRMWithAMS.setupAndReturnAMRMToken(rmBindAddress,
        credentials.getAllTokens());
  currentUser.addToken(amRMToken);
  ApplicationMasterProtocol client = currentUser
      .doAs(new PrivilegedAction<ApplicationMasterProtocol>() {
        @Override
        public ApplicationMasterProtocol run() {
          return (ApplicationMasterProtocol) rpc.getProxy(ApplicationMasterProtocol.class, rm
            .getApplicationMasterService().getBindAddress(), conf);
        }
      });

  RegisterApplicationMasterRequest request = Records
      .newRecord(RegisterApplicationMasterRequest.class);
  RegisterApplicationMasterResponse response =
      client.registerApplicationMaster(request);
  Assert.assertNotNull(response.getClientToAMTokenMasterKey());
  if (UserGroupInformation.isSecurityEnabled()) {
    Assert
      .assertTrue(response.getClientToAMTokenMasterKey().array().length > 0);
  }
  Assert.assertEquals("Register response has bad ACLs", "*",
      response.getApplicationACLs().get(ApplicationAccessType.VIEW_APP));
}

Example 14

Source File: TestAMAuthorization.java From big-c with Apache License 2.0

4 votes

@Test
public void testUnauthorizedAccess() throws Exception {
  MyContainerManager containerManager = new MyContainerManager();
  rm = new MockRMWithAMS(conf, containerManager);
  rm.start();

  MockNM nm1 = rm.registerNode("localhost:1234", 5120);

  RMApp app = rm.submitApp(1024);

  nm1.nodeHeartbeat(true);

  int waitCount = 0;
  while (containerManager.containerTokens == null && waitCount++ < 40) {
    LOG.info("Waiting for AM Launch to happen..");
    Thread.sleep(1000);
  }
  Assert.assertNotNull(containerManager.containerTokens);

  RMAppAttempt attempt = app.getCurrentAppAttempt();
  ApplicationAttemptId applicationAttemptId = attempt.getAppAttemptId();
  waitForLaunchedState(attempt);

  final Configuration conf = rm.getConfig();
  final YarnRPC rpc = YarnRPC.create(conf);
  final InetSocketAddress serviceAddr = conf.getSocketAddr(
      YarnConfiguration.RM_SCHEDULER_ADDRESS,
      YarnConfiguration.DEFAULT_RM_SCHEDULER_ADDRESS,
      YarnConfiguration.DEFAULT_RM_SCHEDULER_PORT);

  UserGroupInformation currentUser = UserGroupInformation
      .createRemoteUser(applicationAttemptId.toString());

  // First try contacting NM without tokens
  ApplicationMasterProtocol client = currentUser
      .doAs(new PrivilegedAction<ApplicationMasterProtocol>() {
        @Override
        public ApplicationMasterProtocol run() {
          return (ApplicationMasterProtocol) rpc.getProxy(ApplicationMasterProtocol.class,
              serviceAddr, conf);
        }
      });
  
  RegisterApplicationMasterRequest request = Records
      .newRecord(RegisterApplicationMasterRequest.class);
  try {
    client.registerApplicationMaster(request);
    Assert.fail("Should fail with authorization error");
  } catch (Exception e) {
    if (isCause(AccessControlException.class, e)) {
      // Because there are no tokens, the request should be rejected as the
      // server side will assume we are trying simple auth.
      String expectedMessage = "";
      if (UserGroupInformation.isSecurityEnabled()) {
        expectedMessage = "Client cannot authenticate via:[TOKEN]";
      } else {
        expectedMessage =
            "SIMPLE authentication is not enabled.  Available:[TOKEN]";
      }
      Assert.assertTrue(e.getCause().getMessage().contains(expectedMessage)); 
    } else {
      throw e;
    }
  }

  // TODO: Add validation of invalid authorization when there's more data in
  // the AMRMToken
}

Example 15

Source File: TestSchedulerUtils.java From big-c with Apache License 2.0

4 votes

@Test
public void testValidateResourceBlacklistRequest() throws Exception {

  MyContainerManager containerManager = new MyContainerManager();
  final MockRMWithAMS rm =
      new MockRMWithAMS(new YarnConfiguration(), containerManager);
  rm.start();

  MockNM nm1 = rm.registerNode("localhost:1234", 5120);

  Map<ApplicationAccessType, String> acls =
      new HashMap<ApplicationAccessType, String>(2);
  acls.put(ApplicationAccessType.VIEW_APP, "*");
  RMApp app = rm.submitApp(1024, "appname", "appuser", acls);

  nm1.nodeHeartbeat(true);

  RMAppAttempt attempt = app.getCurrentAppAttempt();
  ApplicationAttemptId applicationAttemptId = attempt.getAppAttemptId();
  waitForLaunchedState(attempt);

  // Create a client to the RM.
  final Configuration conf = rm.getConfig();
  final YarnRPC rpc = YarnRPC.create(conf);

  UserGroupInformation currentUser = 
      UserGroupInformation.createRemoteUser(applicationAttemptId.toString());
  Credentials credentials = containerManager.getContainerCredentials();
  final InetSocketAddress rmBindAddress =
      rm.getApplicationMasterService().getBindAddress();
  Token<? extends TokenIdentifier> amRMToken =
      MockRMWithAMS.setupAndReturnAMRMToken(rmBindAddress,
        credentials.getAllTokens());
  currentUser.addToken(amRMToken);
  ApplicationMasterProtocol client =
      currentUser.doAs(new PrivilegedAction<ApplicationMasterProtocol>() {
        @Override
        public ApplicationMasterProtocol run() {
          return (ApplicationMasterProtocol) rpc.getProxy(
            ApplicationMasterProtocol.class, rmBindAddress, conf);
        }
      });

  RegisterApplicationMasterRequest request = Records
      .newRecord(RegisterApplicationMasterRequest.class);
  client.registerApplicationMaster(request);

  ResourceBlacklistRequest blacklistRequest =
      ResourceBlacklistRequest.newInstance(
          Collections.singletonList(ResourceRequest.ANY), null);

  AllocateRequest allocateRequest =
      AllocateRequest.newInstance(0, 0.0f, null, null, blacklistRequest);
  boolean error = false;
  try {
    client.allocate(allocateRequest);
  } catch (InvalidResourceBlacklistRequestException e) {
    error = true;
  }

  rm.stop();
  
  Assert.assertTrue(
      "Didn't not catch InvalidResourceBlacklistRequestException", error);
}

Example 16

Source File: TestApplicationMasterLauncher.java From big-c with Apache License 2.0

4 votes

@Test
public void testAMLaunchAndCleanup() throws Exception {
  Logger rootLogger = LogManager.getRootLogger();
  rootLogger.setLevel(Level.DEBUG);
  MyContainerManagerImpl containerManager = new MyContainerManagerImpl();
  MockRMWithCustomAMLauncher rm = new MockRMWithCustomAMLauncher(
      containerManager);
  rm.start();
  MockNM nm1 = rm.registerNode("127.0.0.1:1234", 5120);

  RMApp app = rm.submitApp(2000);

  // kick the scheduling
  nm1.nodeHeartbeat(true);

  int waitCount = 0;
  while (containerManager.launched == false && waitCount++ < 20) {
    LOG.info("Waiting for AM Launch to happen..");
    Thread.sleep(1000);
  }
  Assert.assertTrue(containerManager.launched);

  RMAppAttempt attempt = app.getCurrentAppAttempt();
  ApplicationAttemptId appAttemptId = attempt.getAppAttemptId();
  Assert.assertEquals(appAttemptId.toString(),
      containerManager.attemptIdAtContainerManager);
  Assert.assertEquals(app.getSubmitTime(),
      containerManager.submitTimeAtContainerManager);
  Assert.assertEquals(app.getRMAppAttempt(appAttemptId)
      .getMasterContainer().getId()
      .toString(), containerManager.containerIdAtContainerManager);
  Assert.assertEquals(nm1.getNodeId().toString(),
    containerManager.nmHostAtContainerManager);
  Assert.assertEquals(YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS,
      containerManager.maxAppAttempts);

  MockAM am = new MockAM(rm.getRMContext(), rm
      .getApplicationMasterService(), appAttemptId);
  am.registerAppAttempt();
  am.unregisterAppAttempt();

  //complete the AM container to finish the app normally
  nm1.nodeHeartbeat(attempt.getAppAttemptId(), 1, ContainerState.COMPLETE);
  am.waitForState(RMAppAttemptState.FINISHED);

  waitCount = 0;
  while (containerManager.cleanedup == false && waitCount++ < 20) {
    LOG.info("Waiting for AM Cleanup to happen..");
    Thread.sleep(1000);
  }
  Assert.assertTrue(containerManager.cleanedup);

  am.waitForState(RMAppAttemptState.FINISHED);
  rm.stop();
}