Java Code Examples for org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse#getCompletedContainersStatuses()

The following examples show how to use org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse#getCompletedContainersStatuses() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example 1

Source File: TestApplicationCleanup.java From hadoop with Apache License 2.0

4 votes

@Test(timeout = 60000)
public void testProcessingNMContainerStatusesOnNMRestart() throws Exception {
  conf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, 1);
  MemoryRMStateStore memStore = new MemoryRMStateStore();
  memStore.init(conf);

  // 1. Start the cluster-RM,NM,Submit app with 1024MB,Launch & register AM
  MockRM rm1 = new MockRM(conf, memStore);
  rm1.start();
  int nmMemory = 8192;
  int amMemory = 1024;
  int containerMemory = 2048;
  MockNM nm1 =
      new MockNM("127.0.0.1:1234", nmMemory, rm1.getResourceTrackerService());
  nm1.registerNode();

  RMApp app0 = rm1.submitApp(amMemory);
  MockAM am0 = MockRM.launchAndRegisterAM(app0, rm1, nm1);

  // 2. AM sends ResourceRequest for 1 container with memory 2048MB.
  int noOfContainers = 1;
  List<Container> allocateContainers =
      am0.allocateAndWaitForContainers(noOfContainers, containerMemory, nm1);

  // 3. Verify for number of container allocated by RM
  Assert.assertEquals(noOfContainers, allocateContainers.size());
  Container container = allocateContainers.get(0);

  nm1.nodeHeartbeat(am0.getApplicationAttemptId(), 1, ContainerState.RUNNING);
  nm1.nodeHeartbeat(am0.getApplicationAttemptId(), container.getId()
      .getContainerId(), ContainerState.RUNNING);

  rm1.waitForState(app0.getApplicationId(), RMAppState.RUNNING);

  // 4. Verify Memory Usage by cluster, it should be 3072. AM memory +
  // requested memory. 1024 + 2048=3072
  ResourceScheduler rs = rm1.getRMContext().getScheduler();
  int allocatedMB = rs.getRootQueueMetrics().getAllocatedMB();
  Assert.assertEquals(amMemory + containerMemory, allocatedMB);

  // 5. Re-register NM by sending completed container status
  List<NMContainerStatus> nMContainerStatusForApp =
      createNMContainerStatusForApp(am0);
  nm1.registerNode(nMContainerStatusForApp,
      Arrays.asList(app0.getApplicationId()));

  waitForClusterMemory(nm1, rs, amMemory);

  // 6. Verify for Memory Used, it should be 1024
  Assert.assertEquals(amMemory, rs.getRootQueueMetrics().getAllocatedMB());

  // 7. Send AM heatbeat to RM. Allocated response should contain completed
  // container
  AllocateRequest req =
      AllocateRequest.newInstance(0, 0F, new ArrayList<ResourceRequest>(),
          new ArrayList<ContainerId>(), null);
  AllocateResponse allocate = am0.allocate(req);
  List<ContainerStatus> completedContainersStatuses =
      allocate.getCompletedContainersStatuses();
  Assert.assertEquals(noOfContainers, completedContainersStatuses.size());

  // Application clean up should happen Cluster memory used is 0
  nm1.nodeHeartbeat(am0.getApplicationAttemptId(), 1, ContainerState.COMPLETE);
  waitForClusterMemory(nm1, rs, 0);

  rm1.stop();
}

Example 2

Source File: TestWorkPreservingRMRestart.java From hadoop with Apache License 2.0

4 votes

@Test (timeout = 50000)
public void testReleasedContainerNotRecovered() throws Exception {
  MemoryRMStateStore memStore = new MemoryRMStateStore();
  memStore.init(conf);
  rm1 = new MockRM(conf, memStore);
  MockNM nm1 = new MockNM("h1:1234", 15120, rm1.getResourceTrackerService());
  nm1.registerNode();
  rm1.start();

  RMApp app1 = rm1.submitApp(1024);
  final MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1);

  // Re-start RM
  conf.setInt(YarnConfiguration.RM_NM_EXPIRY_INTERVAL_MS, 8000);
  rm2 = new MockRM(conf, memStore);
  rm2.start();
  nm1.setResourceTrackerService(rm2.getResourceTrackerService());
  rm2.waitForState(app1.getApplicationId(), RMAppState.ACCEPTED);
  am1.setAMRMProtocol(rm2.getApplicationMasterService(), rm2.getRMContext());
  am1.registerAppAttempt(true);

  // try to release a container before the container is actually recovered.
  final ContainerId runningContainer =
      ContainerId.newContainerId(am1.getApplicationAttemptId(), 2);
  am1.allocate(null, Arrays.asList(runningContainer));

  // send container statuses to recover the containers
  List<NMContainerStatus> containerStatuses =
      createNMContainerStatusForApp(am1);
  nm1.registerNode(containerStatuses, null);

  // only the am container should be recovered.
  waitForNumContainersToRecover(1, rm2, am1.getApplicationAttemptId());

  final AbstractYarnScheduler scheduler =
      (AbstractYarnScheduler) rm2.getResourceScheduler();
  // cached release request is cleaned.
  // assertFalse(scheduler.getPendingRelease().contains(runningContainer));

  AllocateResponse response = am1.allocate(null, null);
  // AM gets notified of the completed container.
  boolean receivedCompletedContainer = false;
  for (ContainerStatus status : response.getCompletedContainersStatuses()) {
    if (status.getContainerId().equals(runningContainer)) {
      receivedCompletedContainer = true;
    }
  }
  assertTrue(receivedCompletedContainer);

  GenericTestUtils.waitFor(new Supplier<Boolean>() {
    public Boolean get() {
      // release cache is cleaned up and previous running container is not
      // recovered
      return scheduler.getApplicationAttempt(am1.getApplicationAttemptId())
        .getPendingRelease().isEmpty()
          && scheduler.getRMContainer(runningContainer) == null;
    }
  }, 1000, 20000);
}

Example 3

Source File: TestApplicationCleanup.java From big-c with Apache License 2.0

4 votes

@Test(timeout = 60000)
public void testProcessingNMContainerStatusesOnNMRestart() throws Exception {
  conf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, 1);
  MemoryRMStateStore memStore = new MemoryRMStateStore();
  memStore.init(conf);

  // 1. Start the cluster-RM,NM,Submit app with 1024MB,Launch & register AM
  MockRM rm1 = new MockRM(conf, memStore);
  rm1.start();
  int nmMemory = 8192;
  int amMemory = 1024;
  int containerMemory = 2048;
  MockNM nm1 =
      new MockNM("127.0.0.1:1234", nmMemory, rm1.getResourceTrackerService());
  nm1.registerNode();

  RMApp app0 = rm1.submitApp(amMemory);
  MockAM am0 = MockRM.launchAndRegisterAM(app0, rm1, nm1);

  // 2. AM sends ResourceRequest for 1 container with memory 2048MB.
  int noOfContainers = 1;
  List<Container> allocateContainers =
      am0.allocateAndWaitForContainers(noOfContainers, containerMemory, nm1);

  // 3. Verify for number of container allocated by RM
  Assert.assertEquals(noOfContainers, allocateContainers.size());
  Container container = allocateContainers.get(0);

  nm1.nodeHeartbeat(am0.getApplicationAttemptId(), 1, ContainerState.RUNNING);
  nm1.nodeHeartbeat(am0.getApplicationAttemptId(), container.getId()
      .getContainerId(), ContainerState.RUNNING);

  rm1.waitForState(app0.getApplicationId(), RMAppState.RUNNING);

  // 4. Verify Memory Usage by cluster, it should be 3072. AM memory +
  // requested memory. 1024 + 2048=3072
  ResourceScheduler rs = rm1.getRMContext().getScheduler();
  int allocatedMB = rs.getRootQueueMetrics().getAllocatedMB();
  Assert.assertEquals(amMemory + containerMemory, allocatedMB);

  // 5. Re-register NM by sending completed container status
  List<NMContainerStatus> nMContainerStatusForApp =
      createNMContainerStatusForApp(am0);
  nm1.registerNode(nMContainerStatusForApp,
      Arrays.asList(app0.getApplicationId()));

  waitForClusterMemory(nm1, rs, amMemory);

  // 6. Verify for Memory Used, it should be 1024
  Assert.assertEquals(amMemory, rs.getRootQueueMetrics().getAllocatedMB());

  // 7. Send AM heatbeat to RM. Allocated response should contain completed
  // container
  AllocateRequest req =
      AllocateRequest.newInstance(0, 0F, new ArrayList<ResourceRequest>(),
          new ArrayList<ContainerId>(), null);
  AllocateResponse allocate = am0.allocate(req);
  List<ContainerStatus> completedContainersStatuses =
      allocate.getCompletedContainersStatuses();
  Assert.assertEquals(noOfContainers, completedContainersStatuses.size());

  // Application clean up should happen Cluster memory used is 0
  nm1.nodeHeartbeat(am0.getApplicationAttemptId(), 1, ContainerState.COMPLETE);
  waitForClusterMemory(nm1, rs, 0);

  rm1.stop();
}

Example 4

Source File: TestWorkPreservingRMRestart.java From big-c with Apache License 2.0

4 votes

@Test (timeout = 50000)
public void testReleasedContainerNotRecovered() throws Exception {
  MemoryRMStateStore memStore = new MemoryRMStateStore();
  memStore.init(conf);
  rm1 = new MockRM(conf, memStore);
  MockNM nm1 = new MockNM("h1:1234", 15120, rm1.getResourceTrackerService());
  nm1.registerNode();
  rm1.start();

  RMApp app1 = rm1.submitApp(1024);
  final MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1);

  // Re-start RM
  conf.setInt(YarnConfiguration.RM_NM_EXPIRY_INTERVAL_MS, 8000);
  rm2 = new MockRM(conf, memStore);
  rm2.start();
  nm1.setResourceTrackerService(rm2.getResourceTrackerService());
  rm2.waitForState(app1.getApplicationId(), RMAppState.ACCEPTED);
  am1.setAMRMProtocol(rm2.getApplicationMasterService(), rm2.getRMContext());
  am1.registerAppAttempt(true);

  // try to release a container before the container is actually recovered.
  final ContainerId runningContainer =
      ContainerId.newContainerId(am1.getApplicationAttemptId(), 2);
  am1.allocate(null, Arrays.asList(runningContainer));

  // send container statuses to recover the containers
  List<NMContainerStatus> containerStatuses =
      createNMContainerStatusForApp(am1);
  nm1.registerNode(containerStatuses, null);

  // only the am container should be recovered.
  waitForNumContainersToRecover(1, rm2, am1.getApplicationAttemptId());

  final AbstractYarnScheduler scheduler =
      (AbstractYarnScheduler) rm2.getResourceScheduler();
  // cached release request is cleaned.
  // assertFalse(scheduler.getPendingRelease().contains(runningContainer));

  AllocateResponse response = am1.allocate(null, null);
  // AM gets notified of the completed container.
  boolean receivedCompletedContainer = false;
  for (ContainerStatus status : response.getCompletedContainersStatuses()) {
    if (status.getContainerId().equals(runningContainer)) {
      receivedCompletedContainer = true;
    }
  }
  assertTrue(receivedCompletedContainer);

  GenericTestUtils.waitFor(new Supplier<Boolean>() {
    public Boolean get() {
      // release cache is cleaned up and previous running container is not
      // recovered
      return scheduler.getApplicationAttempt(am1.getApplicationAttemptId())
        .getPendingRelease().isEmpty()
          && scheduler.getRMContainer(runningContainer) == null;
    }
  }, 1000, 20000);
}