Java Code Examples for org.apache.hadoop.yarn.server.utils.BuilderUtils#newNodeId()

The following examples show how to use org.apache.hadoop.yarn.server.utils.BuilderUtils#newNodeId() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: MockNM.java    From big-c with Apache License 2.0 5 votes vote down vote up 
public MockNM(String nodeIdStr, int memory, int vcores,
    ResourceTrackerService resourceTracker, String version) {
  this.memory = memory;
  this.vCores = vcores;
  this.resourceTracker = resourceTracker;
  this.version = version;
  String[] splits = nodeIdStr.split(":");
  nodeId = BuilderUtils.newNodeId(splits[0], Integer.parseInt(splits[1]));
}
Example 2
Source File: TestRMNodeTransitions.java    From big-c with Apache License 2.0 5 votes vote down vote up 
private RMNodeImpl getRebootedNode() {
  NodeId nodeId = BuilderUtils.newNodeId("localhost", 0);
  Resource capability = Resource.newInstance(4096, 4);
  RMNodeImpl node = new RMNodeImpl(nodeId, rmContext,null, 0, 0,
      null, capability, null);
  node.handle(new RMNodeStartedEvent(node.getNodeID(), null, null));
  Assert.assertEquals(NodeState.RUNNING, node.getState());
  node.handle(new RMNodeEvent(node.getNodeID(), RMNodeEventType.REBOOTING));
  Assert.assertEquals(NodeState.REBOOTED, node.getState());
  return node;
}
Example 3
Source File: TestRMNodeTransitions.java    From hadoop with Apache License 2.0 5 votes vote down vote up 
private RMNodeImpl getRunningNode(String nmVersion) {
  NodeId nodeId = BuilderUtils.newNodeId("localhost", 0);
  Resource capability = Resource.newInstance(4096, 4, 4);
  RMNodeImpl node = new RMNodeImpl(nodeId, rmContext,null, 0, 0,
      null, capability, nmVersion);
  node.handle(new RMNodeStartedEvent(node.getNodeID(), null, null));
  Assert.assertEquals(NodeState.RUNNING, node.getState());
  return node;
}
Example 4
Source File: TestRMNodeTransitions.java    From hadoop with Apache License 2.0 5 votes vote down vote up 
private RMNodeImpl getRebootedNode() {
  NodeId nodeId = BuilderUtils.newNodeId("localhost", 0);
  Resource capability = Resource.newInstance(4096, 4, 4);
  RMNodeImpl node = new RMNodeImpl(nodeId, rmContext,null, 0, 0,
      null, capability, null);
  node.handle(new RMNodeStartedEvent(node.getNodeID(), null, null));
  Assert.assertEquals(NodeState.RUNNING, node.getState());
  node.handle(new RMNodeEvent(node.getNodeID(), RMNodeEventType.REBOOTING));
  Assert.assertEquals(NodeState.REBOOTED, node.getState());
  return node;
}
Example 5
Source File: MockNM.java    From hadoop with Apache License 2.0 5 votes vote down vote up 
public MockNM(String nodeIdStr, int memory, int vcores, int gcores,
    ResourceTrackerService resourceTracker, String version) {
  this.memory = memory;
  this.vCores = vcores;
  this.gCores = gcores;
  this.resourceTracker = resourceTracker;
  this.version = version;
  String[] splits = nodeIdStr.split(":");
  nodeId = BuilderUtils.newNodeId(splits[0], Integer.parseInt(splits[1]));
}
Example 6
Source File: TestRMNodeTransitions.java    From big-c with Apache License 2.0 5 votes vote down vote up 
private RMNodeImpl getRunningNode(String nmVersion) {
  NodeId nodeId = BuilderUtils.newNodeId("localhost", 0);
  Resource capability = Resource.newInstance(4096, 4);
  RMNodeImpl node = new RMNodeImpl(nodeId, rmContext,null, 0, 0,
      null, capability, nmVersion);
  node.handle(new RMNodeStartedEvent(node.getNodeID(), null, null));
  Assert.assertEquals(NodeState.RUNNING, node.getState());
  return node;
}
Example 7
Source File: TestRMContainerImpl.java    From big-c with Apache License 2.0 4 votes vote down vote up 
@Test
public void testReleaseWhileRunning() {

  DrainDispatcher drainDispatcher = new DrainDispatcher();
  EventHandler<RMAppAttemptEvent> appAttemptEventHandler = mock(EventHandler.class);
  EventHandler generic = mock(EventHandler.class);
  drainDispatcher.register(RMAppAttemptEventType.class,
      appAttemptEventHandler);
  drainDispatcher.register(RMNodeEventType.class, generic);
  drainDispatcher.init(new YarnConfiguration());
  drainDispatcher.start();
  NodeId nodeId = BuilderUtils.newNodeId("host", 3425);
  ApplicationId appId = BuilderUtils.newApplicationId(1, 1);
  ApplicationAttemptId appAttemptId = BuilderUtils.newApplicationAttemptId(
      appId, 1);
  ContainerId containerId = BuilderUtils.newContainerId(appAttemptId, 1);
  ContainerAllocationExpirer expirer = mock(ContainerAllocationExpirer.class);

  Resource resource = BuilderUtils.newResource(512, 1);
  Priority priority = BuilderUtils.newPriority(5);

  Container container = BuilderUtils.newContainer(containerId, nodeId,
      "host:3465", resource, priority, null);
  ConcurrentMap<ApplicationId, RMApp> rmApps =
      spy(new ConcurrentHashMap<ApplicationId, RMApp>());
  RMApp rmApp = mock(RMApp.class);
  when(rmApp.getRMAppAttempt((ApplicationAttemptId)Matchers.any())).thenReturn(null);
  Mockito.doReturn(rmApp).when(rmApps).get((ApplicationId)Matchers.any());

  RMApplicationHistoryWriter writer = mock(RMApplicationHistoryWriter.class);
  SystemMetricsPublisher publisher = mock(SystemMetricsPublisher.class);
  RMContext rmContext = mock(RMContext.class);
  when(rmContext.getDispatcher()).thenReturn(drainDispatcher);
  when(rmContext.getContainerAllocationExpirer()).thenReturn(expirer);
  when(rmContext.getRMApplicationHistoryWriter()).thenReturn(writer);
  when(rmContext.getRMApps()).thenReturn(rmApps);
  when(rmContext.getSystemMetricsPublisher()).thenReturn(publisher);
  when(rmContext.getYarnConfiguration()).thenReturn(new YarnConfiguration());
  RMContainer rmContainer = new RMContainerImpl(container, appAttemptId,
      nodeId, "user", rmContext);

  assertEquals(RMContainerState.NEW, rmContainer.getState());
  assertEquals(resource, rmContainer.getAllocatedResource());
  assertEquals(nodeId, rmContainer.getAllocatedNode());
  assertEquals(priority, rmContainer.getAllocatedPriority());
  verify(writer).containerStarted(any(RMContainer.class));
  verify(publisher).containerCreated(any(RMContainer.class), anyLong());

  rmContainer.handle(new RMContainerEvent(containerId,
      RMContainerEventType.START));
  drainDispatcher.await();
  assertEquals(RMContainerState.ALLOCATED, rmContainer.getState());
  rmContainer.handle(new RMContainerEvent(containerId,
      RMContainerEventType.ACQUIRED));
  drainDispatcher.await();
  assertEquals(RMContainerState.ACQUIRED, rmContainer.getState());

  rmContainer.handle(new RMContainerEvent(containerId,
      RMContainerEventType.LAUNCHED));
  drainDispatcher.await();
  assertEquals(RMContainerState.RUNNING, rmContainer.getState());
  assertEquals("http://host:3465/node/containerlogs/container_1_0001_01_000001/user",
      rmContainer.getLogURL());

  // In RUNNING state. Verify RELEASED and associated actions.
  reset(appAttemptEventHandler);
  ContainerStatus containerStatus = SchedulerUtils
      .createAbnormalContainerStatus(containerId,
          SchedulerUtils.RELEASED_CONTAINER);
  rmContainer.handle(new RMContainerFinishedEvent(containerId,
      containerStatus, RMContainerEventType.RELEASED));
  drainDispatcher.await();
  assertEquals(RMContainerState.RELEASED, rmContainer.getState());
  assertEquals(SchedulerUtils.RELEASED_CONTAINER,
      rmContainer.getDiagnosticsInfo());
  assertEquals(ContainerExitStatus.ABORTED,
      rmContainer.getContainerExitStatus());
  assertEquals(ContainerState.COMPLETE, rmContainer.getContainerState());
  verify(writer).containerFinished(any(RMContainer.class));
  verify(publisher).containerFinished(any(RMContainer.class), anyLong());

  ArgumentCaptor<RMAppAttemptContainerFinishedEvent> captor = ArgumentCaptor
      .forClass(RMAppAttemptContainerFinishedEvent.class);
  verify(appAttemptEventHandler).handle(captor.capture());
  RMAppAttemptContainerFinishedEvent cfEvent = captor.getValue();
  assertEquals(appAttemptId, cfEvent.getApplicationAttemptId());
  assertEquals(containerStatus, cfEvent.getContainerStatus());
  assertEquals(RMAppAttemptEventType.CONTAINER_FINISHED, cfEvent.getType());
  
  // In RELEASED state. A FINIHSED event may come in.
  rmContainer.handle(new RMContainerFinishedEvent(containerId, SchedulerUtils
      .createAbnormalContainerStatus(containerId, "FinishedContainer"),
      RMContainerEventType.FINISHED));
  assertEquals(RMContainerState.RELEASED, rmContainer.getState());
}
Example 8
Source File: TestResourceTrackerService.java    From big-c with Apache License 2.0 4 votes vote down vote up 
@Test
public void testNodeRegistrationWithMinimumAllocations() throws Exception {
  Configuration conf = new Configuration();
  conf.set(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, "2048");
  conf.set(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES, "4");
  rm = new MockRM(conf);
  rm.start();

  ResourceTrackerService resourceTrackerService
    = rm.getResourceTrackerService();
  RegisterNodeManagerRequest req = Records.newRecord(
      RegisterNodeManagerRequest.class);
  NodeId nodeId = BuilderUtils.newNodeId("host", 1234);
  req.setNodeId(nodeId);

  Resource capability = BuilderUtils.newResource(1024, 1);
  req.setResource(capability);
  RegisterNodeManagerResponse response1 =
      resourceTrackerService.registerNodeManager(req);
  Assert.assertEquals(NodeAction.SHUTDOWN,response1.getNodeAction());
  
  capability.setMemory(2048);
  capability.setVirtualCores(1);
  req.setResource(capability);
  RegisterNodeManagerResponse response2 =
      resourceTrackerService.registerNodeManager(req);
  Assert.assertEquals(NodeAction.SHUTDOWN,response2.getNodeAction());
  
  capability.setMemory(1024);
  capability.setVirtualCores(4);
  req.setResource(capability);
  RegisterNodeManagerResponse response3 =
      resourceTrackerService.registerNodeManager(req);
  Assert.assertEquals(NodeAction.SHUTDOWN,response3.getNodeAction());
  
  capability.setMemory(2048);
  capability.setVirtualCores(4);
  req.setResource(capability);
  RegisterNodeManagerResponse response4 =
      resourceTrackerService.registerNodeManager(req);
  Assert.assertEquals(NodeAction.NORMAL,response4.getNodeAction());
}
Example 9
Source File: TestRMNodeTransitions.java    From big-c with Apache License 2.0 4 votes vote down vote up 
private RMNodeImpl getNewNode(Resource capability) {
  NodeId nodeId = BuilderUtils.newNodeId("localhost", 0);
  RMNodeImpl node = new RMNodeImpl(nodeId, rmContext, null, 0, 0, null, 
      capability, null);
  return node;
}
Example 10
Source File: TestRMNodeTransitions.java    From big-c with Apache License 2.0 4 votes vote down vote up 
private RMNodeImpl getNewNode() {
  NodeId nodeId = BuilderUtils.newNodeId("localhost", 0);
  RMNodeImpl node = new RMNodeImpl(nodeId, rmContext, null, 0, 0, null, null, null);
  return node;
}
Example 11
Source File: TestRMNodeTransitions.java    From big-c with Apache License 2.0 4 votes vote down vote up 
@Test (timeout = 5000)
public void testContainerUpdate() throws InterruptedException{
  //Start the node
  node.handle(new RMNodeStartedEvent(null, null, null));
  
  NodeId nodeId = BuilderUtils.newNodeId("localhost:1", 1);
  RMNodeImpl node2 = new RMNodeImpl(nodeId, rmContext, null, 0, 0, null, null, null);
  node2.handle(new RMNodeStartedEvent(null, null, null));
  
  ContainerId completedContainerIdFromNode1 = BuilderUtils.newContainerId(
      BuilderUtils.newApplicationAttemptId(
          BuilderUtils.newApplicationId(0, 0), 0), 0);
  ContainerId completedContainerIdFromNode2_1 = BuilderUtils.newContainerId(
      BuilderUtils.newApplicationAttemptId(
          BuilderUtils.newApplicationId(1, 1), 1), 1);
  ContainerId completedContainerIdFromNode2_2 = BuilderUtils.newContainerId(
      BuilderUtils.newApplicationAttemptId(
          BuilderUtils.newApplicationId(1, 1), 1), 2);
 
  RMNodeStatusEvent statusEventFromNode1 = getMockRMNodeStatusEvent();
  RMNodeStatusEvent statusEventFromNode2_1 = getMockRMNodeStatusEvent();
  RMNodeStatusEvent statusEventFromNode2_2 = getMockRMNodeStatusEvent();
  
  ContainerStatus containerStatusFromNode1 = mock(ContainerStatus.class);
  ContainerStatus containerStatusFromNode2_1 = mock(ContainerStatus.class);
  ContainerStatus containerStatusFromNode2_2 = mock(ContainerStatus.class);

  doReturn(completedContainerIdFromNode1).when(containerStatusFromNode1)
      .getContainerId();
  doReturn(Collections.singletonList(containerStatusFromNode1))
      .when(statusEventFromNode1).getContainers();
  node.handle(statusEventFromNode1);
  Assert.assertEquals(1, completedContainers.size());
  Assert.assertEquals(completedContainerIdFromNode1,
      completedContainers.get(0).getContainerId());

  completedContainers.clear();

  doReturn(completedContainerIdFromNode2_1).when(containerStatusFromNode2_1)
      .getContainerId();
  doReturn(Collections.singletonList(containerStatusFromNode2_1))
      .when(statusEventFromNode2_1).getContainers();

  doReturn(completedContainerIdFromNode2_2).when(containerStatusFromNode2_2)
      .getContainerId();
  doReturn(Collections.singletonList(containerStatusFromNode2_2))
      .when(statusEventFromNode2_2).getContainers();

  node2.setNextHeartBeat(false);
  node2.handle(statusEventFromNode2_1);
  node2.setNextHeartBeat(true);
  node2.handle(statusEventFromNode2_2);

  Assert.assertEquals(2, completedContainers.size());
  Assert.assertEquals(completedContainerIdFromNode2_1,completedContainers.get(0)
      .getContainerId()); 
  Assert.assertEquals(completedContainerIdFromNode2_2,completedContainers.get(1)
      .getContainerId());   
}
Example 12
Source File: TestRMNodeTransitions.java    From big-c with Apache License 2.0 4 votes vote down vote up 
@Before
public void setUp() throws Exception {
  InlineDispatcher rmDispatcher = new InlineDispatcher();
  
  rmContext =
      new RMContextImpl(rmDispatcher, null, null, null,
          mock(DelegationTokenRenewer.class), null, null, null, null, null);
  NodesListManager nodesListManager = mock(NodesListManager.class);
  HostsFileReader reader = mock(HostsFileReader.class);
  when(nodesListManager.getHostsReader()).thenReturn(reader);
  ((RMContextImpl) rmContext).setNodesListManager(nodesListManager);
  scheduler = mock(YarnScheduler.class);
  doAnswer(
      new Answer<Void>() {

        @Override
        public Void answer(InvocationOnMock invocation) throws Throwable {
          final SchedulerEvent event = (SchedulerEvent)(invocation.getArguments()[0]);
          eventType = event.getType();
          if (eventType == SchedulerEventType.NODE_UPDATE) {
            List<UpdatedContainerInfo> lastestContainersInfoList = 
                ((NodeUpdateSchedulerEvent)event).getRMNode().pullContainerUpdates();
            for(UpdatedContainerInfo lastestContainersInfo : lastestContainersInfoList) {
          	  completedContainers.addAll(lastestContainersInfo.getCompletedContainers()); 
            }
          }
          return null;
        }
      }
      ).when(scheduler).handle(any(SchedulerEvent.class));
  
  rmDispatcher.register(SchedulerEventType.class, 
      new TestSchedulerEventDispatcher());
  
  rmDispatcher.register(NodesListManagerEventType.class,
      new TestNodeListManagerEventDispatcher());

  NodeId nodeId = BuilderUtils.newNodeId("localhost", 0);
  node = new RMNodeImpl(nodeId, rmContext, null, 0, 0, null, null, null);
  nodesListManagerEvent =  null;

}
Example 13
Source File: TestRMContainerImpl.java    From big-c with Apache License 2.0 4 votes vote down vote up 
@Test
public void testExpireWhileRunning() {

  DrainDispatcher drainDispatcher = new DrainDispatcher();
  EventHandler<RMAppAttemptEvent> appAttemptEventHandler = mock(EventHandler.class);
  EventHandler generic = mock(EventHandler.class);
  drainDispatcher.register(RMAppAttemptEventType.class,
      appAttemptEventHandler);
  drainDispatcher.register(RMNodeEventType.class, generic);
  drainDispatcher.init(new YarnConfiguration());
  drainDispatcher.start();
  NodeId nodeId = BuilderUtils.newNodeId("host", 3425);
  ApplicationId appId = BuilderUtils.newApplicationId(1, 1);
  ApplicationAttemptId appAttemptId = BuilderUtils.newApplicationAttemptId(
      appId, 1);
  ContainerId containerId = BuilderUtils.newContainerId(appAttemptId, 1);
  ContainerAllocationExpirer expirer = mock(ContainerAllocationExpirer.class);

  Resource resource = BuilderUtils.newResource(512, 1);
  Priority priority = BuilderUtils.newPriority(5);

  Container container = BuilderUtils.newContainer(containerId, nodeId,
      "host:3465", resource, priority, null);

  RMApplicationHistoryWriter writer = mock(RMApplicationHistoryWriter.class);
  SystemMetricsPublisher publisher = mock(SystemMetricsPublisher.class);
  RMContext rmContext = mock(RMContext.class);
  when(rmContext.getDispatcher()).thenReturn(drainDispatcher);
  when(rmContext.getContainerAllocationExpirer()).thenReturn(expirer);
  when(rmContext.getRMApplicationHistoryWriter()).thenReturn(writer);
  when(rmContext.getSystemMetricsPublisher()).thenReturn(publisher);
  when(rmContext.getYarnConfiguration()).thenReturn(new YarnConfiguration());
  RMContainer rmContainer = new RMContainerImpl(container, appAttemptId,
      nodeId, "user", rmContext);

  assertEquals(RMContainerState.NEW, rmContainer.getState());
  assertEquals(resource, rmContainer.getAllocatedResource());
  assertEquals(nodeId, rmContainer.getAllocatedNode());
  assertEquals(priority, rmContainer.getAllocatedPriority());
  verify(writer).containerStarted(any(RMContainer.class));
  verify(publisher).containerCreated(any(RMContainer.class), anyLong());

  rmContainer.handle(new RMContainerEvent(containerId,
      RMContainerEventType.START));
  drainDispatcher.await();
  assertEquals(RMContainerState.ALLOCATED, rmContainer.getState());

  rmContainer.handle(new RMContainerEvent(containerId,
      RMContainerEventType.ACQUIRED));
  drainDispatcher.await();
  assertEquals(RMContainerState.ACQUIRED, rmContainer.getState());

  rmContainer.handle(new RMContainerEvent(containerId,
      RMContainerEventType.LAUNCHED));
  drainDispatcher.await();
  assertEquals(RMContainerState.RUNNING, rmContainer.getState());
  assertEquals("http://host:3465/node/containerlogs/container_1_0001_01_000001/user",
      rmContainer.getLogURL());

  // In RUNNING state. Verify EXPIRE and associated actions.
  reset(appAttemptEventHandler);
  ContainerStatus containerStatus = SchedulerUtils
      .createAbnormalContainerStatus(containerId,
          SchedulerUtils.EXPIRED_CONTAINER);
  rmContainer.handle(new RMContainerFinishedEvent(containerId,
      containerStatus, RMContainerEventType.EXPIRE));
  drainDispatcher.await();
  assertEquals(RMContainerState.RUNNING, rmContainer.getState());
  verify(writer, never()).containerFinished(any(RMContainer.class));
  verify(publisher, never()).containerFinished(any(RMContainer.class),
      anyLong());
}
Example 14
Source File: TestRMContainerImpl.java    From hadoop with Apache License 2.0 4 votes vote down vote up 
@Test
public void testReleaseWhileRunning() {

  DrainDispatcher drainDispatcher = new DrainDispatcher();
  EventHandler<RMAppAttemptEvent> appAttemptEventHandler = mock(EventHandler.class);
  EventHandler generic = mock(EventHandler.class);
  drainDispatcher.register(RMAppAttemptEventType.class,
      appAttemptEventHandler);
  drainDispatcher.register(RMNodeEventType.class, generic);
  drainDispatcher.init(new YarnConfiguration());
  drainDispatcher.start();
  NodeId nodeId = BuilderUtils.newNodeId("host", 3425);
  ApplicationId appId = BuilderUtils.newApplicationId(1, 1);
  ApplicationAttemptId appAttemptId = BuilderUtils.newApplicationAttemptId(
      appId, 1);
  ContainerId containerId = BuilderUtils.newContainerId(appAttemptId, 1);
  ContainerAllocationExpirer expirer = mock(ContainerAllocationExpirer.class);

  Resource resource = BuilderUtils.newResource(512, 1, 1);
  Priority priority = BuilderUtils.newPriority(5);

  Container container = BuilderUtils.newContainer(containerId, nodeId,
      "host:3465", resource, priority, null);
  ConcurrentMap<ApplicationId, RMApp> rmApps =
      spy(new ConcurrentHashMap<ApplicationId, RMApp>());
  RMApp rmApp = mock(RMApp.class);
  when(rmApp.getRMAppAttempt((ApplicationAttemptId)Matchers.any())).thenReturn(null);
  Mockito.doReturn(rmApp).when(rmApps).get((ApplicationId)Matchers.any());

  RMApplicationHistoryWriter writer = mock(RMApplicationHistoryWriter.class);
  SystemMetricsPublisher publisher = mock(SystemMetricsPublisher.class);
  RMContext rmContext = mock(RMContext.class);
  when(rmContext.getDispatcher()).thenReturn(drainDispatcher);
  when(rmContext.getContainerAllocationExpirer()).thenReturn(expirer);
  when(rmContext.getRMApplicationHistoryWriter()).thenReturn(writer);
  when(rmContext.getRMApps()).thenReturn(rmApps);
  when(rmContext.getSystemMetricsPublisher()).thenReturn(publisher);
  when(rmContext.getYarnConfiguration()).thenReturn(new YarnConfiguration());
  RMContainer rmContainer = new RMContainerImpl(container, appAttemptId,
      nodeId, "user", rmContext);

  assertEquals(RMContainerState.NEW, rmContainer.getState());
  assertEquals(resource, rmContainer.getAllocatedResource());
  assertEquals(nodeId, rmContainer.getAllocatedNode());
  assertEquals(priority, rmContainer.getAllocatedPriority());
  verify(writer).containerStarted(any(RMContainer.class));
  verify(publisher).containerCreated(any(RMContainer.class), anyLong());

  rmContainer.handle(new RMContainerEvent(containerId,
      RMContainerEventType.START));
  drainDispatcher.await();
  assertEquals(RMContainerState.ALLOCATED, rmContainer.getState());
  rmContainer.handle(new RMContainerEvent(containerId,
      RMContainerEventType.ACQUIRED));
  drainDispatcher.await();
  assertEquals(RMContainerState.ACQUIRED, rmContainer.getState());

  rmContainer.handle(new RMContainerEvent(containerId,
      RMContainerEventType.LAUNCHED));
  drainDispatcher.await();
  assertEquals(RMContainerState.RUNNING, rmContainer.getState());
  assertEquals("http://host:3465/node/containerlogs/container_1_0001_01_000001/user",
      rmContainer.getLogURL());

  // In RUNNING state. Verify RELEASED and associated actions.
  reset(appAttemptEventHandler);
  ContainerStatus containerStatus = SchedulerUtils
      .createAbnormalContainerStatus(containerId,
          SchedulerUtils.RELEASED_CONTAINER);
  rmContainer.handle(new RMContainerFinishedEvent(containerId,
      containerStatus, RMContainerEventType.RELEASED));
  drainDispatcher.await();
  assertEquals(RMContainerState.RELEASED, rmContainer.getState());
  assertEquals(SchedulerUtils.RELEASED_CONTAINER,
      rmContainer.getDiagnosticsInfo());
  assertEquals(ContainerExitStatus.ABORTED,
      rmContainer.getContainerExitStatus());
  assertEquals(ContainerState.COMPLETE, rmContainer.getContainerState());
  verify(writer).containerFinished(any(RMContainer.class));
  verify(publisher).containerFinished(any(RMContainer.class), anyLong());

  ArgumentCaptor<RMAppAttemptContainerFinishedEvent> captor = ArgumentCaptor
      .forClass(RMAppAttemptContainerFinishedEvent.class);
  verify(appAttemptEventHandler).handle(captor.capture());
  RMAppAttemptContainerFinishedEvent cfEvent = captor.getValue();
  assertEquals(appAttemptId, cfEvent.getApplicationAttemptId());
  assertEquals(containerStatus, cfEvent.getContainerStatus());
  assertEquals(RMAppAttemptEventType.CONTAINER_FINISHED, cfEvent.getType());
  
  // In RELEASED state. A FINIHSED event may come in.
  rmContainer.handle(new RMContainerFinishedEvent(containerId, SchedulerUtils
      .createAbnormalContainerStatus(containerId, "FinishedContainer"),
      RMContainerEventType.FINISHED));
  assertEquals(RMContainerState.RELEASED, rmContainer.getState());
}
Example 15
Source File: TestResourceTrackerService.java    From hadoop with Apache License 2.0 4 votes vote down vote up 
@Test
public void testNodeRegistrationWithMinimumAllocations() throws Exception {
  Configuration conf = new Configuration();
  conf.set(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, "2048");
  conf.set(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES, "4");
  conf.set(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_GCORES, "4");
  rm = new MockRM(conf);
  rm.start();

  ResourceTrackerService resourceTrackerService
    = rm.getResourceTrackerService();
  RegisterNodeManagerRequest req = Records.newRecord(
      RegisterNodeManagerRequest.class);
  NodeId nodeId = BuilderUtils.newNodeId("host", 1234);
  req.setNodeId(nodeId);

  Resource capability = BuilderUtils.newResource(1024, 1, 1);
  req.setResource(capability);
  RegisterNodeManagerResponse response1 =
      resourceTrackerService.registerNodeManager(req);
  Assert.assertEquals(NodeAction.SHUTDOWN,response1.getNodeAction());
  
  capability.setMemory(2048);
  capability.setVirtualCores(1);
  capability.setGpuCores(1);
  req.setResource(capability);
  RegisterNodeManagerResponse response2 =
      resourceTrackerService.registerNodeManager(req);
  Assert.assertEquals(NodeAction.SHUTDOWN,response2.getNodeAction());
  
  capability.setMemory(1024);
  capability.setVirtualCores(4);
  capability.setGpuCores(4);
  req.setResource(capability);
  RegisterNodeManagerResponse response3 =
      resourceTrackerService.registerNodeManager(req);
  Assert.assertEquals(NodeAction.SHUTDOWN,response3.getNodeAction());
  
  capability.setMemory(2048);
  capability.setVirtualCores(4);
  capability.setGpuCores(4);
  req.setResource(capability);
  RegisterNodeManagerResponse response4 =
      resourceTrackerService.registerNodeManager(req);
  Assert.assertEquals(NodeAction.NORMAL,response4.getNodeAction());
}
Example 16
Source File: TestRMNodeTransitions.java    From hadoop with Apache License 2.0 4 votes vote down vote up 
private RMNodeImpl getNewNode(Resource capability) {
  NodeId nodeId = BuilderUtils.newNodeId("localhost", 0);
  RMNodeImpl node = new RMNodeImpl(nodeId, rmContext, null, 0, 0, null, 
      capability, null);
  return node;
}
Example 17
Source File: TestRMNodeTransitions.java    From hadoop with Apache License 2.0 4 votes vote down vote up 
private RMNodeImpl getNewNode() {
  NodeId nodeId = BuilderUtils.newNodeId("localhost", 0);
  RMNodeImpl node = new RMNodeImpl(nodeId, rmContext, null, 0, 0, null, null, null);
  return node;
}
Example 18
Source File: TestRMNodeTransitions.java    From hadoop with Apache License 2.0 4 votes vote down vote up 
@Test (timeout = 5000)
public void testContainerUpdate() throws InterruptedException{
  //Start the node
  node.handle(new RMNodeStartedEvent(null, null, null));
  
  NodeId nodeId = BuilderUtils.newNodeId("localhost:1", 1);
  RMNodeImpl node2 = new RMNodeImpl(nodeId, rmContext, null, 0, 0, null, null, null);
  node2.handle(new RMNodeStartedEvent(null, null, null));
  
  ContainerId completedContainerIdFromNode1 = BuilderUtils.newContainerId(
      BuilderUtils.newApplicationAttemptId(
          BuilderUtils.newApplicationId(0, 0), 0), 0);
  ContainerId completedContainerIdFromNode2_1 = BuilderUtils.newContainerId(
      BuilderUtils.newApplicationAttemptId(
          BuilderUtils.newApplicationId(1, 1), 1), 1);
  ContainerId completedContainerIdFromNode2_2 = BuilderUtils.newContainerId(
      BuilderUtils.newApplicationAttemptId(
          BuilderUtils.newApplicationId(1, 1), 1), 2);
 
  RMNodeStatusEvent statusEventFromNode1 = getMockRMNodeStatusEvent();
  RMNodeStatusEvent statusEventFromNode2_1 = getMockRMNodeStatusEvent();
  RMNodeStatusEvent statusEventFromNode2_2 = getMockRMNodeStatusEvent();
  
  ContainerStatus containerStatusFromNode1 = mock(ContainerStatus.class);
  ContainerStatus containerStatusFromNode2_1 = mock(ContainerStatus.class);
  ContainerStatus containerStatusFromNode2_2 = mock(ContainerStatus.class);

  doReturn(completedContainerIdFromNode1).when(containerStatusFromNode1)
      .getContainerId();
  doReturn(Collections.singletonList(containerStatusFromNode1))
      .when(statusEventFromNode1).getContainers();
  node.handle(statusEventFromNode1);
  Assert.assertEquals(1, completedContainers.size());
  Assert.assertEquals(completedContainerIdFromNode1,
      completedContainers.get(0).getContainerId());

  completedContainers.clear();

  doReturn(completedContainerIdFromNode2_1).when(containerStatusFromNode2_1)
      .getContainerId();
  doReturn(Collections.singletonList(containerStatusFromNode2_1))
      .when(statusEventFromNode2_1).getContainers();

  doReturn(completedContainerIdFromNode2_2).when(containerStatusFromNode2_2)
      .getContainerId();
  doReturn(Collections.singletonList(containerStatusFromNode2_2))
      .when(statusEventFromNode2_2).getContainers();

  node2.setNextHeartBeat(false);
  node2.handle(statusEventFromNode2_1);
  node2.setNextHeartBeat(true);
  node2.handle(statusEventFromNode2_2);

  Assert.assertEquals(2, completedContainers.size());
  Assert.assertEquals(completedContainerIdFromNode2_1,completedContainers.get(0)
      .getContainerId()); 
  Assert.assertEquals(completedContainerIdFromNode2_2,completedContainers.get(1)
      .getContainerId());   
}
Example 19
Source File: TestRMNodeTransitions.java    From hadoop with Apache License 2.0 4 votes vote down vote up 
@Before
public void setUp() throws Exception {
  InlineDispatcher rmDispatcher = new InlineDispatcher();
  
  rmContext =
      new RMContextImpl(rmDispatcher, null, null, null,
          mock(DelegationTokenRenewer.class), null, null, null, null, null);
  NodesListManager nodesListManager = mock(NodesListManager.class);
  HostsFileReader reader = mock(HostsFileReader.class);
  when(nodesListManager.getHostsReader()).thenReturn(reader);
  ((RMContextImpl) rmContext).setNodesListManager(nodesListManager);
  scheduler = mock(YarnScheduler.class);
  doAnswer(
      new Answer<Void>() {

        @Override
        public Void answer(InvocationOnMock invocation) throws Throwable {
          final SchedulerEvent event = (SchedulerEvent)(invocation.getArguments()[0]);
          eventType = event.getType();
          if (eventType == SchedulerEventType.NODE_UPDATE) {
            List<UpdatedContainerInfo> lastestContainersInfoList = 
                ((NodeUpdateSchedulerEvent)event).getRMNode().pullContainerUpdates();
            for(UpdatedContainerInfo lastestContainersInfo : lastestContainersInfoList) {
          	  completedContainers.addAll(lastestContainersInfo.getCompletedContainers()); 
            }
          }
          return null;
        }
      }
      ).when(scheduler).handle(any(SchedulerEvent.class));
  
  rmDispatcher.register(SchedulerEventType.class, 
      new TestSchedulerEventDispatcher());
  
  rmDispatcher.register(NodesListManagerEventType.class,
      new TestNodeListManagerEventDispatcher());

  NodeId nodeId = BuilderUtils.newNodeId("localhost", 0);
  node = new RMNodeImpl(nodeId, rmContext, null, 0, 0, null, null, null);
  nodesListManagerEvent =  null;

}
Example 20
Source File: TestRMContainerImpl.java    From hadoop with Apache License 2.0 4 votes vote down vote up 
@Test
public void testExpireWhileRunning() {

  DrainDispatcher drainDispatcher = new DrainDispatcher();
  EventHandler<RMAppAttemptEvent> appAttemptEventHandler = mock(EventHandler.class);
  EventHandler generic = mock(EventHandler.class);
  drainDispatcher.register(RMAppAttemptEventType.class,
      appAttemptEventHandler);
  drainDispatcher.register(RMNodeEventType.class, generic);
  drainDispatcher.init(new YarnConfiguration());
  drainDispatcher.start();
  NodeId nodeId = BuilderUtils.newNodeId("host", 3425);
  ApplicationId appId = BuilderUtils.newApplicationId(1, 1);
  ApplicationAttemptId appAttemptId = BuilderUtils.newApplicationAttemptId(
      appId, 1);
  ContainerId containerId = BuilderUtils.newContainerId(appAttemptId, 1);
  ContainerAllocationExpirer expirer = mock(ContainerAllocationExpirer.class);

  Resource resource = BuilderUtils.newResource(512, 1, 1);
  Priority priority = BuilderUtils.newPriority(5);

  Container container = BuilderUtils.newContainer(containerId, nodeId,
      "host:3465", resource, priority, null);

  RMApplicationHistoryWriter writer = mock(RMApplicationHistoryWriter.class);
  SystemMetricsPublisher publisher = mock(SystemMetricsPublisher.class);
  RMContext rmContext = mock(RMContext.class);
  when(rmContext.getDispatcher()).thenReturn(drainDispatcher);
  when(rmContext.getContainerAllocationExpirer()).thenReturn(expirer);
  when(rmContext.getRMApplicationHistoryWriter()).thenReturn(writer);
  when(rmContext.getSystemMetricsPublisher()).thenReturn(publisher);
  when(rmContext.getYarnConfiguration()).thenReturn(new YarnConfiguration());
  RMContainer rmContainer = new RMContainerImpl(container, appAttemptId,
      nodeId, "user", rmContext);

  assertEquals(RMContainerState.NEW, rmContainer.getState());
  assertEquals(resource, rmContainer.getAllocatedResource());
  assertEquals(nodeId, rmContainer.getAllocatedNode());
  assertEquals(priority, rmContainer.getAllocatedPriority());
  verify(writer).containerStarted(any(RMContainer.class));
  verify(publisher).containerCreated(any(RMContainer.class), anyLong());

  rmContainer.handle(new RMContainerEvent(containerId,
      RMContainerEventType.START));
  drainDispatcher.await();
  assertEquals(RMContainerState.ALLOCATED, rmContainer.getState());

  rmContainer.handle(new RMContainerEvent(containerId,
      RMContainerEventType.ACQUIRED));
  drainDispatcher.await();
  assertEquals(RMContainerState.ACQUIRED, rmContainer.getState());

  rmContainer.handle(new RMContainerEvent(containerId,
      RMContainerEventType.LAUNCHED));
  drainDispatcher.await();
  assertEquals(RMContainerState.RUNNING, rmContainer.getState());
  assertEquals("http://host:3465/node/containerlogs/container_1_0001_01_000001/user",
      rmContainer.getLogURL());

  // In RUNNING state. Verify EXPIRE and associated actions.
  reset(appAttemptEventHandler);
  ContainerStatus containerStatus = SchedulerUtils
      .createAbnormalContainerStatus(containerId,
          SchedulerUtils.EXPIRED_CONTAINER);
  rmContainer.handle(new RMContainerFinishedEvent(containerId,
      containerStatus, RMContainerEventType.EXPIRE));
  drainDispatcher.await();
  assertEquals(RMContainerState.RUNNING, rmContainer.getState());
  verify(writer, never()).containerFinished(any(RMContainer.class));
  verify(publisher, never()).containerFinished(any(RMContainer.class),
      anyLong());
}