org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerEvent Java Examples

@SuppressWarnings("unchecked")
public synchronized void containerLaunchedOnNode(ContainerId containerId,
    NodeId nodeId) {
  // Inform the container
  RMContainer rmContainer = getRMContainer(containerId);
  if (rmContainer == null) {
    // Some unknown container sneaked into the system. Kill it.
    rmContext.getDispatcher().getEventHandler()
      .handle(new RMNodeCleanContainerEvent(nodeId, containerId));
    return;
  }

  rmContainer.handle(new RMContainerEvent(containerId,
      RMContainerEventType.LAUNCHED));
}
 

public synchronized ContainersAndNMTokensAllocation
    pullNewlyAllocatedContainersAndNMTokens() {
  List<Container> returnContainerList =
      new ArrayList<Container>(newlyAllocatedContainers.size());
  List<NMToken> nmTokens = new ArrayList<NMToken>();
  for (Iterator<RMContainer> i = newlyAllocatedContainers.iterator(); i
    .hasNext();) {
    RMContainer rmContainer = i.next();
    Container container = rmContainer.getContainer();
    try {
      // create container token and NMToken altogether.
      container.setContainerToken(rmContext.getContainerTokenSecretManager()
        .createContainerToken(container.getId(), container.getNodeId(),
          getUser(), container.getResource(), container.getPriority(),
          rmContainer.getCreationTime(), this.logAggregationContext));
      NMToken nmToken =
          rmContext.getNMTokenSecretManager().createAndGetNMToken(getUser(),
            getApplicationAttemptId(), container);
      if (nmToken != null) {
        nmTokens.add(nmToken);
      }
    } catch (IllegalArgumentException e) {
      // DNS might be down, skip returning this container.
      LOG.error("Error trying to assign container token and NM token to" +
          " an allocated container " + container.getId(), e);
      continue;
    }
    returnContainerList.add(container);
    i.remove();
    rmContainer.handle(new RMContainerEvent(rmContainer.getContainerId(),
      RMContainerEventType.ACQUIRED));
  }
  return new ContainersAndNMTokensAllocation(returnContainerList, nmTokens);
}
 

@SuppressWarnings("unchecked")
public synchronized void containerLaunchedOnNode(ContainerId containerId,
    NodeId nodeId) {
  // Inform the container
  RMContainer rmContainer = getRMContainer(containerId);
  if (rmContainer == null) {
    // Some unknown container sneaked into the system. Kill it.
    rmContext.getDispatcher().getEventHandler()
      .handle(new RMNodeCleanContainerEvent(nodeId, containerId));
    return;
  }

  rmContainer.handle(new RMContainerEvent(containerId,
      RMContainerEventType.LAUNCHED));
}
 

public synchronized ContainersAndNMTokensAllocation
    pullNewlyAllocatedContainersAndNMTokens() {
  List<Container> returnContainerList =
      new ArrayList<Container>(newlyAllocatedContainers.size());
  List<NMToken> nmTokens = new ArrayList<NMToken>();
  for (Iterator<RMContainer> i = newlyAllocatedContainers.iterator(); i
    .hasNext();) {
    RMContainer rmContainer = i.next();
    Container container = rmContainer.getContainer();
    try {
      // create container token and NMToken altogether.
      container.setContainerToken(rmContext.getContainerTokenSecretManager()
        .createContainerToken(container.getId(), container.getNodeId(),
          getUser(), container.getResource(), container.getPriority(),
          rmContainer.getCreationTime(), this.logAggregationContext));
      NMToken nmToken =
          rmContext.getNMTokenSecretManager().createAndGetNMToken(getUser(),
            getApplicationAttemptId(), container);
      if (nmToken != null) {
        nmTokens.add(nmToken);
      }
    } catch (IllegalArgumentException e) {
      // DNS might be down, skip returning this container.
      LOG.error("Error trying to assign container token and NM token to" +
          " an allocated container " + container.getId(), e);
      continue;
    }
    returnContainerList.add(container);
    i.remove();
    rmContainer.handle(new RMContainerEvent(rmContainer.getContainerId(),
      RMContainerEventType.ACQUIRED));
  }
  return new ContainersAndNMTokensAllocation(returnContainerList, nmTokens);
}
 

public static void intercept(@This Object rmContainerImpl,
                             @Argument(0) RMContainerEvent rmContainerEvent) {
    try {
        ContainerId cID = rmContainerEvent.getContainerId();
        ApplicationAttemptId applicationAttemptId = cID.getApplicationAttemptId();
        String applicationId = applicationAttemptId.getApplicationId().toString();
        String attemptId = applicationAttemptId.toString();

        Header header = Header.newBuilder()
            .withId(applicationId)
            .withApplicationID(applicationId)
            .withAttemptID(attemptId)
            .withContainerID(cID.toString())
            .build();

        RMContainerImpl rmc = (RMContainerImpl) rmContainerImpl;

        ResourceManagerEventProtos.ContainerEvent.Builder eventBuilder = ResourceManagerEventProtos.ContainerEvent.newBuilder()
            .setType(rmContainerEvent.getType().name())
            .setState(rmc.getState().name())
            .setStartTime(rmc.getCreationTime())
            .setLogUrl(rmc.getLogURL());

        if (rmc.getContainer() != null && rmc.getContainer().getResource() != null) {
            eventBuilder.setVcoresReserved(rmc.getContainer().getResource().getVirtualCores());
            eventBuilder.setMemoryReserved(rmc.getContainer().getResource().getMemory());
        }

        if (rmc.getAllocatedNode() != null) {
            eventBuilder.setContainerHostname(rmc.getAllocatedNode().getHost());
        }

        ContainerStatus containerStatus = (ContainerStatus) getField().get(rmContainerImpl);
        if (containerStatus != null) {
            eventBuilder
                .setIsFinished(true)
                .setExitStatus(rmc.getContainerExitStatus())
                .setReason(rmc.getDiagnosticsInfo())
                .setFinishTime(rmc.getFinishTime());
        }

        eventHandler.accept(System.currentTimeMillis(), header, eventBuilder.build());
    } catch (Throwable ignored) {
    }
}
 

@Test
@AgentAttachmentRule.Enforce
public void ContainerResourceMonitoringModule_should_attach_to_recordCpuUsage() throws ClassNotFoundException, NoSuchMethodException, IllegalAccessException, InstantiationException, InvocationTargetException, NoSuchFieldException {
    assertThat(ByteBuddyAgent.install(), instanceOf(Instrumentation.class));

    final Header[] header = new Header[1];
    final Object[] event = new Object[1];
    RMContainerTracer.initEventHandler((t, h, o) -> {
        header[0] = h;
        event[0] = o;
    });

    ClassFileTransformer classFileTransformer = new RMContainerTracer.RMContainerImplTracer().installOnByteBuddyAgent();

    try {
        Class<?> clazz = classLoader.loadClass(RMContainerImpl.class.getName());
        Constructor<?> constructor = clazz.getDeclaredConstructor(Container.class,
            ApplicationAttemptId.class, NodeId.class, String.class,
            RMContext.class);
        constructor.setAccessible(true);

        RMContext rmContext = mock(RMContext.class);
        Configuration yarnConf = new Configuration();
        when(rmContext.getYarnConfiguration())
            .thenReturn(yarnConf);
        Dispatcher dispatcher = mock(Dispatcher.class);
        EventHandler eventHandler = mock(EventHandler.class);
        when(dispatcher.getEventHandler())
            .thenReturn(eventHandler);
        when(rmContext.getDispatcher())
            .thenReturn(dispatcher);

        RMApplicationHistoryWriter rmApplicationHistoryWriter = mock(RMApplicationHistoryWriter.class);
        when(rmContext.getRMApplicationHistoryWriter())
            .thenReturn(rmApplicationHistoryWriter);

        SystemMetricsPublisher systemMetricsPublisher = mock(SystemMetricsPublisher.class);
        when(rmContext.getSystemMetricsPublisher())
            .thenReturn(systemMetricsPublisher);

        Object inFormat = constructor.newInstance(mock(Container.class), mock(ApplicationAttemptId.class), mock(NodeId.class), "user", rmContext);

        Method m = clazz.getDeclaredMethod("handle", RMContainerEvent.class);

        ContainerId cid = mock(ContainerId.class);
        ApplicationAttemptId applicationAttemptId = mock(ApplicationAttemptId.class);
        when(cid.toString())
            .thenReturn("cid");
        when(cid.getApplicationAttemptId())
            .thenReturn(applicationAttemptId);
        when(applicationAttemptId.toString())
            .thenReturn("appattempt_id");
        ApplicationId applicationId = mock(ApplicationId.class);
        when(applicationAttemptId.getApplicationId())
            .thenReturn(applicationId);
        when(applicationId.toString())
            .thenReturn("app_id");
        m.invoke(inFormat, new RMContainerEvent(cid, RMContainerEventType.START));

        assertNotNull(header[0]);
        assertNotNull(event[0]);

    } finally {
        ByteBuddyAgent.getInstrumentation().removeTransformer(classFileTransformer);
    }
}
 

synchronized public RMContainer allocate(NodeType type, FSSchedulerNode node,
    Priority priority, ResourceRequest request,
    Container container) {
  // Update allowed locality level
  NodeType allowed = allowedLocalityLevel.get(priority);
  if (allowed != null) {
    if (allowed.equals(NodeType.OFF_SWITCH) &&
        (type.equals(NodeType.NODE_LOCAL) ||
            type.equals(NodeType.RACK_LOCAL))) {
      this.resetAllowedLocalityLevel(priority, type);
    }
    else if (allowed.equals(NodeType.RACK_LOCAL) &&
        type.equals(NodeType.NODE_LOCAL)) {
      this.resetAllowedLocalityLevel(priority, type);
    }
  }

  // Required sanity check - AM can call 'allocate' to update resource 
  // request without locking the scheduler, hence we need to check
  if (getTotalRequiredResources(priority) <= 0) {
    return null;
  }
  
  // Create RMContainer
  RMContainer rmContainer = new RMContainerImpl(container, 
      getApplicationAttemptId(), node.getNodeID(),
      appSchedulingInfo.getUser(), rmContext);

  // Add it to allContainers list.
  newlyAllocatedContainers.add(rmContainer);
  liveContainers.put(container.getId(), rmContainer);    

  // Update consumption and track allocations
  List<ResourceRequest> resourceRequestList = appSchedulingInfo.allocate(
      type, node, priority, request, container);
  this.attemptResourceUsage.incUsed(container.getResource());

  // Update resource requests related to "request" and store in RMContainer
  ((RMContainerImpl) rmContainer).setResourceRequests(resourceRequestList);

  // Inform the container
  rmContainer.handle(
      new RMContainerEvent(container.getId(), RMContainerEventType.START));

  if (LOG.isDebugEnabled()) {
    LOG.debug("allocate: applicationAttemptId=" 
        + container.getId().getApplicationAttemptId() 
        + " container=" + container.getId() + " host="
        + container.getNodeId().getHost() + " type=" + type);
  }
  RMAuditLogger.logSuccess(getUser(), 
      AuditConstants.ALLOC_CONTAINER, "SchedulerApp", 
      getApplicationId(), container.getId());
  
  return rmContainer;
}
 

synchronized public RMContainer allocate(NodeType type, FiCaSchedulerNode node,
    Priority priority, ResourceRequest request, 
    Container container) {

  if (isStopped) {
    return null;
  }
  
  // Required sanity check - AM can call 'allocate' to update resource 
  // request without locking the scheduler, hence we need to check
  if (getTotalRequiredResources(priority) <= 0) {
    return null;
  }
  
  // Create RMContainer
  RMContainer rmContainer = new RMContainerImpl(container, this
      .getApplicationAttemptId(), node.getNodeID(),
      appSchedulingInfo.getUser(), this.rmContext);

  // Add it to allContainers list.
  newlyAllocatedContainers.add(rmContainer);
  liveContainers.put(container.getId(), rmContainer);    

  // Update consumption and track allocations
  List<ResourceRequest> resourceRequestList = appSchedulingInfo.allocate(
      type, node, priority, request, container);

  attemptResourceUsage.incUsed(node.getPartition(), container.getResource());

  // Update resource requests related to "request" and store in RMContainer
  ((RMContainerImpl)rmContainer).setResourceRequests(resourceRequestList);

  // Inform the container
  rmContainer.handle(
      new RMContainerEvent(container.getId(), RMContainerEventType.START));

  if (LOG.isDebugEnabled()) {
    LOG.debug("allocate: applicationAttemptId=" 
        + container.getId().getApplicationAttemptId() 
        + " container=" + container.getId() + " host="
        + container.getNodeId().getHost() + " type=" + type);
  }
  RMAuditLogger.logSuccess(getUser(),
      AuditConstants.ALLOC_CONTAINER, "SchedulerApp",
      getApplicationId(), container.getId());
  
  return rmContainer;
}
 

synchronized public RMContainer allocate(NodeType type, FSSchedulerNode node,
    Priority priority, ResourceRequest request,
    Container container) {
  // Update allowed locality level
  NodeType allowed = allowedLocalityLevel.get(priority);
  if (allowed != null) {
    if (allowed.equals(NodeType.OFF_SWITCH) &&
        (type.equals(NodeType.NODE_LOCAL) ||
            type.equals(NodeType.RACK_LOCAL))) {
      this.resetAllowedLocalityLevel(priority, type);
    }
    else if (allowed.equals(NodeType.RACK_LOCAL) &&
        type.equals(NodeType.NODE_LOCAL)) {
      this.resetAllowedLocalityLevel(priority, type);
    }
  }

  // Required sanity check - AM can call 'allocate' to update resource 
  // request without locking the scheduler, hence we need to check
  if (getTotalRequiredResources(priority) <= 0) {
    return null;
  }
  
  // Create RMContainer
  RMContainer rmContainer = new RMContainerImpl(container, 
      getApplicationAttemptId(), node.getNodeID(),
      appSchedulingInfo.getUser(), rmContext);

  // Add it to allContainers list.
  newlyAllocatedContainers.add(rmContainer);
  liveContainers.put(container.getId(), rmContainer);    

  // Update consumption and track allocations
  List<ResourceRequest> resourceRequestList = appSchedulingInfo.allocate(
      type, node, priority, request, container);
  Resources.addTo(currentConsumption, container.getResource());

  // Update resource requests related to "request" and store in RMContainer
  ((RMContainerImpl) rmContainer).setResourceRequests(resourceRequestList);

  // Inform the container
  rmContainer.handle(
      new RMContainerEvent(container.getId(), RMContainerEventType.START));

  if (LOG.isDebugEnabled()) {
    LOG.debug("allocate: applicationAttemptId=" 
        + container.getId().getApplicationAttemptId() 
        + " container=" + container.getId() + " host="
        + container.getNodeId().getHost() + " type=" + type);
  }
  RMAuditLogger.logSuccess(getUser(), 
      AuditConstants.ALLOC_CONTAINER, "SchedulerApp", 
      getApplicationId(), container.getId());
  
  return rmContainer;
}
 

synchronized public boolean containerResume(RMContainer rmContainer,Resource toResume){
 
 
 
 ContainerId containerId = rmContainer.getContainerId();
 
 if(isStopped){
  return false;
 }
 
 if(!isSuspending){
  return false;
 }
 
 if(!containersSuspended.contains(containerId)){
  return false;
 }
 //add resumed resource
 rmContainer.addResumedResource(toResume);
 
 //we try to update its resource consumption
 rmContainer.handle(
            new RMContainerEvent(containerId,RMContainerEventType.RESUME)  
     );

 //if all of its resource has been resumed
 if(!rmContainer.isSuspending()){
 //delete contaienr from containersSuspended
 this.containersSuspended.remove(containerId);
 }  
 
 //update resource usage
 queue.getMetrics().allocateResources(getUser(), 1, toResume, true);
 //update app resource usage
 Resources.addTo(currentConsumption,toResume);
 //inform RMContainer
 if(this.containersSuspended.size() == 0){  
  isSuspending = false;
  LOG.info("application "+this.getApplicationId()+"has been out of the suspended list");
 }
 
 LOG.info("app "+this.getApplicationAttemptId()+" consume resource"+currentConsumption);

   if (LOG.isDebugEnabled()) {
     LOG.debug("allocate: applicationAttemptId=" 
         + this.getApplicationAttemptId() 
         + " container=" + containerId + " host="
         + rmContainer.getContainer().getNodeId().getHost() 
       );
   }
 
 return true;
 
}
 

synchronized public RMContainer allocate(NodeType type, FiCaSchedulerNode node,
    Priority priority, ResourceRequest request, 
    Container container) {

  if (isStopped) {
    return null;
  }
  
  // Required sanity check - AM can call 'allocate' to update resource 
  // request without locking the scheduler, hence we need to check
  if (getTotalRequiredResources(priority) <= 0) {
    return null;
  }
  
  // Create RMContainer
  RMContainer rmContainer = new RMContainerImpl(container, this
      .getApplicationAttemptId(), node.getNodeID(),
      appSchedulingInfo.getUser(), this.rmContext);

  // Add it to allContainers list.
  newlyAllocatedContainers.add(rmContainer);
  liveContainers.put(container.getId(), rmContainer);    

  // Update consumption and track allocations
  List<ResourceRequest> resourceRequestList = appSchedulingInfo.allocate(
      type, node, priority, request, container);
  Resources.addTo(currentConsumption, container.getResource());
  
  // Update resource requests related to "request" and store in RMContainer 
  ((RMContainerImpl)rmContainer).setResourceRequests(resourceRequestList);

  // Inform the container
  rmContainer.handle(
      new RMContainerEvent(container.getId(), RMContainerEventType.START));

  if (LOG.isDebugEnabled()) {
    LOG.debug("allocate: applicationAttemptId=" 
        + container.getId().getApplicationAttemptId() 
        + " container=" + container.getId() + " host="
        + container.getNodeId().getHost() + " type=" + type);
  }
  RMAuditLogger.logSuccess(getUser(), 
      AuditConstants.ALLOC_CONTAINER, "SchedulerApp", 
      getApplicationId(), container.getId());
  
  return rmContainer;
}