Java Code Examples for org.apache.hadoop.yarn.api.records.Container#getId()

public RMContainerImpl(Container container,
    ApplicationAttemptId appAttemptId, NodeId nodeId,
    String user, RMContext rmContext, long creationTime) {
  this.stateMachine = stateMachineFactory.make(this);
  this.containerId = container.getId();
  this.nodeId = nodeId;
  this.container = container;
  this.appAttemptId = appAttemptId;
  this.user = user;
  this.creationTime = creationTime;
  this.rmContext = rmContext;
  this.eventHandler = rmContext.getDispatcher().getEventHandler();
  this.containerAllocationExpirer = rmContext.getContainerAllocationExpirer();
  this.isAMContainer = false;
  this.resourceRequests = null;

  ReentrantReadWriteLock lock = new ReentrantReadWriteLock();
  this.readLock = lock.readLock();
  this.writeLock = lock.writeLock();

  rmContext.getRMApplicationHistoryWriter().containerStarted(this);
  rmContext.getSystemMetricsPublisher().containerCreated(
      this, this.creationTime);
}
 

/**
 * launch a new container with the given life time
 */
public void addNewContainer(Container container, long lifeTimeMS) {
  LOG.debug(MessageFormat.format("NodeManager {0} launches a new " +
          "container ({1}).", node.getNodeID(), container.getId()));
  if (lifeTimeMS != -1) {
    // normal container
    ContainerSimulator cs = new ContainerSimulator(container.getId(),
            container.getResource(), lifeTimeMS + System.currentTimeMillis(),
            lifeTimeMS);
    containerQueue.add(cs);
    runningContainers.put(cs.getId(), cs);
  } else {
    // AM container
    // -1 means AMContainer
    synchronized(amContainerList) {
      amContainerList.add(container.getId());
    }
  }
}
 

@Override
public void transition(SubQuery subQuery, SubQueryEvent event) {
  try {
    SubQueryContainerAllocationEvent allocationEvent =
        (SubQueryContainerAllocationEvent) event;
    for (Container container : allocationEvent.getAllocatedContainer()) {
      ContainerId cId = container.getId();
      if (subQuery.containers.containsKey(cId)) {
        subQuery.eventHandler.handle(new SubQueryDiagnosticsUpdateEvent(subQuery.getId(),
            "Duplicated containers are allocated: " + cId.toString()));
        subQuery.eventHandler.handle(new SubQueryEvent(subQuery.getId(), SubQueryEventType.SQ_INTERNAL_ERROR));
      }
      subQuery.containers.put(cId, container);
    }
    LOG.info("SubQuery (" + subQuery.getId() + ") has " + subQuery.containers.size() + " containers!");
    subQuery.eventHandler.handle(
        new TaskRunnerGroupEvent(EventType.CONTAINER_REMOTE_LAUNCH,
            subQuery.getId(), allocationEvent.getAllocatedContainer()));

    subQuery.eventHandler.handle(new SubQueryEvent(subQuery.getId(), SubQueryEventType.SQ_START));
  } catch (Throwable t) {
    subQuery.eventHandler.handle(new SubQueryDiagnosticsUpdateEvent(subQuery.getId(),
        ExceptionUtils.getStackTrace(t)));
    subQuery.eventHandler.handle(new SubQueryEvent(subQuery.getId(), SubQueryEventType.SQ_INTERNAL_ERROR));
  }
}
 

protected ContainerId storeAttempt(RMStateStore store,
    ApplicationAttemptId attemptId,
    String containerIdStr, Token<AMRMTokenIdentifier> appToken,
    SecretKey clientTokenMasterKey, TestDispatcher dispatcher)
    throws Exception {

  RMAppAttemptMetrics mockRmAppAttemptMetrics = 
      mock(RMAppAttemptMetrics.class);
  Container container = new ContainerPBImpl();
  container.setId(ConverterUtils.toContainerId(containerIdStr));
  RMAppAttempt mockAttempt = mock(RMAppAttempt.class);
  when(mockAttempt.getAppAttemptId()).thenReturn(attemptId);
  when(mockAttempt.getMasterContainer()).thenReturn(container);
  when(mockAttempt.getAMRMToken()).thenReturn(appToken);
  when(mockAttempt.getClientTokenMasterKey())
      .thenReturn(clientTokenMasterKey);
  when(mockAttempt.getRMAppAttemptMetrics())
      .thenReturn(mockRmAppAttemptMetrics);
  when(mockRmAppAttemptMetrics.getAggregateAppResourceUsage())
      .thenReturn(new AggregateAppResourceUsage(0, 0));
  dispatcher.attemptId = attemptId;
  store.storeNewApplicationAttempt(mockAttempt);
  waitNotify(dispatcher);
  return container.getId();
}
 

private void pushNewContainerToDelayed(List<Container> containers){
  long expireTime = getHeldContainerExpireTime(System.currentTimeMillis());

  synchronized (delayedContainerManager) {
    for (Container container : containers) {
      if (heldContainers.put(container.getId(), new HeldContainer(container,
          -1, expireTime, null, this.containerSignatureMatcher)) != null) {
        throw new TezUncheckedException("New container " + container.getId()
            + " is already held.");
      }
      long nextScheduleTime = delayedContainerManager.maxScheduleTimeSeen;
      if (delayedContainerManager.maxScheduleTimeSeen == -1) {
        nextScheduleTime = System.currentTimeMillis();
      }
      Resources.addTo(allocatedResources, container.getResource());
      delayedContainerManager.addDelayedContainer(container,
        nextScheduleTime + 1);
    }
  }
  delayedContainerManager.triggerScheduling(false);      
}
 

/**
 * launch a new container with the given life time
 */
public void addNewContainer(Container container, long lifeTimeMS) {
  LOG.debug(MessageFormat.format("NodeManager {0} launches a new " +
          "container ({1}).", node.getNodeID(), container.getId()));
  if (lifeTimeMS != -1) {
    // normal container
    ContainerSimulator cs = new ContainerSimulator(container.getId(),
            container.getResource(), lifeTimeMS + System.currentTimeMillis(),
            lifeTimeMS);
    containerQueue.add(cs);
    runningContainers.put(cs.getId(), cs);
  } else {
    // AM container
    // -1 means AMContainer
    synchronized(amContainerList) {
      amContainerList.add(container.getId());
    }
  }
}
 

public AMContainerImpl(Container container, ContainerHeartbeatHandler chh,
    TaskCommunicatorManagerInterface tal, ContainerSignatureMatcher signatureMatcher,
    AppContext appContext, int schedulerId, int launcherId, int taskCommId, String auxiliaryService) {
  ReentrantReadWriteLock rwLock = new ReentrantReadWriteLock();
  this.readLock = rwLock.readLock();
  this.writeLock = rwLock.writeLock();
  this.container = container;
  this.containerId = container.getId();
  this.eventHandler = appContext.getEventHandler();
  this.signatureMatcher = signatureMatcher;
  this.appContext = appContext;
  this.containerHeartbeatHandler = chh;
  this.taskCommunicatorManagerInterface = tal;
  this.failedAssignments = new LinkedList<TezTaskAttemptID>();
  this.schedulerId = schedulerId;
  this.launcherId = launcherId;
  this.taskCommId = taskCommId;
  this.auxiliaryService = auxiliaryService;
  this.stateMachine = new StateMachineTez<>(stateMachineFactory.make(this), this);
  augmentStateMachine();
}
 

private void pushNewContainerToDelayed(List<Container> containers){
  long expireTime = -1;
  if (idleContainerTimeoutMin > 0) {
    long currentTime = System.currentTimeMillis();
    expireTime = currentTime + idleContainerTimeoutMin;
  }

  synchronized (delayedContainerManager) {
    for (Container container : containers) {
      if (heldContainers.put(container.getId(), new HeldContainer(container,
          -1, expireTime, null)) != null) {
        throw new TezUncheckedException("New container " + container.getId()
            + " is already held.");
      }
      long nextScheduleTime = delayedContainerManager.maxScheduleTimeSeen;
      if (delayedContainerManager.maxScheduleTimeSeen == -1) {
        nextScheduleTime = System.currentTimeMillis();
      }
      Resources.addTo(allocatedResources, container.getResource());
      delayedContainerManager.addDelayedContainer(container,
        nextScheduleTime + 1);
    }
  }
  delayedContainerManager.triggerScheduling(false);      
}
 

public StartContainerEvent(Container container,
    ContainerLaunchContext containerLaunchContext) {
  super(container.getId(), container.getNodeId(),
      container.getContainerToken(), ContainerEventType.START_CONTAINER);
  this.container = container;
  this.containerLaunchContext = containerLaunchContext;
}
 

public ContainerLauncherLaunchRequestEvent(ContainerLaunchContext clc,
                                           Container container, int launcherId, int schedulerId,
                                           int taskCommId) {
  super(container.getId(), container.getNodeId(), container
      .getContainerToken(), ContainerLauncherEventType.CONTAINER_LAUNCH_REQUEST,
      launcherId, schedulerId, taskCommId);
  this.clc = clc;
  this.container = container;
}
 

ContainerId get(TaskAttemptId tId) {
  Container taskContainer;
  if (tId.getTaskId().getTaskType().equals(TaskType.MAP)) {
    taskContainer = maps.get(tId);
  } else {
    taskContainer = reduces.get(tId);
  }

  if (taskContainer == null) {
    return null;
  } else {
    return taskContainer.getId();
  }
}
 

public RMContainerImpl(Container container,
    ApplicationAttemptId appAttemptId, NodeId nodeId,
    String user, RMContext rmContext, long creationTime) {
  this.stateMachine = stateMachineFactory.make(this);
  this.containerId = container.getId();
  this.nodeId = nodeId;
  this.container = container;
  this.appAttemptId = appAttemptId;
  this.user = user;
  this.creationTime = creationTime;
  this.rmContext = rmContext;
  this.eventHandler = rmContext.getDispatcher().getEventHandler();
  this.containerAllocationExpirer = rmContext.getContainerAllocationExpirer();
  this.isAMContainer = false;
  this.resourceRequests  = null;
  this.resumeOpportunity = 0;
  
  this.utilization = 1;
  this.suspendTime = new LinkedList<Long>();
  this.resumeTime  = new LinkedList<Long>();

  ReentrantReadWriteLock lock = new ReentrantReadWriteLock();
  this.readLock = lock.readLock();
  this.writeLock = lock.writeLock();
  
  this.PR_NUMBER=rmContext.getYarnConfiguration().getInt(
  		"yarn.resourcemanager.monitor.capacity.preemption.pr_number", 2);
  		

  rmContext.getRMApplicationHistoryWriter().containerStarted(this);
  rmContext.getSystemMetricsPublisher().containerCreated(
      this, this.creationTime);
}
 

synchronized public void containerCompleted(RMContainer rmContainer,
    ContainerStatus containerStatus, RMContainerEventType event) {
  
  Container container = rmContainer.getContainer();
  ContainerId containerId = container.getId();
  
  // Remove from the list of newly allocated containers if found
  newlyAllocatedContainers.remove(rmContainer);
  
  // Inform the container
  rmContainer.handle(
      new RMContainerFinishedEvent(
          containerId,
          containerStatus, 
          event)
      );
  LOG.info("Completed container: " + rmContainer.getContainerId() + 
      " in state: " + rmContainer.getState() + " event:" + event);
  
  // Remove from the list of containers
  liveContainers.remove(rmContainer.getContainerId());

  RMAuditLogger.logSuccess(getUser(), 
      AuditConstants.RELEASE_CONTAINER, "SchedulerApp", 
      getApplicationId(), containerId);
  
  // Update usage metrics 
  Resource containerResource = rmContainer.getContainer().getResource();
  queue.getMetrics().releaseResources(getUser(), 1, containerResource);
  Resources.subtractFrom(currentConsumption, containerResource);

  // remove from preemption map if it is completed
  preemptionMap.remove(rmContainer);

  // Clear resource utilization metrics cache.
  lastMemoryAggregateAllocationUpdateTime = -1;
}
 

public ContainerProxy(QueryMasterTask.QueryMasterTaskContext context, Configuration conf,
                      ExecutionBlockId executionBlockId, Container container) {
  this.context = context;
  this.conf = conf;
  this.state = ContainerState.PREP;
  this.container = container;
  this.executionBlockId = executionBlockId;
  this.containerID = container.getId();
}
 

public ContainerRemoteLaunchEvent(TaskAttemptId taskAttemptID,
    ContainerLaunchContext containerLaunchContext,
    Container allocatedContainer, Task remoteTask) {
  super(taskAttemptID, allocatedContainer.getId(), StringInterner
    .weakIntern(allocatedContainer.getNodeId().toString()),
    allocatedContainer.getContainerToken(),
    ContainerLauncher.EventType.CONTAINER_REMOTE_LAUNCH);
  this.allocatedContainer = allocatedContainer;
  this.containerLaunchContext = containerLaunchContext;
  this.task = remoteTask;
}
 

ContainerId get(TaskAttemptId tId) {
  Container taskContainer;
  if (tId.getTaskId().getTaskType().equals(TaskType.MAP)) {
    taskContainer = maps.get(tId);
  } else {
    taskContainer = reduces.get(tId);
  }

  if (taskContainer == null) {
    return null;
  } else {
    return taskContainer.getId();
  }
}
 

protected synchronized StartedContainer createStartedContainer(
    Container container) throws YarnException, IOException {
  StartedContainer startedContainer = new StartedContainer(container.getId(),
      container.getNodeId(), container.getContainerToken());
  return startedContainer;
}
 

synchronized public boolean containerSuspend(RMContainer rmContainer,
  ContainerStatus containerStatus, RMContainerEventType event){
 //we try to find it from live container list
 LOG.info("app suspend "+rmContainer.getContainerId());
 if (!liveContainers.keySet().contains(rmContainer.getContainerId())){
  LOG.info("container not found "+rmContainer.getContainerId());
  return false;
 }
 
 isSuspending = true;

 Container container = rmContainer.getContainer();
 ContainerId containerId = container.getId();
 
 
 if(!this.containersSuspended.contains(rmContainer.getContainerId())){
 //add to suspended set if this container is first suspended
 containersSuspended.add(containerId);
 }
	
 // Inform the container
 
 rmContainer.handle(
       new RMContainerFinishedEvent(
           containerId,
           containerStatus, 
           event)
 );

 // Update usage metrics,we release resource here,to support increamental suspension  
 Resource toPreempted = rmContainer.getLastPreemptedResource();
 queue.getMetrics().releaseResources(getUser(), 1, toPreempted);
 Resources.subtractFrom(currentConsumption, toPreempted);

 LOG.info("app suspend container: " + rmContainer.getContainerId() + 
        " in state: " + rmContainer.getState() + " resource:" + toPreempted);
 
 // Clear resource utilization metrics cache.
 lastMemoryAggregateAllocationUpdateTime = -1;
 
 return true; 	  
}
 

protected synchronized StartedContainer createStartedContainer(
    Container container) throws YarnException, IOException {
  StartedContainer startedContainer = new StartedContainer(container.getId(),
      container.getNodeId(), container.getContainerToken());
  return startedContainer;
}
 

public synchronized void onContainersAllocated(List<Container> containers) {
  String zkHost = cli.getOptionValue("zkHost");
  String solrArchive = cli.getOptionValue("solr");
  String hdfsHome = cli.getOptionValue("hdfs_home");

  Path pathToRes = new Path(solrArchive);
  FileStatus jarStat = null;
  try {
    jarStat = FileSystem.get(conf).getFileStatus(pathToRes);
  } catch (IOException e) {
    throw new RuntimeException(e);
  }
  LocalResource solrPackageRes = Records.newRecord(LocalResource.class);
  solrPackageRes.setResource(ConverterUtils.getYarnUrlFromPath(pathToRes));
  solrPackageRes.setSize(jarStat.getLen());
  solrPackageRes.setTimestamp(jarStat.getModificationTime());
  solrPackageRes.setType(LocalResourceType.ARCHIVE);
  solrPackageRes.setVisibility(LocalResourceVisibility.APPLICATION);

  Map<String, LocalResource> localResourcesMap = new HashMap<String, LocalResource>();
  localResourcesMap.put("solr", solrPackageRes);

  String acceptShutdownFrom = "-Dyarn.acceptShutdownFrom=" + inetAddresses;

  log.info("Using " + acceptShutdownFrom);

  String dasha = "";
  if (hdfsHome != null) {
    dasha += " -a '-Dsolr.hdfs.home=" + hdfsHome + " -Dsolr.directoryFactory=HdfsDirectoryFactory -Dsolr.lock.type=hdfs %s'";
  } else {
    dasha += "-a '%s'";
  }

  dasha = String.format(dasha, acceptShutdownFrom);

  String command = "/bin/bash ./solr/bin/solr -f -c -p %d -k %s -m " + memory + "m -z " + zkHost + dasha + " -V";
  for (Container container : containers) {
    ContainerId containerId = container.getId();

    // increment the port if running on the same host
    int jettyPort = nextPort++;
    String jettyHost = container.getNodeId().getHost();
    Set<Integer> portsOnHost = solrHosts.get(jettyHost);
    if (portsOnHost == null) {
      portsOnHost = new HashSet<Integer>();
      solrHosts.put(jettyHost, portsOnHost);
    }
    portsOnHost.add(jettyPort);
    log.info("Added port " + jettyPort + " to host: " + jettyHost);

    try {
      // Launch container by create ContainerLaunchContext
      ContainerLaunchContext ctx = Records.newRecord(ContainerLaunchContext.class);
      ctx.setLocalResources(localResourcesMap);

      String cmd = String.format(command, jettyPort, randomStopKey);
      log.info("\n\nRunning command: " + cmd);

      ctx.setCommands(Collections.singletonList(
              cmd + " >" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/stdout 2>&1"
      ));
      log.info("Launching container " + containerId);
      nmClient.startContainer(container, ctx);
    } catch (Exception exc) {
      log.error("Failed to start container to run Solr on port " + jettyPort + " due to: " + exc, exc);
    }
  }
}