org.apache.hadoop.yarn.api.records.Container Java Examples

@Test
public void testFinishingExpire() {
  Container amContainer = allocateApplicationAttempt();
  launchApplicationAttempt(amContainer);
  runApplicationAttempt(amContainer, "host", 8042, "oldtrackingurl", false);
  FinalApplicationStatus finalStatus = FinalApplicationStatus.SUCCEEDED;
  String trackingUrl = "mytrackingurl";
  String diagnostics = "Successful";
  unregisterApplicationAttempt(amContainer, finalStatus, trackingUrl,
      diagnostics);
  applicationAttempt.handle(
      new RMAppAttemptEvent(
          applicationAttempt.getAppAttemptId(),
          RMAppAttemptEventType.EXPIRE));
  testAppAttemptFinishedState(amContainer, finalStatus, trackingUrl,
      diagnostics, 0, false);
}
 

@SuppressWarnings("deprecation")
@Test
public void testAllocateResponseWithoutIncDecContainers() {
  AllocateResponse r =
      AllocateResponse.newInstance(3, new ArrayList<ContainerStatus>(),
          new ArrayList<Container>(), new ArrayList<NodeReport>(), null,
          AMCommand.AM_RESYNC, 3, null, new ArrayList<NMToken>(), null, null);

  // serde
  AllocateResponseProto p = ((AllocateResponsePBImpl) r).getProto();
  r = new AllocateResponsePBImpl(p);

  // check value
  Assert.assertEquals(0, r.getIncreasedContainers().size());
  Assert.assertEquals(0, r.getDecreasedContainers().size());
}
 

@Test (timeout = 10000)
public void testAMRMClientAsyncShutDown() throws Exception {
  Configuration conf = new Configuration();
  TestCallbackHandler callbackHandler = new TestCallbackHandler();
  @SuppressWarnings("unchecked")
  AMRMClient<ContainerRequest> client = mock(AMRMClientImpl.class);

  createAllocateResponse(new ArrayList<ContainerStatus>(),
    new ArrayList<Container>(), null);
  when(client.allocate(anyFloat())).thenThrow(
    new ApplicationAttemptNotFoundException("app not found, shut down"));

  AMRMClientAsync<ContainerRequest> asyncClient =
      AMRMClientAsync.createAMRMClientAsync(client, 10, callbackHandler);
  asyncClient.init(conf);
  asyncClient.start();

  asyncClient.registerApplicationMaster("localhost", 1234, null);

  Thread.sleep(50);

  verify(client, times(1)).allocate(anyFloat());
  asyncClient.stop();
}
 

private void pushNewContainerToDelayed(List<Container> containers){
  long expireTime = -1;
  if (idleContainerTimeoutMin > 0) {
    long currentTime = System.currentTimeMillis();
    expireTime = currentTime + idleContainerTimeoutMin;
  }

  synchronized (delayedContainerManager) {
    for (Container container : containers) {
      if (heldContainers.put(container.getId(), new HeldContainer(container,
          -1, expireTime, null)) != null) {
        throw new TezUncheckedException("New container " + container.getId()
            + " is already held.");
      }
      long nextScheduleTime = delayedContainerManager.maxScheduleTimeSeen;
      if (delayedContainerManager.maxScheduleTimeSeen == -1) {
        nextScheduleTime = System.currentTimeMillis();
      }
      Resources.addTo(allocatedResources, container.getResource());
      delayedContainerManager.addDelayedContainer(container,
        nextScheduleTime + 1);
    }
  }
  delayedContainerManager.triggerScheduling(false);      
}
 

public void startContainerAsync(
    Container container, ContainerLaunchContext containerLaunchContext) {
  if (containers.putIfAbsent(container.getId(),
      new StatefulContainer(this, container.getId())) != null) {
    callbackHandler.onStartContainerError(container.getId(),
        RPCUtil.getRemoteException("Container " + container.getId() +
            " is already started or scheduled to start"));
  }
  try {
    events.put(new StartContainerEvent(container, containerLaunchContext));
  } catch (InterruptedException e) {
    LOG.warn("Exception when scheduling the event of starting Container " +
        container.getId());
    callbackHandler.onStartContainerError(container.getId(), e);
  }
}
 

private boolean canAssignTaskToContainer(
    CookieContainerRequest cookieContainerRequest, Container container) {
  HeldContainer heldContainer = heldContainers.get(container.getId());
  if (heldContainer == null || heldContainer.isNew()) { // New container.
    return true;
  } else {
    if (LOG.isDebugEnabled()) {
      LOG.debug("Trying to match task to a held container, "
          + " containerId=" + heldContainer.container.getId());
    }
    if (containerSignatureMatcher.isSuperSet(heldContainer
        .getFirstContainerSignature(), cookieContainerRequest.getCookie()
        .getContainerSignature())) {
      if (LOG.isDebugEnabled()) {
        LOG.debug("Matched delayed container to task"
          + " containerId=" + heldContainer.container.getId());
      }
      return true;
    }
  }
  if (LOG.isDebugEnabled()) {
    LOG.debug("Failed to match delayed container to task"
      + " containerId=" + heldContainer.container.getId());
  }
  return false;
}
 

@Test
public void testFinishingExpire() {
  Container amContainer = allocateApplicationAttempt();
  launchApplicationAttempt(amContainer);
  runApplicationAttempt(amContainer, "host", 8042, "oldtrackingurl", false);
  FinalApplicationStatus finalStatus = FinalApplicationStatus.SUCCEEDED;
  String trackingUrl = "mytrackingurl";
  String diagnostics = "Successful";
  unregisterApplicationAttempt(amContainer, finalStatus, trackingUrl,
      diagnostics);
  applicationAttempt.handle(
      new RMAppAttemptEvent(
          applicationAttempt.getAppAttemptId(),
          RMAppAttemptEventType.EXPIRE));
  testAppAttemptFinishedState(amContainer, finalStatus, trackingUrl,
      diagnostics, 0, false);
}
 

/**
 * Tries assigning the list of specified containers. Optionally, release
 * containers or add them to the delayed container queue.
 *
 * The flags apply to all containers in the specified lists. So, separate
 * calls should be made based on the expected behaviour.
 *
 * @param containers
 *          The list of containers to be assigned. The list *may* be modified
 *          in place based on allocations and releases.
 * @return Assignments.
 */
private synchronized Map<CookieContainerRequest, Container>
    assignNewlyAllocatedContainers(Iterable<Container> containers) {

  boolean amInCompletionState = (getContext().getAMState() == AMState.COMPLETED);
  Map<CookieContainerRequest, Container> assignedContainers =
      new HashMap<CookieContainerRequest, Container>();

  if (!amInCompletionState) {
    assignNewContainersWithLocation(containers,
        NODE_LOCAL_ASSIGNER, assignedContainers);
    assignNewContainersWithLocation(containers,
        RACK_LOCAL_ASSIGNER, assignedContainers);
    assignNewContainersWithLocation(containers,
        NON_LOCAL_ASSIGNER, assignedContainers);
  }

  // Release any unassigned containers given by the RM
  if (containers.iterator().hasNext()) {
    LOG.info("Releasing newly assigned containers which could not be allocated");
  }
  releaseUnassignedContainers(containers);

  return assignedContainers;
}
 

@SuppressWarnings("unchecked")
private void containerAssigned(Container allocated, 
                                ContainerRequest assigned) {
  // Update resource requests
  decContainerReq(assigned);

  // send the container-assigned event to task attempt
  eventHandler.handle(new TaskAttemptContainerAssignedEvent(
      assigned.attemptID, allocated, applicationACLs));

  assignedRequests.add(allocated, assigned.attemptID);

  if (LOG.isDebugEnabled()) {
    LOG.info("Assigned container (" + allocated + ") "
        + " to task " + assigned.attemptID + " on node "
        + allocated.getNodeId().toString());
  }
}
 

private ContainerRequest assignWithoutLocality(Container allocated) {
  ContainerRequest assigned = null;
  
  Priority priority = allocated.getPriority();
  if (PRIORITY_FAST_FAIL_MAP.equals(priority)) {
    LOG.info("Assigning container " + allocated + " to fast fail map");
    assigned = assignToFailedMap(allocated);
  } else if (PRIORITY_REDUCE.equals(priority)) {
    if (LOG.isDebugEnabled()) {
      LOG.debug("Assigning container " + allocated + " to reduce");
    }
    assigned = assignToReduce(allocated);
  }
    
  return assigned;
}
 

@Override
public TaskInstance getTask(Container container) {
	numberOfRemainingTasks--;
	numberOfRunningTasks++;
	String node = container.getNodeId().getHost();

	if (HiWayConfiguration.verbose)
		WorkflowDriver.writeToStdout("Looking for task on container " + container.getId() + " on node " + node + "; Queue:" + queues.get(node).toString());

	TaskInstance task = queues.get(node).remove();

	WorkflowDriver.writeToStdout("Assigned task " + task + " to container " + container.getId() + "@" + node);
	task.incTries();

	return task;
}
 

public void doBookKeepingForAssignedContainer(
    CookieContainerRequest assigned, Container container,
    String matchedLocation, boolean honorLocalityFlags) {
  if (assigned == null) {
    return;
  }
  Object task = getTask(assigned);
  assert task != null;

  LOG.info("Assigning container to task"
    + ", container=" + container
    + ", task=" + task
    + ", containerHost=" + container.getNodeId().getHost()
    + ", localityMatchType=" + locality
    + ", matchedLocation=" + matchedLocation
    + ", honorLocalityFlags=" + honorLocalityFlags
    + ", reusedContainer="
    + containerAssignments.containsKey(container.getId())
    + ", delayedContainers=" + delayedContainerManager.delayedContainers.size()
    + ", containerResourceMemory=" + container.getResource().getMemory()
    + ", containerResourceVCores="
    + container.getResource().getVirtualCores());

  assignContainer(task, container, assigned);
}
 

private static RMAppAttempt createRMAppAttempt(
    ApplicationAttemptId appAttemptId) {
  RMAppAttempt appAttempt = mock(RMAppAttempt.class);
  when(appAttempt.getAppAttemptId()).thenReturn(appAttemptId);
  when(appAttempt.getHost()).thenReturn("test host");
  when(appAttempt.getRpcPort()).thenReturn(-100);
  Container container = mock(Container.class);
  when(container.getId())
      .thenReturn(ContainerId.newContainerId(appAttemptId, 1));
  when(appAttempt.getMasterContainer()).thenReturn(container);
  when(appAttempt.getDiagnostics()).thenReturn("test diagnostics info");
  when(appAttempt.getTrackingUrl()).thenReturn("test tracking url");
  when(appAttempt.getOriginalTrackingUrl()).thenReturn(
      "test original tracking url");
  return appAttempt;
}
 

@Override
public void onContainersAllocated(List<Container> allocatedContainers) {
  allocatedContainerNum.addAndGet(allocatedContainers.size());
  ApplicationMaster.this.allocatedContainers.addAll(allocatedContainers);
  if (allocatedContainerNum.get() == args.totalContainerNum) {
    startAllContainers();
  }
}
 

private boolean canAssignTaskToContainer(
    CookieContainerRequest cookieContainerRequest, Container container) {
  HeldContainer heldContainer = heldContainers.get(container.getId());
  Object task = getTask(cookieContainerRequest);
  if (task instanceof TaskAttempt
      && ((TaskAttempt) task).getTask() != null
      && ((TaskAttempt) task).getTask().getNodesWithRunningAttempts().contains(container.getNodeId())) {
    return false;
  }
  if (heldContainer == null || heldContainer.isNew()) { // New container.
    return true;
  } else {
    if (LOG.isDebugEnabled()) {
      LOG.debug("Trying to match task to a held container, "
          + " containerId=" + heldContainer.container.getId());
    }
    if (containerSignatureMatcher.isSuperSet(heldContainer
        .getLastAssignedContainerSignature(), cookieContainerRequest.getCookie()
        .getContainerSignature())) {
      if (LOG.isDebugEnabled()) {
        LOG.debug("Matched delayed container to task"
          + " containerId=" + heldContainer.container.getId());
      }
      return true;
    }
  }
  if (LOG.isDebugEnabled()) {
    LOG.debug("Failed to match delayed container to task"
      + " containerId=" + heldContainer.container.getId());
  }
  return false;
}
 

public synchronized List<Container> getResources() throws IOException {
  if(LOG.isDebugEnabled()) {
    LOG.debug("getResources begin:" + " application=" + applicationId
      + " #ask=" + ask.size());

    for (ResourceRequest request : ask) {
      LOG.debug("getResources:" + " application=" + applicationId
        + " ask-request=" + request);
    }
  }
  
  // Get resources from the ResourceManager
  Allocation allocation = resourceManager.getResourceScheduler().allocate(
      applicationAttemptId, new ArrayList<ResourceRequest>(ask),
      new ArrayList<ContainerId>(), null, null);
  System.out.println("-=======" + applicationAttemptId);
  System.out.println("----------" + resourceManager.getRMContext().getRMApps()
      .get(applicationId).getRMAppAttempt(applicationAttemptId));
  List<Container> containers = allocation.getContainers();

  // Clear state for next interaction with ResourceManager
  ask.clear();
  
  if(LOG.isDebugEnabled()) {
    LOG.debug("getResources() for " + applicationId + ":"
      + " ask=" + ask.size() + " recieved=" + containers.size());
  }
  
  return containers;
}
 

@Override
public void onContainerStarted(ContainerId containerId, Map<String, ByteBuffer> allServiceResponse) {
	Container container = containers.get(containerId);
	if (container != null) {
		am.getNmClientAsync().getContainerStatusAsync(containerId, container.getNodeId());
	}
}
 

private synchronized void initLocalNewContainerList() {
  if (this.allocatedContainers != null) {
    return;
  }
  AllocateResponseProtoOrBuilder p = viaProto ? proto : builder;
  List<ContainerProto> list = p.getAllocatedContainersList();
  allocatedContainers = new ArrayList<Container>();

  for (ContainerProto c : list) {
    allocatedContainers.add(convertFromProtoFormat(c));
  }
}
 

public List<Container> takeAllocatedContainers() {
  List<Container> ret = allocatedContainers;
  if (ret == null) {
    return null;
  }
  allocatedContainers = null;
  synchronized (ret) {
    ret.notify();
  }
  return ret;
}
 

@Override
public int compareTo(Container other) {
  if (this.getId().compareTo(other.getId()) == 0) {
    if (this.getNodeId().compareTo(other.getNodeId()) == 0) {
      return this.getResource().compareTo(other.getResource());
    } else {
      return this.getNodeId().compareTo(other.getNodeId());
    }
  } else {
    return this.getId().compareTo(other.getId());
  }
}
 

AMContainer createAmContainer(Container container,
                              ContainerHeartbeatHandler chh,
                              TaskCommunicatorManagerInterface tal,
                              ContainerSignatureMatcher signatureMatcher,
                              AppContext appContext, int schedulerId,
                              int launcherId, int taskCommId, String auxiliaryService) {
  AMContainer amc = new AMContainerImpl(container, chh, tal,
      signatureMatcher, appContext, schedulerId, launcherId, taskCommId, auxiliaryService);
  return amc;
}
 

private void assignContainers(List<Container> allocatedContainers) {
  Iterator<Container> it = allocatedContainers.iterator();
  while (it.hasNext()) {
    Container allocated = it.next();
    ContainerRequest assigned = assignWithoutLocality(allocated);
    if (assigned != null) {
      containerAssigned(allocated, assigned);
      it.remove();
    }
  }

  assignMapsWithLocality(allocatedContainers);
}
 

@Test
public void
    testFinalSavingToFinishedWithContainerFinished() {
  Container amContainer = allocateApplicationAttempt();
  launchApplicationAttempt(amContainer);
  runApplicationAttempt(amContainer, "host", 8042, "oldtrackingurl", false);
  FinalApplicationStatus finalStatus = FinalApplicationStatus.SUCCEEDED;
  String trackingUrl = "mytrackingurl";
  String diagnostics = "Successful";
  applicationAttempt.handle(new RMAppAttemptUnregistrationEvent(
    applicationAttempt.getAppAttemptId(), trackingUrl, finalStatus,
    diagnostics));
  assertEquals(RMAppAttemptState.FINAL_SAVING,
    applicationAttempt.getAppAttemptState());
  assertEquals(YarnApplicationAttemptState.RUNNING,
      applicationAttempt.createApplicationAttemptState());
  // Container_finished event comes before Attempt_Saved event.
  NodeId anyNodeId = NodeId.newInstance("host", 1234);
  applicationAttempt.handle(new RMAppAttemptContainerFinishedEvent(
    applicationAttempt.getAppAttemptId(), BuilderUtils.newContainerStatus(
      amContainer.getId(), ContainerState.COMPLETE, "", 0), anyNodeId));
  assertEquals(RMAppAttemptState.FINAL_SAVING,
    applicationAttempt.getAppAttemptState());
  // send attempt_saved
  sendAttemptUpdateSavedEvent(applicationAttempt);
  testAppAttemptFinishedState(amContainer, finalStatus, trackingUrl,
    diagnostics, 0, false);
}
 

private void launchTaskRunners(ExecutionBlockId executionBlockId, Collection<Container> containers) {
  // Query in standby mode doesn't need launch Worker.
  // But, Assign ExecutionBlock to assigned tajo worker
  for(Container eachContainer: containers) {
    TajoContainerProxy containerProxy = new TajoContainerProxy(queryTaskContext, tajoConf,
        eachContainer, executionBlockId);
    executorService.submit(new LaunchRunner(eachContainer.getId(), containerProxy));
  }
}
 

@Override
public void handle(ContainerAllocatorEvent event) {
  ContainerId cId =
      ContainerId.newContainerId(getContext().getApplicationAttemptId(),
        containerCount++);
  NodeId nodeId = NodeId.newInstance(NM_HOST, NM_PORT);
  Resource resource = Resource.newInstance(1234, 2);
  ContainerTokenIdentifier containerTokenIdentifier =
      new ContainerTokenIdentifier(cId, nodeId.toString(), "user",
      resource, System.currentTimeMillis() + 10000, 42, 42,
      Priority.newInstance(0), 0);
  Token containerToken = newContainerToken(nodeId, "password".getBytes(),
        containerTokenIdentifier);
  Container container = Container.newInstance(cId, nodeId,
      NM_HOST + ":" + NM_HTTP_PORT, resource, null, containerToken);
  JobID id = TypeConverter.fromYarn(applicationId);
  JobId jobId = TypeConverter.toYarn(id);
  getContext().getEventHandler().handle(new JobHistoryEvent(jobId, 
      new NormalizedResourceEvent(
          org.apache.hadoop.mapreduce.TaskType.REDUCE,
      100)));
  getContext().getEventHandler().handle(new JobHistoryEvent(jobId, 
      new NormalizedResourceEvent(
          org.apache.hadoop.mapreduce.TaskType.MAP,
      100)));
  getContext().getEventHandler().handle(
      new TaskAttemptContainerAssignedEvent(event.getAttemptID(),
          container, null));
}
 

public static ApplicationAttemptStateData newInstance(
  ApplicationAttemptId attemptId, Container masterContainer,
  Credentials attemptTokens, long startTime, long memorySeconds,
  long vcoreSeconds, long gcoreSeconds) {
return newInstance(attemptId, masterContainer, attemptTokens,
    startTime, null, "N/A", "", null, ContainerExitStatus.INVALID, 0,
    memorySeconds, vcoreSeconds, gcoreSeconds);
}
 

private void runApplicationAttempt(Container container,
    String host, 
    int rpcPort, 
    String trackingUrl, boolean unmanagedAM) {
  applicationAttempt.handle(
      new RMAppAttemptRegistrationEvent(
          applicationAttempt.getAppAttemptId(),
          host, rpcPort, trackingUrl));
  
  testAppAttemptRunningState(container, host, rpcPort, trackingUrl, 
      unmanagedAM);
}
 

@Lock(CapacityScheduler.class)
@Override
protected synchronized void completedContainer(RMContainer rmContainer,
    ContainerStatus containerStatus, RMContainerEventType event) {
  if (rmContainer == null) {
    LOG.info("Null container completed...");
    return;
  }
  
  Container container = rmContainer.getContainer();
  
  // Get the application for the finished container
  FiCaSchedulerApp application =
      getCurrentAttemptForContainer(container.getId());
  ApplicationId appId =
      container.getId().getApplicationAttemptId().getApplicationId();
  if (application == null) {
    LOG.info("Container " + container + " of" + " unknown application "
        + appId + " completed or suspended with event " + event);
    return;
  }
  
  // Get the node on which the container was allocated
  FiCaSchedulerNode node = getNode(container.getNodeId());
  
  // Inform the queue
  LeafQueue queue = (LeafQueue)application.getQueue();
  queue.completedContainer(clusterResource, application, node, 
      rmContainer, containerStatus, event, null, true);

  LOG.info("Application attempt " + application.getApplicationAttemptId()
      + " released container " + container.getId() + " on node: " + node
      + " with event: " + event);
}
 

@Test
public void testUnregisterToSuccessfulFinishing() {
  Container amContainer = allocateApplicationAttempt();
  launchApplicationAttempt(amContainer);
  runApplicationAttempt(amContainer, "host", 8042, "oldtrackingurl", false);
  unregisterApplicationAttempt(amContainer,
      FinalApplicationStatus.SUCCEEDED, "mytrackingurl", "Successful");
}
 

@VisibleForTesting
void addDelayedContainer(Container container,
    long nextScheduleTime) {
  HeldContainer delayedContainer = heldContainers.get(container.getId());
  if (delayedContainer == null) {
    LOG.warn("Attempting to add a non-running container to the"
        + " delayed container list, containerId=" + container.getId());
    return;
  } else {
    delayedContainer.setNextScheduleTime(nextScheduleTime);
  }
  if (maxScheduleTimeSeen < nextScheduleTime) {
    maxScheduleTimeSeen = nextScheduleTime;
  }
  if (LOG.isDebugEnabled()) {
    LOG.debug("Adding container to delayed queue"
      + ", containerId=" + delayedContainer.getContainer().getId()
      + ", nextScheduleTime=" + delayedContainer.getNextScheduleTime()
      + ", containerExpiry=" + delayedContainer.getContainerExpiryTime());
  }
  boolean added =  false;
  synchronized(this) {
    added = delayedContainers.offer(delayedContainer);
    if (drainedDelayedContainersForTest != null) {
      synchronized (drainedDelayedContainersForTest) {
        drainedDelayedContainersForTest.set(false);
      }
    }
    this.notify();
  }
  if (!added) {
    releaseUnassignedContainers(Lists.newArrayList(container));
  }
}