Java Code Examples for org.apache.hadoop.yarn.api.records.ContainerId#toString()

@Override
public void removeContainer(ContainerId containerId)
    throws IOException {
  if (LOG.isDebugEnabled()) {
    LOG.debug("removeContainer: containerId=" + containerId);
  }

  String keyPrefix = CONTAINERS_KEY_PREFIX + containerId.toString();
  try {
    WriteBatch batch = db.createWriteBatch();
    try {
      batch.delete(bytes(keyPrefix + CONTAINER_REQUEST_KEY_SUFFIX));
      batch.delete(bytes(keyPrefix + CONTAINER_DIAGS_KEY_SUFFIX));
      batch.delete(bytes(keyPrefix + CONTAINER_LAUNCHED_KEY_SUFFIX));
      batch.delete(bytes(keyPrefix + CONTAINER_KILLED_KEY_SUFFIX));
      batch.delete(bytes(keyPrefix + CONTAINER_EXIT_CODE_KEY_SUFFIX));
      db.write(batch);
    } finally {
      batch.close();
    }
  } catch (DBException e) {
    throw new IOException(e);
  }
}
 

@Test
public void testGetTaskShouldDie() throws InterruptedException, ExecutionException {
  ListeningExecutorService executor = null;
  try {
    ExecutorService rawExecutor = Executors.newFixedThreadPool(1);
    executor = MoreExecutors.listeningDecorator(rawExecutor);
    ApplicationId appId = ApplicationId.newInstance(10000, 1);
    ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(appId, 1);
    ContainerId containerId = ContainerId.newInstance(appAttemptId, 1);

    TezTaskUmbilicalForTest umbilical = new TezTaskUmbilicalForTest();
    ContainerContext containerContext = new ContainerContext(containerId.toString());

    ContainerReporter containerReporter = new ContainerReporter(umbilical, containerContext, 100);
    ListenableFuture<ContainerTask> getTaskFuture = executor.submit(containerReporter);

    getTaskFuture.get();
    assertEquals(1, umbilical.getTaskInvocations);

  } finally {
    executor.shutdownNow();
  }
}
 

public String getResourcesOption(ContainerId containerId) {
  String containerName = containerId.toString();

  StringBuilder sb = new StringBuilder("cgroups=");

  if (isCpuWeightEnabled()) {
    sb.append(pathForCgroup(CONTROLLER_CPU, containerName) + "/tasks");
    sb.append(",");
  }

  if (sb.charAt(sb.length() - 1) == ',') {
    sb.deleteCharAt(sb.length() - 1);
  }

  return sb.toString();
}
 

/**
 * Reacquires state for a container - reads the classid from the cgroup
 * being used for the container being reacquired
 * @param containerId if of the container being reacquired.
 * @return (potentially empty) list of privileged operations
 * @throws ResourceHandlerException
 */

@Override
public List<PrivilegedOperation> reacquireContainer(ContainerId containerId)
    throws ResourceHandlerException {
  String containerIdStr = containerId.toString();

  if (LOG.isDebugEnabled()) {
    LOG.debug("Attempting to reacquire classId for container: " +
        containerIdStr);
  }

  String classIdStrFromFile = cGroupsHandler.getCGroupParam(
      CGroupsHandler.CGroupController.NET_CLS, containerIdStr,
      CGroupsHandler.CGROUP_PARAM_CLASSID);
  int classId = trafficController
      .getClassIdFromFileContents(classIdStrFromFile);

  LOG.info("Reacquired containerId -> classId mapping: " + containerIdStr
      + " -> " + classId);
  containerIdClassIdMap.put(containerId, classId);

  return null;
}
 

@Override
public void storeContainer(ContainerId containerId,
    StartContainerRequest startRequest) throws IOException {
  if (LOG.isDebugEnabled()) {
    LOG.debug("storeContainer: containerId= " + containerId
        + ", startRequest= " + startRequest);
  }

  String key = CONTAINERS_KEY_PREFIX + containerId.toString()
      + CONTAINER_REQUEST_KEY_SUFFIX;
  try {
    db.put(bytes(key),
      ((StartContainerRequestPBImpl) startRequest).getProto().toByteArray());
  } catch (DBException e) {
    throw new IOException(e);
  }
}
 

private void setupLimits(ContainerId containerId,
                         Resource containerResource) throws IOException {
  String containerName = containerId.toString();

  if (isCpuWeightEnabled()) {
    int containerVCores = containerResource.getVirtualCores();
    createCgroup(CONTROLLER_CPU, containerName);
    int cpuShares = CPU_DEFAULT_WEIGHT * containerVCores;
    updateCgroup(CONTROLLER_CPU, containerName, "shares",
        String.valueOf(cpuShares));
    if (strictResourceUsageMode) {
      int nodeVCores =
          conf.getInt(YarnConfiguration.NM_VCORES,
            YarnConfiguration.DEFAULT_NM_VCORES);
      if (nodeVCores != containerVCores) {
        float containerCPU =
            (containerVCores * yarnProcessors) / (float) nodeVCores;
        int[] limits = getOverallLimits(containerCPU);
        updateCgroup(CONTROLLER_CPU, containerName, CPU_PERIOD_US,
          String.valueOf(limits[0]));
        updateCgroup(CONTROLLER_CPU, containerName, CPU_QUOTA_US,
          String.valueOf(limits[1]));
      }
    }
  }
}
 

@SuppressWarnings("unchecked")
private void stopContainerInternal(NMTokenIdentifier nmTokenIdentifier,
    ContainerId containerID) throws YarnException, IOException {
  String containerIDStr = containerID.toString();
  Container container = this.context.getContainers().get(containerID);
  LOG.info("Stopping container with container Id: " + containerIDStr);
  authorizeGetAndStopContainerRequest(containerID, container, true,
    nmTokenIdentifier);

  if (container == null) {
    if (!nodeStatusUpdater.isContainerRecentlyStopped(containerID)) {
      throw RPCUtil.getRemoteException("Container " + containerIDStr
        + " is not handled by this NodeManager");
    }
  } else {
    context.getNMStateStore().storeContainerKilled(containerID);
    dispatcher.getEventHandler().handle(
      new ContainerKillEvent(containerID,
          ContainerExitStatus.KILLED_BY_APPMASTER,
          "Container killed by the ApplicationMaster."));

    NMAuditLogger.logSuccess(container.getUser(),    
      AuditConstants.STOP_CONTAINER, "ContainerManageImpl", containerID
        .getApplicationAttemptId().getApplicationId(), containerID);

    // TODO: Move this code to appropriate place once kill_container is
    // implemented.
    nodeStatusUpdater.sendOutofBandHeartBeat();
  }
}
 

@Override
public void storeContainerLaunched(ContainerId containerId)
    throws IOException {
  String key = CONTAINERS_KEY_PREFIX + containerId.toString()
      + CONTAINER_LAUNCHED_KEY_SUFFIX;
  try {
    db.put(bytes(key), EMPTY_VALUE);
  } catch (DBException e) {
    throw new IOException(e);
  }
}
 

@Override
public void storeContainerLaunched(ContainerId containerId)
    throws IOException {
  if (LOG.isDebugEnabled()) {
    LOG.debug("storeContainerLaunched: containerId=" + containerId);
  }

  String key = CONTAINERS_KEY_PREFIX + containerId.toString()
      + CONTAINER_LAUNCHED_KEY_SUFFIX;
  try {
    db.put(bytes(key), EMPTY_VALUE);
  } catch (DBException e) {
    throw new IOException(e);
  }
}
 

@Override
public boolean ResumeContainer(ContainerId containerId) {
	// we may do more check here, do it later
	String id = containerId.toString();
	DockerCommand command = new DockerCommand(id,null,DockerCommand.RESUME);
	return ExecuteCommand(command);
}
 

/**
 * It will validate to make sure all the containers belong to correct
 * application attempt id. If not then it will throw
 * {@link InvalidContainerReleaseException}
 * 
 * @param containerReleaseList
 *          containers to be released as requested by application master.
 * @param appAttemptId
 *          Application attempt Id
 * @throws InvalidContainerReleaseException
 */
public static void
    validateContainerReleaseRequest(List<ContainerId> containerReleaseList,
        ApplicationAttemptId appAttemptId)
        throws InvalidContainerReleaseException {
  for (ContainerId cId : containerReleaseList) {
    if (!appAttemptId.equals(cId.getApplicationAttemptId())) {
      throw new InvalidContainerReleaseException(
          "Cannot release container : "
              + cId.toString()
              + " not belonging to this application attempt : "
              + appAttemptId);
    }
  }
}
 

@Override
public StartContainersResponse
    startContainers(StartContainersRequest requests)
        throws YarnException {
  StartContainerRequest request = requests.getStartContainerRequests().get(0);
  LOG.info("Container started by MyContainerManager: " + request);
  launched = true;
  Map<String, String> env =
      request.getContainerLaunchContext().getEnvironment();

  Token containerToken = request.getContainerToken();
  ContainerTokenIdentifier tokenId = null;

  try {
    tokenId = BuilderUtils.newContainerTokenIdentifier(containerToken);
  } catch (IOException e) {
    throw RPCUtil.getRemoteException(e);
  }

  ContainerId containerId = tokenId.getContainerID();
  containerIdAtContainerManager = containerId.toString();
  attemptIdAtContainerManager =
      containerId.getApplicationAttemptId().toString();
  nmHostAtContainerManager = tokenId.getNmHostAddress();
  submitTimeAtContainerManager =
      Long.parseLong(env.get(ApplicationConstants.APP_SUBMIT_TIME_ENV));
  maxAppAttempts =
      Integer.parseInt(env.get(ApplicationConstants.MAX_APP_ATTEMPTS_ENV));
  return StartContainersResponse.newInstance(
    new HashMap<String, ByteBuffer>(), new ArrayList<ContainerId>(),
    new HashMap<ContainerId, SerializedException>());
}
 

@Override
public void storeContainerKilled(ContainerId containerId)
    throws IOException {
  String key = CONTAINERS_KEY_PREFIX + containerId.toString()
      + CONTAINER_KILLED_KEY_SUFFIX;
  try {
    db.put(bytes(key), EMPTY_VALUE);
  } catch (DBException e) {
    throw new IOException(e);
  }
}
 

@Test
public void testForCorruptedAggregatedLogs() throws Exception {
  Configuration conf = new Configuration();
  File workDir = new File(testWorkDir, "testReadAcontainerLogs1");
  Path remoteAppLogFile =
      new Path(workDir.getAbsolutePath(), "aggregatedLogFile");
  Path srcFileRoot = new Path(workDir.getAbsolutePath(), "srcFiles");
  ContainerId testContainerId = TestContainerId.newContainerId(1, 1, 1, 1);
  Path t =
      new Path(srcFileRoot, testContainerId.getApplicationAttemptId()
          .getApplicationId().toString());
  Path srcFilePath = new Path(t, testContainerId.toString());

  long numChars = 950000;

  writeSrcFileAndALog(srcFilePath, "stdout", numChars, remoteAppLogFile,
     srcFileRoot, testContainerId);

  LogReader logReader = new LogReader(conf, remoteAppLogFile);
  LogKey rLogKey = new LogKey();
  DataInputStream dis = logReader.next(rLogKey);
  Writer writer = new StringWriter();
  try {
    LogReader.readAcontainerLogs(dis, writer);
  } catch (Exception e) {
    if(e.toString().contains("NumberFormatException")) {
      Assert.fail("Aggregated logs are corrupted.");
    }
  }
}
 

public static String getContainerPath(ContainerId containerId){
    String applicationId = containerId.getApplicationAttemptId().getApplicationId().toString();

    String containerIdStr = containerId.toString();
    String containerPath = RegistryUtils.componentPath(
            JstormAMConstant.REGISTYR_APP_TYPE, JstormAMContext.clusterName,applicationId,containerIdStr);
    return containerPath;
}
 

static String sourceName(ContainerId containerId) {
  return RECORD_INFO.name() + "_" + containerId.toString();
}
 

private static String getTaskRunnerId(ContainerId containerId) {
  return containerId.toString();
}
 

public ContainerData(ContainerId cId, ContainerLaunchContext context) {
  this.cId = cId;
  this.cIdStr = cId.toString();
  this.launchContext = context;
}
 

@SuppressWarnings("unchecked")
@Test(timeout = 5000)
public void testTaskAttemptFailureViaHeartbeat() throws IOException, TezException {

  TaskCommunicatorManagerWrapperForTest wrapper = new TaskCommunicatorManagerWrapperForTest();

  TaskSpec taskSpec1 = wrapper.createTaskSpec();
  AMContainerTask amContainerTask1 = new AMContainerTask(taskSpec1, null, null, false, 10);

  TaskSpec taskSpec2 = wrapper.createTaskSpec();
  AMContainerTask amContainerTask2 = new AMContainerTask(taskSpec2, null, null, false, 10);

  ContainerId containerId1 = wrapper.createContainerId(1);
  wrapper.registerRunningContainer(containerId1);
  wrapper.registerTaskAttempt(containerId1, amContainerTask1);

  ContainerId containerId2 = wrapper.createContainerId(2);
  wrapper.registerRunningContainer(containerId2);
  wrapper.registerTaskAttempt(containerId2, amContainerTask2);

  List<TezEvent> events = new LinkedList<>();

  EventMetaData sourceInfo1 =
      new EventMetaData(EventMetaData.EventProducerConsumerType.PROCESSOR, "testVertex", null,
          taskSpec1.getTaskAttemptID());
  TaskAttemptFailedEvent failedEvent1 = new TaskAttemptFailedEvent("non-fatal test error",
      TaskFailureType.NON_FATAL);
  TezEvent failedEventT1 = new TezEvent(failedEvent1, sourceInfo1);
  events.add(failedEventT1);
  TaskHeartbeatRequest taskHeartbeatRequest1 =
      new TaskHeartbeatRequest(containerId1.toString(), taskSpec1.getTaskAttemptID(), events, 0,
          0, 0);
  wrapper.getTaskCommunicatorManager().heartbeat(taskHeartbeatRequest1);

  ArgumentCaptor<Event> argumentCaptor = ArgumentCaptor.forClass(Event.class);
  verify(wrapper.getEventHandler(), times(1)).handle(argumentCaptor.capture());
  assertTrue(argumentCaptor.getAllValues().get(0) instanceof TaskAttemptEventAttemptFailed);
  TaskAttemptEventAttemptFailed failedEvent =
      (TaskAttemptEventAttemptFailed) argumentCaptor.getAllValues().get(0);
  assertEquals(TaskFailureType.NON_FATAL, failedEvent.getTaskFailureType());
  assertTrue(failedEvent.getDiagnosticInfo().contains("non-fatal"));

  events.clear();
  reset(wrapper.getEventHandler());

  EventMetaData sourceInfo2 =
      new EventMetaData(EventMetaData.EventProducerConsumerType.PROCESSOR, "testVertex", null,
          taskSpec2.getTaskAttemptID());
  TaskAttemptFailedEvent failedEvent2 = new TaskAttemptFailedEvent("-fatal- test error",
      TaskFailureType.FATAL);
  TezEvent failedEventT2 = new TezEvent(failedEvent2, sourceInfo2);
  events.add(failedEventT2);
  TaskHeartbeatRequest taskHeartbeatRequest2 =
      new TaskHeartbeatRequest(containerId2.toString(), taskSpec2.getTaskAttemptID(), events, 0,
          0, 0);
  wrapper.getTaskCommunicatorManager().heartbeat(taskHeartbeatRequest2);

  argumentCaptor = ArgumentCaptor.forClass(Event.class);
  verify(wrapper.getEventHandler(), times(1)).handle(argumentCaptor.capture());
  assertTrue(argumentCaptor.getAllValues().get(0) instanceof TaskAttemptEventAttemptFailed);
  failedEvent = (TaskAttemptEventAttemptFailed) argumentCaptor.getAllValues().get(0);
  assertEquals(TaskFailureType.FATAL, failedEvent.getTaskFailureType());
  assertTrue(failedEvent.getDiagnosticInfo().contains("-fatal-"));
}
 

/**
 * Create an event to record the start of an MR AppMaster
 *
 * @param appAttemptId
 *          the application attempt id.
 * @param startTime
 *          the start time of the AM.
 * @param containerId
 *          the containerId of the AM.
 * @param nodeManagerHost
 *          the node on which the AM is running.
 * @param nodeManagerPort
 *          the port on which the AM is running.
 * @param nodeManagerHttpPort
 *          the httpPort for the node running the AM.
 * @param forcedJobStateOnShutDown
 *          the state to force the job into
 */
public AMStartedEvent(ApplicationAttemptId appAttemptId, long startTime,
    ContainerId containerId, String nodeManagerHost, int nodeManagerPort,
    int nodeManagerHttpPort, String forcedJobStateOnShutDown,
    long submitTime) {
  datum.applicationAttemptId = new Utf8(appAttemptId.toString());
  datum.startTime = startTime;
  datum.containerId = new Utf8(containerId.toString());
  datum.nodeManagerHost = new Utf8(nodeManagerHost);
  datum.nodeManagerPort = nodeManagerPort;
  datum.nodeManagerHttpPort = nodeManagerHttpPort;
  this.forcedJobStateOnShutDown = forcedJobStateOnShutDown;
  this.submitTime = submitTime;
}