Java Code Examples for org.apache.hadoop.yarn.api.records.ContainerId#fromString()

The following examples show how to use org.apache.hadoop.yarn.api.records.ContainerId#fromString() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: ApplicationMaster.java    From TensorFlowOnYARN with Apache License 2.0 5 votes vote down vote up 
private void setupPreviousRunningContainers(RegisterApplicationMasterResponse response) {
  String containerIdStr =
      System.getenv(Environment.CONTAINER_ID.name());
  ContainerId containerId = ContainerId.fromString(containerIdStr);
  appAttemptId = containerId.getApplicationAttemptId();
  List<Container> previousAMRunningContainers =
      response.getContainersFromPreviousAttempts();
  LOG.info(appAttemptId + " received " + previousAMRunningContainers.size()
      + " previous attempts' running containers on AM registration.");
  for (Container container : previousAMRunningContainers) {
    launchedContainers.add(container.getId());
  }
  allocatedContainerNum.addAndGet(previousAMRunningContainers.size());
}
Example 2
Source File: YarnContainerHeartbeatServlet.java    From samza with Apache License 2.0 5 votes vote down vote up 
@Override
protected void doGet(HttpServletRequest req, HttpServletResponse resp)
    throws ServletException, IOException {
  ContainerId yarnContainerId;
  PrintWriter printWriter = resp.getWriter();
  String containerIdParam = req.getParameter(YARN_CONTAINER_ID);
  ContainerHeartbeatResponse response;
  resp.setContentType(APPLICATION_JSON);
  boolean alive = false;
  try {
    yarnContainerId = ContainerId.fromString(containerIdParam);
    for (YarnContainer yarnContainer : yarnAppState.runningProcessors.values()) {
      if (yarnContainer.id().compareTo(yarnContainerId) == 0) {
        alive = true;
        break;
      }
    }
    if (!alive) {
      heartbeatsExpiredCount.inc();
    }
    response = new ContainerHeartbeatResponse(alive);
    printWriter.write(mapper.writeValueAsString(response));
  } catch (IllegalArgumentException e) {
    LOG.error("Container ID {} passed is invalid", containerIdParam);
    resp.sendError(HttpServletResponse.SC_BAD_REQUEST, e.getMessage());
  }
}
Example 3
Source File: ConverterUtils.java    From hadoop with Apache License 2.0 4 votes vote down vote up 
public static ContainerId toContainerId(String containerIdStr) {
  return ContainerId.fromString(containerIdStr);
}
Example 4
Source File: TestCgroupsLCEResourcesHandler.java    From hadoop with Apache License 2.0 4 votes vote down vote up 
@Test
public void testContainerLimits() throws IOException {
  LinuxContainerExecutor mockLCE = new MockLinuxContainerExecutor();
  CustomCgroupsLCEResourceHandler handler =
      new CustomCgroupsLCEResourceHandler();
  handler.generateLimitsMode = true;
  YarnConfiguration conf = new YarnConfiguration();
  final int numProcessors = 4;
  ResourceCalculatorPlugin plugin =
      Mockito.mock(ResourceCalculatorPlugin.class);
  Mockito.doReturn(numProcessors).when(plugin).getNumProcessors();
  handler.setConf(conf);
  handler.initConfig();

  // create mock cgroup
  File cgroupMountDir = createMockCgroupMount(cgroupDir);

  // create mock mtab
  File mockMtab = createMockMTab(cgroupDir);

  // setup our handler and call init()
  handler.setMtabFile(mockMtab.getAbsolutePath());
  handler.init(mockLCE, plugin);

  // check values
  // default case - files shouldn't exist, strict mode off by default
  ContainerId id = ContainerId.fromString("container_1_1_1_1");
  handler.preExecute(id, Resource.newInstance(1024, 1));
  File containerDir = new File(cgroupMountDir, id.toString());
  Assert.assertTrue(containerDir.exists());
  Assert.assertTrue(containerDir.isDirectory());
  File periodFile = new File(containerDir, "cpu.cfs_period_us");
  File quotaFile = new File(containerDir, "cpu.cfs_quota_us");
  Assert.assertFalse(periodFile.exists());
  Assert.assertFalse(quotaFile.exists());

  // no files created because we're using all cpu
  FileUtils.deleteQuietly(containerDir);
  conf.setBoolean(
    YarnConfiguration.NM_LINUX_CONTAINER_CGROUPS_STRICT_RESOURCE_USAGE, true);
  handler.initConfig();
  handler.preExecute(id,
    Resource.newInstance(1024, YarnConfiguration.DEFAULT_NM_VCORES));
  Assert.assertTrue(containerDir.exists());
  Assert.assertTrue(containerDir.isDirectory());
  periodFile = new File(containerDir, "cpu.cfs_period_us");
  quotaFile = new File(containerDir, "cpu.cfs_quota_us");
  Assert.assertFalse(periodFile.exists());
  Assert.assertFalse(quotaFile.exists());

  // 50% of CPU
  FileUtils.deleteQuietly(containerDir);
  conf.setBoolean(
    YarnConfiguration.NM_LINUX_CONTAINER_CGROUPS_STRICT_RESOURCE_USAGE, true);
  handler.initConfig();
  handler.preExecute(id,
    Resource.newInstance(1024, YarnConfiguration.DEFAULT_NM_VCORES / 2));
  Assert.assertTrue(containerDir.exists());
  Assert.assertTrue(containerDir.isDirectory());
  periodFile = new File(containerDir, "cpu.cfs_period_us");
  quotaFile = new File(containerDir, "cpu.cfs_quota_us");
  Assert.assertTrue(periodFile.exists());
  Assert.assertTrue(quotaFile.exists());
  Assert.assertEquals(500 * 1000, readIntFromFile(periodFile));
  Assert.assertEquals(1000 * 1000, readIntFromFile(quotaFile));

  // CGroups set to 50% of CPU, container set to 50% of YARN CPU
  FileUtils.deleteQuietly(containerDir);
  conf.setBoolean(
    YarnConfiguration.NM_LINUX_CONTAINER_CGROUPS_STRICT_RESOURCE_USAGE, true);
  conf
    .setInt(YarnConfiguration.NM_RESOURCE_PERCENTAGE_PHYSICAL_CPU_LIMIT, 50);
  handler.initConfig();
  handler.init(mockLCE, plugin);
  handler.preExecute(id,
    Resource.newInstance(1024, YarnConfiguration.DEFAULT_NM_VCORES / 2));
  Assert.assertTrue(containerDir.exists());
  Assert.assertTrue(containerDir.isDirectory());
  periodFile = new File(containerDir, "cpu.cfs_period_us");
  quotaFile = new File(containerDir, "cpu.cfs_quota_us");
  Assert.assertTrue(periodFile.exists());
  Assert.assertTrue(quotaFile.exists());
  Assert.assertEquals(1000 * 1000, readIntFromFile(periodFile));
  Assert.assertEquals(1000 * 1000, readIntFromFile(quotaFile));

  FileUtils.deleteQuietly(cgroupDir);
}
Example 5
Source File: Hadoop26YarnAMClient.java    From twill with Apache License 2.0 4 votes vote down vote up 
@Override
protected final ContainerId containerIdLookup(String containerIdStr) {
  return (ContainerId.fromString(containerIdStr));
}
Example 6
Source File: ConverterUtils.java    From big-c with Apache License 2.0 4 votes vote down vote up 
public static ContainerId toContainerId(String containerIdStr) {
  return ContainerId.fromString(containerIdStr);
}
Example 7
Source File: TestCgroupsLCEResourcesHandler.java    From big-c with Apache License 2.0 4 votes vote down vote up 
@Test
public void testContainerLimits() throws IOException {
  LinuxContainerExecutor mockLCE = new MockLinuxContainerExecutor();
  CustomCgroupsLCEResourceHandler handler =
      new CustomCgroupsLCEResourceHandler();
  handler.generateLimitsMode = true;
  YarnConfiguration conf = new YarnConfiguration();
  final int numProcessors = 4;
  ResourceCalculatorPlugin plugin =
      Mockito.mock(ResourceCalculatorPlugin.class);
  Mockito.doReturn(numProcessors).when(plugin).getNumProcessors();
  handler.setConf(conf);
  handler.initConfig();

  // create mock cgroup
  File cgroupMountDir = createMockCgroupMount(cgroupDir);

  // create mock mtab
  File mockMtab = createMockMTab(cgroupDir);

  // setup our handler and call init()
  handler.setMtabFile(mockMtab.getAbsolutePath());
  handler.init(mockLCE, plugin);

  // check values
  // default case - files shouldn't exist, strict mode off by default
  ContainerId id = ContainerId.fromString("container_1_1_1_1");
  handler.preExecute(id, Resource.newInstance(1024, 1));
  File containerDir = new File(cgroupMountDir, id.toString());
  Assert.assertTrue(containerDir.exists());
  Assert.assertTrue(containerDir.isDirectory());
  File periodFile = new File(containerDir, "cpu.cfs_period_us");
  File quotaFile = new File(containerDir, "cpu.cfs_quota_us");
  Assert.assertFalse(periodFile.exists());
  Assert.assertFalse(quotaFile.exists());

  // no files created because we're using all cpu
  FileUtils.deleteQuietly(containerDir);
  conf.setBoolean(
    YarnConfiguration.NM_LINUX_CONTAINER_CGROUPS_STRICT_RESOURCE_USAGE, true);
  handler.initConfig();
  handler.preExecute(id,
    Resource.newInstance(1024, YarnConfiguration.DEFAULT_NM_VCORES));
  Assert.assertTrue(containerDir.exists());
  Assert.assertTrue(containerDir.isDirectory());
  periodFile = new File(containerDir, "cpu.cfs_period_us");
  quotaFile = new File(containerDir, "cpu.cfs_quota_us");
  Assert.assertFalse(periodFile.exists());
  Assert.assertFalse(quotaFile.exists());

  // 50% of CPU
  FileUtils.deleteQuietly(containerDir);
  conf.setBoolean(
    YarnConfiguration.NM_LINUX_CONTAINER_CGROUPS_STRICT_RESOURCE_USAGE, true);
  handler.initConfig();
  handler.preExecute(id,
    Resource.newInstance(1024, YarnConfiguration.DEFAULT_NM_VCORES / 2));
  Assert.assertTrue(containerDir.exists());
  Assert.assertTrue(containerDir.isDirectory());
  periodFile = new File(containerDir, "cpu.cfs_period_us");
  quotaFile = new File(containerDir, "cpu.cfs_quota_us");
  Assert.assertTrue(periodFile.exists());
  Assert.assertTrue(quotaFile.exists());
  Assert.assertEquals(500 * 1000, readIntFromFile(periodFile));
  Assert.assertEquals(1000 * 1000, readIntFromFile(quotaFile));

  // CGroups set to 50% of CPU, container set to 50% of YARN CPU
  FileUtils.deleteQuietly(containerDir);
  conf.setBoolean(
    YarnConfiguration.NM_LINUX_CONTAINER_CGROUPS_STRICT_RESOURCE_USAGE, true);
  conf
    .setInt(YarnConfiguration.NM_RESOURCE_PERCENTAGE_PHYSICAL_CPU_LIMIT, 50);
  handler.initConfig();
  handler.init(mockLCE, plugin);
  handler.preExecute(id,
    Resource.newInstance(1024, YarnConfiguration.DEFAULT_NM_VCORES / 2));
  Assert.assertTrue(containerDir.exists());
  Assert.assertTrue(containerDir.isDirectory());
  periodFile = new File(containerDir, "cpu.cfs_period_us");
  quotaFile = new File(containerDir, "cpu.cfs_quota_us");
  Assert.assertTrue(periodFile.exists());
  Assert.assertTrue(quotaFile.exists());
  Assert.assertEquals(1000 * 1000, readIntFromFile(periodFile));
  Assert.assertEquals(1000 * 1000, readIntFromFile(quotaFile));

  FileUtils.deleteQuietly(cgroupDir);
}
Example 8
Source File: TimelineCachePluginImpl.java    From tez with Apache License 2.0 4 votes vote down vote up 
private Set<TimelineEntityGroupId> convertToTimelineEntityGroupIds(String entityType, String entityId) {
  if (entityType == null || entityType.isEmpty()
      || entityId == null || entityId.isEmpty()) {
    return null;
  }
  if (entityType.equals(EntityTypes.TEZ_DAG_ID.name()) ||
      entityType.equals(EntityTypes.TEZ_DAG_EXTRA_INFO.name())) {
    TezDAGID dagId = TezDAGID.fromString(entityId);
    if (dagId != null) {
      return createTimelineEntityGroupIds(dagId);
    }
  } else if (entityType.equals(EntityTypes.TEZ_VERTEX_ID.name())) {
    TezVertexID vertexID = TezVertexID.fromString(entityId);
    if (vertexID != null) {
      return createTimelineEntityGroupIds(vertexID.getDAGId());
    }

  } else if (entityType.equals(EntityTypes.TEZ_TASK_ID.name())) {
    TezTaskID taskID = TezTaskID.fromString(entityId);
    if (taskID != null) {
      return createTimelineEntityGroupIds(taskID.getVertexID().getDAGId());
    }
  } else if (entityType.equals(EntityTypes.TEZ_TASK_ATTEMPT_ID.name())) {
    TezTaskAttemptID taskAttemptID = TezTaskAttemptID.fromString(entityId);
    if (taskAttemptID != null) {
      return createTimelineEntityGroupIds(taskAttemptID.getTaskID().getVertexID().getDAGId());
    }
  } else if (entityType.equals(EntityTypes.TEZ_CONTAINER_ID.name())) {
    String cId = entityId;
    if (cId.startsWith("tez_")) {
      cId = cId.substring(4);
    }
    ContainerId containerId = ContainerId.fromString(cId);
    if (containerId != null) {
      return Sets.newHashSet(TimelineEntityGroupId.newInstance(
          containerId.getApplicationAttemptId().getApplicationId(),
          containerId.getApplicationAttemptId().getApplicationId().toString()));
    }
  }
  return null;
}