org.apache.hadoop.yarn.api.records.ContainerReport Java Examples

public int validateContainerCount(ApplicationAttemptId attemptId) throws Exception {
  int runningContainerCount = 0;
  for (ContainerReport containerReport : this.client.getContainers(attemptId)) {
    if (containerReport.getContainerState() == ContainerState.RUNNING) {
      ++runningContainerCount;
    }
  }
  // expected containers to be the configured job containers plus the AppMaster container
  int containerExpected = this.config.getContainerCount() + 1;

  if (runningContainerCount == containerExpected) {
    log.info("Container count matches. " + runningContainerCount + " containers are running.");
    return runningContainerCount;
  } else {
    throw new SamzaException("Container count does not match. " + runningContainerCount + " containers are running, while " + containerExpected + " is expected.");
  }
}
 

@Test(timeout = 10000)
public void testGetContainers() throws YarnException, IOException {
  Configuration conf = new Configuration();
  final AHSClient client = new MockAHSClient();
  client.init(conf);
  client.start();

  ApplicationId applicationId = ApplicationId.newInstance(1234, 5);
  ApplicationAttemptId appAttemptId =
      ApplicationAttemptId.newInstance(applicationId, 1);
  List<ContainerReport> reports = client.getContainers(appAttemptId);
  Assert.assertNotNull(reports);
  Assert.assertEquals(reports.get(0).getContainerId(),
    (ContainerId.newContainerId(appAttemptId, 1)));
  Assert.assertEquals(reports.get(1).getContainerId(),
    (ContainerId.newContainerId(appAttemptId, 2)));
  client.stop();
}
 

@Override
public ContainerReport getContainerReport(ContainerId containerId)
    throws YarnException, IOException {
  try {
    GetContainerReportRequest request = Records
        .newRecord(GetContainerReportRequest.class);
    request.setContainerId(containerId);
    GetContainerReportResponse response = rmClient
        .getContainerReport(request);
    return response.getContainerReport();
  } catch (YarnException e) {
    if (!historyServiceEnabled) {
      // Just throw it as usual if historyService is not enabled.
      throw e;
    }
    // Even if history-service is enabled, treat all exceptions still the same
    // except the following
    if (e.getClass() != ApplicationNotFoundException.class
        && e.getClass() != ContainerNotFoundException.class) {
      throw e;
    }
    return historyClient.getContainerReport(containerId);
  }
}
 

/**
 * Lists the containers matching the given application attempts
 * 
 * @param appAttemptId
 * @throws YarnException
 * @throws IOException
 */
private void listContainers(String appAttemptId) throws YarnException,
    IOException {
  PrintWriter writer = new PrintWriter(
      new OutputStreamWriter(sysout, Charset.forName("UTF-8")));

  List<ContainerReport> appsReport = client
      .getContainers(ConverterUtils.toApplicationAttemptId(appAttemptId));
  writer.println("Total number of containers " + ":" + appsReport.size());
  writer.printf(CONTAINER_PATTERN, "Container-Id", "Start Time",
      "Finish Time", "State", "Host", "Node Http Address", "LOG-URL");
  for (ContainerReport containerReport : appsReport) {
    writer.printf(
        CONTAINER_PATTERN,
        containerReport.getContainerId(),
        Times.format(containerReport.getCreationTime()),
        Times.format(containerReport.getFinishTime()),      
        containerReport.getContainerState(), containerReport
            .getAssignedNode(), containerReport.getNodeHttpAddress() == null
                ? "N/A" : containerReport.getNodeHttpAddress(),
        containerReport.getLogUrl());
  }
  writer.flush();
}
 

@Test
public void testContainerReport() throws IOException, YarnException {
  ApplicationId appId = ApplicationId.newInstance(0, 1);
  ApplicationAttemptId appAttemptId =
      ApplicationAttemptId.newInstance(appId, 1);
  ContainerId containerId = ContainerId.newContainerId(appAttemptId, 1);
  GetContainerReportRequest request =
      GetContainerReportRequest.newInstance(containerId);
  GetContainerReportResponse response =
      clientService.getContainerReport(request);
  ContainerReport container = response.getContainerReport();
  Assert.assertNotNull(container);
  Assert.assertEquals(containerId, container.getContainerId());
  Assert.assertEquals("http://0.0.0.0:8188/applicationhistory/logs/" +
      "test host:100/container_0_0001_01_000001/" +
      "container_0_0001_01_000001/user1", container.getLogUrl());
}
 

@Override
public ContainerReport getContainerReport(ContainerId containerId)
    throws YarnException, IOException {
  try {
    GetContainerReportRequest request = Records
        .newRecord(GetContainerReportRequest.class);
    request.setContainerId(containerId);
    GetContainerReportResponse response = rmClient
        .getContainerReport(request);
    return response.getContainerReport();
  } catch (YarnException e) {
    if (!historyServiceEnabled) {
      // Just throw it as usual if historyService is not enabled.
      throw e;
    }
    // Even if history-service is enabled, treat all exceptions still the same
    // except the following
    if (e.getClass() != ApplicationNotFoundException.class
        && e.getClass() != ContainerNotFoundException.class) {
      throw e;
    }
    return historyClient.getContainerReport(containerId);
  }
}
 

@Test(timeout = 10000)
public void testGetContainerReport() throws YarnException, IOException {
  Configuration conf = new Configuration();
  final AHSClient client = new MockAHSClient();
  client.init(conf);
  client.start();

  List<ApplicationReport> expectedReports =
      ((MockAHSClient) client).getReports();

  ApplicationId applicationId = ApplicationId.newInstance(1234, 5);
  ApplicationAttemptId appAttemptId =
      ApplicationAttemptId.newInstance(applicationId, 1);
  ContainerId containerId = ContainerId.newContainerId(appAttemptId, 1);
  ContainerReport report = client.getContainerReport(containerId);
  Assert.assertNotNull(report);
  Assert.assertEquals(report.getContainerId().toString(), (ContainerId
    .newContainerId(expectedReports.get(0).getCurrentApplicationAttemptId(), 1))
    .toString());
  client.stop();
}
 

public ContainerInfo(ContainerReport container) {
  containerId = container.getContainerId().toString();
  if (container.getAllocatedResource() != null) {
    allocatedMB = container.getAllocatedResource().getMemory();
    allocatedVCores = container.getAllocatedResource().getVirtualCores();
    allocatedGCores = container.getAllocatedResource().getGpuCores();
  }
  if (container.getAssignedNode() != null) {
    assignedNodeId = container.getAssignedNode().toString();
  }
  priority = container.getPriority().getPriority();
  startedTime = container.getCreationTime();
  finishedTime = container.getFinishTime();
  elapsedTime = Times.elapsed(startedTime, finishedTime);
  diagnosticsInfo = container.getDiagnosticsInfo();
  logUrl = container.getLogUrl();
  containerExitStatus = container.getContainerExitStatus();
  containerState = container.getContainerState();
  nodeHttpAddress = container.getNodeHttpAddress();
}
 

@Test
public void testContainers() throws IOException, YarnException {
  ApplicationId appId = ApplicationId.newInstance(0, 1);
  ApplicationAttemptId appAttemptId =
      ApplicationAttemptId.newInstance(appId, 1);
  ContainerId containerId = ContainerId.newContainerId(appAttemptId, 1);
  ContainerId containerId1 = ContainerId.newContainerId(appAttemptId, 2);
  GetContainersRequest request =
      GetContainersRequest.newInstance(appAttemptId);
  GetContainersResponse response =
      clientService.getContainers(request);
  List<ContainerReport> containers = response.getContainerList();
  Assert.assertNotNull(containers);
  Assert.assertEquals(containerId, containers.get(0).getContainerId());
  Assert.assertEquals(containerId1, containers.get(1).getContainerId());
}
 

@Override
public Map<ContainerId, ContainerReport> getContainers(
    ApplicationAttemptId appAttemptId) throws YarnException, IOException {
  ApplicationReportExt app = getApplication(
      appAttemptId.getApplicationId(), ApplicationReportField.USER_AND_ACLS);
  checkAccess(app);
  TimelineEntities entities = timelineDataManager.getEntities(
      ContainerMetricsConstants.ENTITY_TYPE,
      new NameValuePair(
          ContainerMetricsConstants.PARENT_PRIMARIY_FILTER,
          appAttemptId.toString()), null, null, null,
      null, null, Long.MAX_VALUE, EnumSet.allOf(Field.class),
      UserGroupInformation.getLoginUser());
  Map<ContainerId, ContainerReport> containers =
      new LinkedHashMap<ContainerId, ContainerReport>();
  if (entities != null && entities.getEntities() != null) {
    for (TimelineEntity entity : entities.getEntities()) {
      ContainerReport container = convertToContainerReport(
          entity, serverHttpAddress, app.appReport.getUser());
      containers.put(container.getContainerId(), container);
    }
  }
  return containers;
}
 

/**
 * Lists the containers matching the given application attempts
 * 
 * @param appAttemptId
 * @throws YarnException
 * @throws IOException
 */
private void listContainers(String appAttemptId) throws YarnException,
    IOException {
  PrintWriter writer = new PrintWriter(
      new OutputStreamWriter(sysout, Charset.forName("UTF-8")));

  List<ContainerReport> appsReport = client
      .getContainers(ConverterUtils.toApplicationAttemptId(appAttemptId));
  writer.println("Total number of containers " + ":" + appsReport.size());
  writer.printf(CONTAINER_PATTERN, "Container-Id", "Start Time",
      "Finish Time", "State", "Host", "Node Http Address", "LOG-URL");
  for (ContainerReport containerReport : appsReport) {
    writer.printf(
        CONTAINER_PATTERN,
        containerReport.getContainerId(),
        Times.format(containerReport.getCreationTime()),
        Times.format(containerReport.getFinishTime()),      
        containerReport.getContainerState(), containerReport
            .getAssignedNode(), containerReport.getNodeHttpAddress() == null
                ? "N/A" : containerReport.getNodeHttpAddress(),
        containerReport.getLogUrl());
  }
  writer.flush();
}
 

private ContainerReport getContainer(
    ContainerId containerId,
    HashMap<ApplicationAttemptId, List<ContainerReport>> containersToAppAttemptMapping)
    throws YarnException, IOException {
  List<ContainerReport> containersForAppAttempt =
      containersToAppAttemptMapping.get(containerId
          .getApplicationAttemptId());
  if (containersForAppAttempt == null) {
    throw new ApplicationNotFoundException(containerId
        .getApplicationAttemptId().getApplicationId() + " is not found ");
  }
  Iterator<ContainerReport> iterator = containersForAppAttempt.iterator();
  while (iterator.hasNext()) {
    ContainerReport next = iterator.next();
    if (next.getContainerId().equals(containerId)) {
      return next;
    }
  }
  throw new ContainerNotFoundException(containerId + " is not found ");
}
 

public ContainerInfo(ContainerReport container) {
  containerId = container.getContainerId().toString();
  if (container.getAllocatedResource() != null) {
    allocatedMB = container.getAllocatedResource().getMemory();
    allocatedVCores = container.getAllocatedResource().getVirtualCores();
  }
  if (container.getAssignedNode() != null) {
    assignedNodeId = container.getAssignedNode().toString();
  }
  priority = container.getPriority().getPriority();
  startedTime = container.getCreationTime();
  finishedTime = container.getFinishTime();
  elapsedTime = Times.elapsed(startedTime, finishedTime);
  diagnosticsInfo = container.getDiagnosticsInfo();
  logUrl = container.getLogUrl();
  containerExitStatus = container.getContainerExitStatus();
  containerState = container.getContainerState();
  nodeHttpAddress = container.getNodeHttpAddress();
}
 

@Override
public Map<ContainerId, ContainerReport> getContainers(
    ApplicationAttemptId appAttemptId) throws YarnException, IOException {
  ApplicationReportExt app = getApplication(
      appAttemptId.getApplicationId(), ApplicationReportField.USER_AND_ACLS);
  checkAccess(app);
  TimelineEntities entities = timelineDataManager.getEntities(
      ContainerMetricsConstants.ENTITY_TYPE,
      new NameValuePair(
          ContainerMetricsConstants.PARENT_PRIMARIY_FILTER,
          appAttemptId.toString()), null, null, null,
      null, null, Long.MAX_VALUE, EnumSet.allOf(Field.class),
      UserGroupInformation.getLoginUser());
  Map<ContainerId, ContainerReport> containers =
      new LinkedHashMap<ContainerId, ContainerReport>();
  if (entities != null && entities.getEntities() != null) {
    for (TimelineEntity entity : entities.getEntities()) {
      ContainerReport container = convertToContainerReport(
          entity, serverHttpAddress, app.appReport.getUser());
      containers.put(container.getContainerId(), container);
    }
  }
  return containers;
}
 

private boolean hasAMContainer(ContainerId containerId,
    Collection<ContainerReport> containers) {
  for (ContainerReport container : containers) {
    if (containerId.equals(container.getContainerId())) {
      return true;
    }
  }
  return false;
}
 

@Override
public List<ContainerReport>
    getContainers(ApplicationAttemptId appAttemptId) throws YarnException,
        IOException {
  when(mockContainersResponse.getContainerList()).thenReturn(
    getContainersReport(appAttemptId));
  when(historyClient.getContainers(any(ApplicationAttemptId.class)))
  .thenReturn(getContainersFromAHS(appAttemptId));
  return super.getContainers(appAttemptId);
}
 

@Override
public ContainerReport getContainerReport() {
  if (this.containerReport != null) {
    return this.containerReport;
  }
  GetContainerReportResponseProtoOrBuilder p = viaProto ? proto : builder;
  if (!p.hasContainerReport()) {
    return null;
  }
  this.containerReport = convertFromProtoFormat(p.getContainerReport());
  return this.containerReport;
}
 

private void addLocalContainersToProto() {
  maybeInitBuilder();
  builder.clearContainers();
  if (containerList == null) {
    return;
  }
  Iterable<ContainerReportProto> iterable =
      new Iterable<ContainerReportProto>() {
        @Override
        public Iterator<ContainerReportProto> iterator() {
          return new Iterator<ContainerReportProto>() {

            Iterator<ContainerReport> iter = containerList.iterator();

            @Override
            public boolean hasNext() {
              return iter.hasNext();
            }

            @Override
            public ContainerReportProto next() {
              return convertToProtoFormat(iter.next());
            }

            @Override
            public void remove() {
              throw new UnsupportedOperationException();

            }
          };

        }
      };
  builder.addAllContainers(iterable);
}
 

@Test
public void testGetContainers() throws Exception {
  final ApplicationAttemptId appAttemptId =
      ApplicationAttemptId.newInstance(ApplicationId.newInstance(0, 1), 1);
  Collection<ContainerReport> containers;
  if (callerUGI == null) {
    containers = historyManager.getContainers(appAttemptId).values();
  } else {
    try {
      containers = callerUGI.doAs(
          new PrivilegedExceptionAction<Collection<ContainerReport>> () {
        @Override
        public Collection<ContainerReport> run() throws Exception {
          return historyManager.getContainers(appAttemptId).values();
        }
      });
      if (callerUGI != null && callerUGI.getShortUserName().equals("user3")) {
        // The exception is expected
        Assert.fail();
      }
    } catch (AuthorizationException e) {
      if (callerUGI != null && callerUGI.getShortUserName().equals("user3")) {
        // The exception is expected
        return;
      }
      throw e;
    }
  }
  Assert.assertNotNull(containers);
  Assert.assertEquals(SCALE, containers.size());
}
 

@Override
public ContainerReport getContainerReport(ContainerId containerId)
    throws YarnException, IOException {
  GetContainerReportRequest request = GetContainerReportRequest
      .newInstance(containerId);
  GetContainerReportResponse response = ahsClient.getContainerReport(request);
  return response.getContainerReport();
}
 

@Override
public List<ContainerReport> getContainers(
    ApplicationAttemptId applicationAttemptId) throws YarnException,
    IOException {
  GetContainersRequest request = GetContainersRequest
      .newInstance(applicationAttemptId);
  GetContainersResponse response = ahsClient.getContainers(request);
  return response.getContainerList();
}
 

@Test
public void testValidateContainerCount() throws Exception {
  List<ContainerReport> containerReports = new ArrayList<>();
  for (int i = 0; i <= containerCount; i++) {
    ContainerReport report = mock(ContainerReport.class);
    when(report.getContainerState()).thenReturn(ContainerState.RUNNING);
    containerReports.add(report);
  }
  when(client.getContainers(attemptId)).thenReturn(containerReports);
  assertTrue(tool.validateContainerCount(attemptId) == (containerCount + 1));

  containerReports.remove(0);
  exception.expect(SamzaException.class);
  tool.validateContainerCount(attemptId);
}
 

@Public
@Unstable
public static GetContainerReportResponse newInstance(
    ContainerReport containerReport) {
  GetContainerReportResponse response =
      Records.newRecord(GetContainerReportResponse.class);
  response.setContainerReport(containerReport);
  return response;
}
 

@Override
public List<ContainerReport>
    getContainers(ApplicationAttemptId appAttemptId) throws YarnException,
        IOException {
  when(mockContainersResponse.getContainerList()).thenReturn(
    getContainersReport(appAttemptId));
  return super.getContainers(appAttemptId);
}
 

@Override
public ContainerReport getContainerReport(ContainerId containerId)
    throws YarnException, IOException {
  when(mockContainerResponse.getContainerReport()).thenReturn(
    getContainer(containerId));
  return super.getContainerReport(containerId);
}
 

@Override
public ContainerReport getAMContainer(ApplicationAttemptId appAttemptId)
    throws IOException {
  ApplicationReport app =
      getApplication(appAttemptId.getApplicationId());
  return convertToContainerReport(historyStore.getAMContainer(appAttemptId),
      app == null ? null : app.getUser());
}
 

@Test
public void testGetContainerReport() throws Exception {
  ApplicationCLI cli = createAndGetAppCLI();
  ApplicationId applicationId = ApplicationId.newInstance(1234, 5);
  ApplicationAttemptId attemptId = ApplicationAttemptId.newInstance(
      applicationId, 1);
  ContainerId containerId = ContainerId.newContainerId(attemptId, 1);
  ContainerReport container = ContainerReport.newInstance(containerId, null,
      NodeId.newInstance("host", 1234), Priority.UNDEFINED, 1234, 5678,
      "diagnosticInfo", "logURL", 0, ContainerState.COMPLETE,
      "http://" + NodeId.newInstance("host", 2345).toString());
  when(client.getContainerReport(any(ContainerId.class))).thenReturn(
      container);
  int result = cli.run(new String[] { "container", "-status",
      containerId.toString() });
  assertEquals(0, result);
  verify(client).getContainerReport(containerId);
  ByteArrayOutputStream baos = new ByteArrayOutputStream();
  PrintWriter pw = new PrintWriter(baos);
  pw.println("Container Report : ");
  pw.println("\tContainer-Id : container_1234_0005_01_000001");
  pw.println("\tStart-Time : 1234");
  pw.println("\tFinish-Time : 5678");
  pw.println("\tState : COMPLETE");
  pw.println("\tLOG-URL : logURL");
  pw.println("\tHost : host:1234");
  pw.println("\tNodeHttpAddress : http://host:2345");
  pw.println("\tDiagnostics : diagnosticInfo");
  pw.close();
  String appReportStr = baos.toString("UTF-8");
  Assert.assertEquals(appReportStr, sysOutStream.toString());
  verify(sysOut, times(1)).println(isA(String.class));
}
 

@Test(timeout = 10000)
public void testGetContainers() throws YarnException, IOException {
  Configuration conf = new Configuration();
  conf.setBoolean(YarnConfiguration.APPLICATION_HISTORY_ENABLED,
      true);
  
  final YarnClient client = new MockYarnClient();
  client.init(conf);
  client.start();

  ApplicationId applicationId = ApplicationId.newInstance(1234, 5);
  ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(
      applicationId, 1);
  List<ContainerReport> reports = client.getContainers(appAttemptId);
  Assert.assertNotNull(reports);
  Assert.assertEquals(reports.get(0).getContainerId(),
      (ContainerId.newContainerId(appAttemptId, 1)));
  Assert.assertEquals(reports.get(1).getContainerId(),
      (ContainerId.newContainerId(appAttemptId, 2)));
  Assert.assertEquals(reports.get(2).getContainerId(),
      (ContainerId.newContainerId(appAttemptId, 3)));
  
  //First2 containers should come from RM with updated state information and 
  // 3rd container is not there in RM and should
  Assert.assertEquals(ContainerState.RUNNING,
      (reports.get(0).getContainerState()));
  Assert.assertEquals(ContainerState.RUNNING,
      (reports.get(1).getContainerState()));
  Assert.assertEquals(ContainerState.COMPLETE,
      (reports.get(2).getContainerState()));
  client.stop();
}
 

@Override
public GetContainersResponse getContainers(GetContainersRequest request)
    throws YarnException, IOException {
  GetContainersResponse response =
      GetContainersResponse.newInstance(new ArrayList<ContainerReport>(
        history.getContainers(request.getApplicationAttemptId()).values()));
  return response;
}
 

@Test(timeout = 15000)
public void testGetContainersOnHA() throws Exception {
  List<ContainerReport> reports =
      client.getContainers(cluster.createFakeApplicationAttemptId());
  Assert.assertTrue(reports != null && !reports.isEmpty());
  Assert.assertEquals(cluster.createFakeContainerReports(),
      reports);
}