org.apache.hadoop.yarn.util.Times Java Examples

public ReduceTaskAttemptInfo(TaskAttempt ta, TaskType type) {
  super(ta, type, false);

  this.shuffleFinishTime = ta.getShuffleFinishTime();
  this.mergeFinishTime = ta.getSortFinishTime();
  this.elapsedShuffleTime = Times.elapsed(this.startTime,
      this.shuffleFinishTime, false);
  if (this.elapsedShuffleTime == -1) {
    this.elapsedShuffleTime = 0;
  }
  this.elapsedMergeTime = Times.elapsed(this.shuffleFinishTime,
      this.mergeFinishTime, false);
  if (this.elapsedMergeTime == -1) {
    this.elapsedMergeTime = 0;
  }
  this.elapsedReduceTime = Times.elapsed(this.mergeFinishTime,
      this.finishTime, false);
  if (this.elapsedReduceTime == -1) {
    this.elapsedReduceTime = 0;
  }
}
 

@Override
protected void render(Block html) {
  html._(MetricsOverviewTable.class);
  ResourceManager rm = getInstance(ResourceManager.class);
  ClusterInfo cinfo = new ClusterInfo(rm);
  info("Cluster overview").
    _("Cluster ID:", cinfo.getClusterId()).
    _("ResourceManager state:", cinfo.getState()).
    _("ResourceManager HA state:", cinfo.getHAState()).
    _("ResourceManager RMStateStore:", cinfo.getRMStateStore()).
    _("ResourceManager started on:", Times.format(cinfo.getStartedOn())).
    _("ResourceManager version:", cinfo.getRMBuildVersion() +
        " on " + cinfo.getRMVersionBuiltOn()).
    _("Hadoop version:", cinfo.getHadoopBuildVersion() +
        " on " + cinfo.getHadoopVersionBuiltOn());
  html._(InfoBlock.class);
}
 

@Override
protected void render(Block html) {
  SCMMetricsInfo metricsInfo = new SCMMetricsInfo(
      CleanerMetrics.getInstance(), ClientSCMMetrics.getInstance(),
          SharedCacheUploaderMetrics.getInstance());
  info("Shared Cache Manager overview").
      _("Started on:", Times.format(scm.getStartTime())).
      _("Cache hits: ", metricsInfo.getCacheHits()).
      _("Cache misses: ", metricsInfo.getCacheMisses()).
      _("Cache releases: ", metricsInfo.getCacheReleases()).
      _("Accepted uploads: ", metricsInfo.getAcceptedUploads()).
      _("Rejected uploads: ", metricsInfo.getRejectUploads()).
      _("Deleted files by the cleaner: ", metricsInfo.getTotalDeletedFiles()).
      _("Processed files by the cleaner: ", metricsInfo.getTotalProcessedFiles());
  html._(InfoBlock.class);
}
 

public AppInfo(ApplicationReport app) {
  appId = app.getApplicationId().toString();
  if (app.getCurrentApplicationAttemptId() != null) {
    currentAppAttemptId = app.getCurrentApplicationAttemptId().toString();
  }
  user = app.getUser();
  queue = app.getQueue();
  name = app.getName();
  type = app.getApplicationType();
  host = app.getHost();
  rpcPort = app.getRpcPort();
  appState = app.getYarnApplicationState();
  diagnosticsInfo = app.getDiagnostics();
  trackingUrl = app.getTrackingUrl();
  originalTrackingUrl = app.getOriginalTrackingUrl();
  submittedTime = app.getStartTime();
  startedTime = app.getStartTime();
  finishedTime = app.getFinishTime();
  elapsedTime = Times.elapsed(startedTime, finishedTime);
  finalAppStatus = app.getFinalApplicationStatus();
  progress = app.getProgress() * 100; // in percent
  if (app.getApplicationTags() != null && !app.getApplicationTags().isEmpty()) {
    this.applicationTags = CSV_JOINER.join(app.getApplicationTags());
  }
}
 

public ContainerInfo(ContainerReport container) {
  containerId = container.getContainerId().toString();
  if (container.getAllocatedResource() != null) {
    allocatedMB = container.getAllocatedResource().getMemory();
    allocatedVCores = container.getAllocatedResource().getVirtualCores();
  }
  if (container.getAssignedNode() != null) {
    assignedNodeId = container.getAssignedNode().toString();
  }
  priority = container.getPriority().getPriority();
  startedTime = container.getCreationTime();
  finishedTime = container.getFinishTime();
  elapsedTime = Times.elapsed(startedTime, finishedTime);
  diagnosticsInfo = container.getDiagnosticsInfo();
  logUrl = container.getLogUrl();
  containerExitStatus = container.getContainerExitStatus();
  containerState = container.getContainerState();
  nodeHttpAddress = container.getNodeHttpAddress();
}
 

/**
 * Lists the containers matching the given application attempts
 * 
 * @param appAttemptId
 * @throws YarnException
 * @throws IOException
 */
private void listContainers(String appAttemptId) throws YarnException,
    IOException {
  PrintWriter writer = new PrintWriter(
      new OutputStreamWriter(sysout, Charset.forName("UTF-8")));

  List<ContainerReport> appsReport = client
      .getContainers(ConverterUtils.toApplicationAttemptId(appAttemptId));
  writer.println("Total number of containers " + ":" + appsReport.size());
  writer.printf(CONTAINER_PATTERN, "Container-Id", "Start Time",
      "Finish Time", "State", "Host", "Node Http Address", "LOG-URL");
  for (ContainerReport containerReport : appsReport) {
    writer.printf(
        CONTAINER_PATTERN,
        containerReport.getContainerId(),
        Times.format(containerReport.getCreationTime()),
        Times.format(containerReport.getFinishTime()),      
        containerReport.getContainerState(), containerReport
            .getAssignedNode(), containerReport.getNodeHttpAddress() == null
                ? "N/A" : containerReport.getNodeHttpAddress(),
        containerReport.getLogUrl());
  }
  writer.flush();
}
 

/**
 *
 * @param context
 */
public AppInfo(StramAppContext context)
{
  this.appId = context.getApplicationID().toString();
  this.name = context.getApplicationName();
  this.docLink = context.getApplicationDocLink();
  this.user = context.getUser().toString();
  this.startTime = context.getStartTime();
  this.elapsedTime = Times.elapsed(this.startTime, 0);
  this.appPath = context.getApplicationPath();
  this.appMasterTrackingUrl = context.getAppMasterTrackingUrl();
  this.stats = context.getStats();
  this.gatewayAddress = context.getGatewayAddress();
  this.version = VersionInfo.APEX_VERSION.getBuildVersion();
  this.attributes = new TreeMap<>();
  for (Map.Entry<Attribute<Object>, Object> entry : AttributeMap.AttributeInitializer.getAllAttributes(context, DAGContext.class).entrySet()) {
    this.attributes.put(entry.getKey().getSimpleName(), entry.getKey().codec.toString(entry.getValue()));
  }
  this.gatewayConnected = context.isGatewayConnected();
  this.appDataSources = context.getAppDataSources();
  this.metrics = context.getMetrics();
}
 

public TaskAttemptInfo(TaskAttempt ta, TaskType type, Boolean isRunning) {
  final TaskAttemptReport report = ta.getReport();
  this.type = type.toString();
  this.id = MRApps.toString(ta.getID());
  this.nodeHttpAddress = ta.getNodeHttpAddress();
  this.startTime = report.getStartTime();
  this.finishTime = report.getFinishTime();
  this.assignedContainerId = ConverterUtils.toString(report.getContainerId());
  this.assignedContainer = report.getContainerId();
  this.progress = report.getProgress() * 100;
  this.status = report.getStateString();
  this.state = report.getTaskAttemptState();
  this.elapsedTime = Times
      .elapsed(this.startTime, this.finishTime, isRunning);
  if (this.elapsedTime == -1) {
    this.elapsedTime = 0;
  }
  this.diagnostics = report.getDiagnosticInfo();
  this.rack = ta.getNodeRackName();
}
 

public ReduceTaskAttemptInfo(TaskAttempt ta, TaskType type) {
  super(ta, type, false);

  this.shuffleFinishTime = ta.getShuffleFinishTime();
  this.mergeFinishTime = ta.getSortFinishTime();
  this.elapsedShuffleTime = Times.elapsed(this.startTime,
      this.shuffleFinishTime, false);
  if (this.elapsedShuffleTime == -1) {
    this.elapsedShuffleTime = 0;
  }
  this.elapsedMergeTime = Times.elapsed(this.shuffleFinishTime,
      this.mergeFinishTime, false);
  if (this.elapsedMergeTime == -1) {
    this.elapsedMergeTime = 0;
  }
  this.elapsedReduceTime = Times.elapsed(this.mergeFinishTime,
      this.finishTime, false);
  if (this.elapsedReduceTime == -1) {
    this.elapsedReduceTime = 0;
  }
}
 

public TaskInfo(Task task) {
  TaskType ttype = task.getType();
  this.type = ttype.toString();
  TaskReport report = task.getReport();
  this.startTime = report.getStartTime();
  this.finishTime = report.getFinishTime();
  this.state = report.getTaskState();
  this.elapsedTime = Times.elapsed(this.startTime, this.finishTime,
    this.state == TaskState.RUNNING);
  if (this.elapsedTime == -1) {
    this.elapsedTime = 0;
  }
  this.progress = report.getProgress() * 100;
  this.status =  report.getStatus();
  this.id = MRApps.toString(task.getID());
  this.taskNum = task.getID().getId();
  this.successful = getSuccessfulAttempt(task);
  if (successful != null) {
    this.successfulAttempt = MRApps.toString(successful.getID());
  } else {
    this.successfulAttempt = "";
  }
}
 

public TaskInfo(Task task) {
  TaskType ttype = task.getType();
  this.type = ttype.toString();
  TaskReport report = task.getReport();
  this.startTime = report.getStartTime();
  this.finishTime = report.getFinishTime();
  this.state = report.getTaskState();
  this.elapsedTime = Times.elapsed(this.startTime, this.finishTime,
    this.state == TaskState.RUNNING);
  if (this.elapsedTime == -1) {
    this.elapsedTime = 0;
  }
  this.progress = report.getProgress() * 100;
  this.status =  report.getStatus();
  this.id = MRApps.toString(task.getID());
  this.taskNum = task.getID().getId();
  this.successful = getSuccessfulAttempt(task);
  if (successful != null) {
    this.successfulAttempt = MRApps.toString(successful.getID());
  } else {
    this.successfulAttempt = "";
  }
}
 

public TaskAttemptInfo(TaskAttempt ta, TaskType type, Boolean isRunning) {
  final TaskAttemptReport report = ta.getReport();
  this.type = type.toString();
  this.id = MRApps.toString(ta.getID());
  this.nodeHttpAddress = ta.getNodeHttpAddress();
  this.startTime = report.getStartTime();
  this.finishTime = report.getFinishTime();
  this.assignedContainerId = ConverterUtils.toString(report.getContainerId());
  this.assignedContainer = report.getContainerId();
  this.progress = report.getProgress() * 100;
  this.status = report.getStateString();
  this.state = report.getTaskAttemptState();
  this.elapsedTime = Times
      .elapsed(this.startTime, this.finishTime, isRunning);
  if (this.elapsedTime == -1) {
    this.elapsedTime = 0;
  }
  this.diagnostics = report.getDiagnosticInfo();
  this.rack = ta.getNodeRackName();
}
 

/**
 * Lists the containers matching the given application attempts
 * 
 * @param appAttemptId
 * @throws YarnException
 * @throws IOException
 */
private void listContainers(String appAttemptId) throws YarnException,
    IOException {
  PrintWriter writer = new PrintWriter(
      new OutputStreamWriter(sysout, Charset.forName("UTF-8")));

  List<ContainerReport> appsReport = client
      .getContainers(ConverterUtils.toApplicationAttemptId(appAttemptId));
  writer.println("Total number of containers " + ":" + appsReport.size());
  writer.printf(CONTAINER_PATTERN, "Container-Id", "Start Time",
      "Finish Time", "State", "Host", "Node Http Address", "LOG-URL");
  for (ContainerReport containerReport : appsReport) {
    writer.printf(
        CONTAINER_PATTERN,
        containerReport.getContainerId(),
        Times.format(containerReport.getCreationTime()),
        Times.format(containerReport.getFinishTime()),      
        containerReport.getContainerState(), containerReport
            .getAssignedNode(), containerReport.getNodeHttpAddress() == null
                ? "N/A" : containerReport.getNodeHttpAddress(),
        containerReport.getLogUrl());
  }
  writer.flush();
}
 

@Override
protected void render(Block html) {
  html._(MetricsOverviewTable.class);
  ResourceManager rm = getInstance(ResourceManager.class);
  ClusterInfo cinfo = new ClusterInfo(rm);
  info("Cluster overview").
    _("Cluster ID:", cinfo.getClusterId()).
    _("ResourceManager state:", cinfo.getState()).
    _("ResourceManager HA state:", cinfo.getHAState()).
    _("ResourceManager RMStateStore:", cinfo.getRMStateStore()).
    _("ResourceManager started on:", Times.format(cinfo.getStartedOn())).
    _("ResourceManager version:", cinfo.getRMBuildVersion() +
        " on " + cinfo.getRMVersionBuiltOn()).
    _("Hadoop version:", cinfo.getHadoopBuildVersion() +
        " on " + cinfo.getHadoopVersionBuiltOn());
  html._(InfoBlock.class);
}
 

@Override
protected void render(Block html) {
  SCMMetricsInfo metricsInfo = new SCMMetricsInfo(
      CleanerMetrics.getInstance(), ClientSCMMetrics.getInstance(),
          SharedCacheUploaderMetrics.getInstance());
  info("Shared Cache Manager overview").
      _("Started on:", Times.format(scm.getStartTime())).
      _("Cache hits: ", metricsInfo.getCacheHits()).
      _("Cache misses: ", metricsInfo.getCacheMisses()).
      _("Cache releases: ", metricsInfo.getCacheReleases()).
      _("Accepted uploads: ", metricsInfo.getAcceptedUploads()).
      _("Rejected uploads: ", metricsInfo.getRejectUploads()).
      _("Deleted files by the cleaner: ", metricsInfo.getTotalDeletedFiles()).
      _("Processed files by the cleaner: ", metricsInfo.getTotalProcessedFiles());
  html._(InfoBlock.class);
}
 

public AppInfo(ApplicationReport app) {
  appId = app.getApplicationId().toString();
  if (app.getCurrentApplicationAttemptId() != null) {
    currentAppAttemptId = app.getCurrentApplicationAttemptId().toString();
  }
  user = app.getUser();
  queue = app.getQueue();
  name = app.getName();
  type = app.getApplicationType();
  host = app.getHost();
  rpcPort = app.getRpcPort();
  appState = app.getYarnApplicationState();
  diagnosticsInfo = app.getDiagnostics();
  trackingUrl = app.getTrackingUrl();
  originalTrackingUrl = app.getOriginalTrackingUrl();
  submittedTime = app.getStartTime();
  startedTime = app.getStartTime();
  finishedTime = app.getFinishTime();
  elapsedTime = Times.elapsed(startedTime, finishedTime);
  finalAppStatus = app.getFinalApplicationStatus();
  progress = app.getProgress() * 100; // in percent
  if (app.getApplicationTags() != null && !app.getApplicationTags().isEmpty()) {
    this.applicationTags = CSV_JOINER.join(app.getApplicationTags());
  }
}
 

public ContainerInfo(ContainerReport container) {
  containerId = container.getContainerId().toString();
  if (container.getAllocatedResource() != null) {
    allocatedMB = container.getAllocatedResource().getMemory();
    allocatedVCores = container.getAllocatedResource().getVirtualCores();
    allocatedGCores = container.getAllocatedResource().getGpuCores();
  }
  if (container.getAssignedNode() != null) {
    assignedNodeId = container.getAssignedNode().toString();
  }
  priority = container.getPriority().getPriority();
  startedTime = container.getCreationTime();
  finishedTime = container.getFinishTime();
  elapsedTime = Times.elapsed(startedTime, finishedTime);
  diagnosticsInfo = container.getDiagnosticsInfo();
  logUrl = container.getLogUrl();
  containerExitStatus = container.getContainerExitStatus();
  containerState = container.getContainerState();
  nodeHttpAddress = container.getNodeHttpAddress();
}
 

/**
 *
 * @param context
 */
public AppInfo(StramAppContext context)
{
  this.appId = context.getApplicationID().toString();
  this.name = context.getApplicationName();
  this.docLink = context.getApplicationDocLink();
  this.user = context.getUser().toString();
  this.startTime = context.getStartTime();
  this.elapsedTime = Times.elapsed(this.startTime, 0);
  this.appPath = context.getApplicationPath();
  this.appMasterTrackingUrl = context.getAppMasterTrackingUrl();
  this.stats = context.getStats();
  this.gatewayAddress = context.getGatewayAddress();
  this.version = VersionInfo.APEX_VERSION.getBuildVersion();
  this.attributes = new TreeMap<>();
  for (Map.Entry<Attribute<Object>, Object> entry : AttributeMap.AttributeInitializer.getAllAttributes(context, DAGContext.class).entrySet()) {
    this.attributes.put(entry.getKey().getSimpleName(), entry.getKey().codec.toString(entry.getValue()));
  }
  this.gatewayConnected = context.isGatewayConnected();
  this.appDataSources = context.getAppDataSources();
  this.metrics = context.getMetrics();
}
 

private static void readContainerLogs(DataInputStream valueStream,
    PrintStream out, long logUploadedTime) throws IOException {
  byte[] buf = new byte[65535];

  String fileType = valueStream.readUTF();
  String fileLengthStr = valueStream.readUTF();
  long fileLength = Long.parseLong(fileLengthStr);
  out.print("LogType:");
  out.println(fileType);
  if (logUploadedTime != -1) {
    out.print("Log Upload Time:");
    out.println(Times.format(logUploadedTime));
  }
  out.print("LogLength:");
  out.println(fileLengthStr);
  out.println("Log Contents:");

  long curRead = 0;
  long pendingRead = fileLength - curRead;
  int toRead =
            pendingRead > buf.length ? buf.length : (int) pendingRead;
  int len = valueStream.read(buf, 0, toRead);
  while (len != -1 && curRead < fileLength) {
    out.write(buf, 0, len);
    curRead += len;

    pendingRead = fileLength - curRead;
    toRead =
              pendingRead > buf.length ? buf.length : (int) pendingRead;
    len = valueStream.read(buf, 0, toRead);
  }
  out.println("End of LogType:" + fileType);
  out.println("");
}
 

public AppInfo(App app, AppContext context) {
  this.appId = context.getApplicationID().toString();
  this.name = context.getApplicationName().toString();
  this.user = context.getUser().toString();
  this.startedOn = context.getStartTime();
  this.elapsedTime = Times.elapsed(this.startedOn, 0);
}
 

public void verifyAMJobGeneric(Job job, String id, String user, String name,
    String state, long startTime, long finishTime, long elapsedTime,
    int mapsTotal, int mapsCompleted, int reducesTotal, int reducesCompleted,
    float reduceProgress, float mapProgress) {
  JobReport report = job.getReport();

  WebServicesTestUtils.checkStringMatch("id", MRApps.toString(job.getID()),
      id);
  WebServicesTestUtils.checkStringMatch("user", job.getUserName().toString(),
      user);
  WebServicesTestUtils.checkStringMatch("name", job.getName(), name);
  WebServicesTestUtils.checkStringMatch("state", job.getState().toString(),
      state);

  assertEquals("startTime incorrect", report.getStartTime(), startTime);
  assertEquals("finishTime incorrect", report.getFinishTime(), finishTime);
  assertEquals("elapsedTime incorrect",
      Times.elapsed(report.getStartTime(), report.getFinishTime()),
      elapsedTime);
  assertEquals("mapsTotal incorrect", job.getTotalMaps(), mapsTotal);
  assertEquals("mapsCompleted incorrect", job.getCompletedMaps(),
      mapsCompleted);
  assertEquals("reducesTotal incorrect", job.getTotalReduces(), reducesTotal);
  assertEquals("reducesCompleted incorrect", job.getCompletedReduces(),
      reducesCompleted);
  assertEquals("mapProgress incorrect", report.getMapProgress() * 100,
      mapProgress, 0);
  assertEquals("reduceProgress incorrect", report.getReduceProgress() * 100,
      reduceProgress, 0);
}
 

/**
 * The content of this page is the attempts block
 * @return AttemptsBlock.class
 */
@Override protected Class<? extends SubView> content() {
  HistoryInfo info = new HistoryInfo();
  info("History Server").
    _("BuildVersion", info.getHadoopBuildVersion()
      + " on " + info.getHadoopVersionBuiltOn()).
    _("History Server started on", Times.format(info.getStartedOn()));
  return InfoBlock.class;
}
 

/**
 * The content of this page is the attempts block
 * @return AttemptsBlock.class
 */
@Override protected Class<? extends SubView> content() {
  HistoryInfo info = new HistoryInfo();
  info("History Server").
    _("BuildVersion", info.getHadoopBuildVersion()
      + " on " + info.getHadoopVersionBuiltOn()).
    _("History Server started on", Times.format(info.getStartedOn()));
  return InfoBlock.class;
}
 

public void verifyAMJobGeneric(Job job, String id, String user, String name,
    String state, long startTime, long finishTime, long elapsedTime,
    int mapsTotal, int mapsCompleted, int reducesTotal, int reducesCompleted,
    float reduceProgress, float mapProgress) {
  JobReport report = job.getReport();

  WebServicesTestUtils.checkStringMatch("id", MRApps.toString(job.getID()),
      id);
  WebServicesTestUtils.checkStringMatch("user", job.getUserName().toString(),
      user);
  WebServicesTestUtils.checkStringMatch("name", job.getName(), name);
  WebServicesTestUtils.checkStringMatch("state", job.getState().toString(),
      state);

  assertEquals("startTime incorrect", report.getStartTime(), startTime);
  assertEquals("finishTime incorrect", report.getFinishTime(), finishTime);
  assertEquals("elapsedTime incorrect",
      Times.elapsed(report.getStartTime(), report.getFinishTime()),
      elapsedTime);
  assertEquals("mapsTotal incorrect", job.getTotalMaps(), mapsTotal);
  assertEquals("mapsCompleted incorrect", job.getCompletedMaps(),
      mapsCompleted);
  assertEquals("reducesTotal incorrect", job.getTotalReduces(), reducesTotal);
  assertEquals("reducesCompleted incorrect", job.getCompletedReduces(),
      reducesCompleted);
  assertEquals("mapProgress incorrect", report.getMapProgress() * 100,
      mapProgress, 0);
  assertEquals("reduceProgress incorrect", report.getReduceProgress() * 100,
      reduceProgress, 0);
}
 

public AppInfo(App app, AppContext context) {
  this.appId = context.getApplicationID().toString();
  this.name = context.getApplicationName().toString();
  this.user = context.getUser().toString();
  this.startedOn = context.getStartTime();
  this.elapsedTime = Times.elapsed(this.startedOn, 0);
}
 

private static void readContainerLogs(DataInputStream valueStream,
    PrintStream out, long logUploadedTime) throws IOException {
  byte[] buf = new byte[65535];

  String fileType = valueStream.readUTF();
  String fileLengthStr = valueStream.readUTF();
  long fileLength = Long.parseLong(fileLengthStr);
  out.print("LogType:");
  out.println(fileType);
  if (logUploadedTime != -1) {
    out.print("Log Upload Time:");
    out.println(Times.format(logUploadedTime));
  }
  out.print("LogLength:");
  out.println(fileLengthStr);
  out.println("Log Contents:");

  long curRead = 0;
  long pendingRead = fileLength - curRead;
  int toRead =
            pendingRead > buf.length ? buf.length : (int) pendingRead;
  int len = valueStream.read(buf, 0, toRead);
  while (len != -1 && curRead < fileLength) {
    out.write(buf, 0, len);
    curRead += len;

    pendingRead = fileLength - curRead;
    toRead =
              pendingRead > buf.length ? buf.length : (int) pendingRead;
    len = valueStream.read(buf, 0, toRead);
  }
  out.println("End of LogType:" + fileType);
  out.println("");
}
 

@Override
protected void render(Block html) {
  html._(MetricsOverviewTable.class);

  ResourceScheduler sched = rm.getResourceScheduler();
  String type = $(NODE_STATE);
  String labelFilter = $(NODE_LABEL, CommonNodeLabelsManager.ANY).trim();
  TBODY<TABLE<Hamlet>> tbody =
      html.table("#nodes").thead().tr()
          .th(".nodelabels", "Node Labels")
          .th(".rack", "Rack")
          .th(".state", "Node State")
          .th(".nodeaddress", "Node Address")
          .th(".nodehttpaddress", "Node HTTP Address")
          .th(".lastHealthUpdate", "Last health-update")
          .th(".healthReport", "Health-report")
          .th(".containers", "Containers")
          .th(".mem", "Mem Used")
          .th(".mem", "Mem Avail")
          .th(".vcores", "VCores Used")
          .th(".vcores", "VCores Avail")
          .th(".nodeManagerVersion", "Version")._()._().tbody();
  NodeState stateFilter = null;
  if (type != null && !type.isEmpty()) {
    stateFilter = NodeState.valueOf(StringUtils.toUpperCase(type));
  }
  Collection<RMNode> rmNodes = this.rm.getRMContext().getRMNodes().values();
  boolean isInactive = false;
  if (stateFilter != null) {
    switch (stateFilter) {
    case DECOMMISSIONED:
    case LOST:
    case REBOOTED:
      rmNodes = this.rm.getRMContext().getInactiveRMNodes().values();
      isInactive = true;
      break;
    default:
      LOG.debug("Unexpected state filter for inactive RM node");
    }
  }
  for (RMNode ni : rmNodes) {
    if (stateFilter != null) {
      NodeState state = ni.getState();
      if (!stateFilter.equals(state)) {
        continue;
      }
    } else {
      // No filter. User is asking for all nodes. Make sure you skip the
      // unhealthy nodes.
      if (ni.getState() == NodeState.UNHEALTHY) {
        continue;
      }
    }
    // Besides state, we need to filter label as well.
    if (!labelFilter.equals(RMNodeLabelsManager.ANY)) {
      if (labelFilter.isEmpty()) {
        // Empty label filter means only shows nodes without label
        if (!ni.getNodeLabels().isEmpty()) {
          continue;
        }
      } else if (!ni.getNodeLabels().contains(labelFilter)) {
        // Only nodes have given label can show on web page.
        continue;
      }
    }
    NodeInfo info = new NodeInfo(ni, sched);
    int usedMemory = (int) info.getUsedMemory();
    int availableMemory = (int) info.getAvailableMemory();
    TR<TBODY<TABLE<Hamlet>>> row =
        tbody.tr().td(StringUtils.join(",", info.getNodeLabels()))
            .td(info.getRack()).td(info.getState()).td(info.getNodeId());
    if (isInactive) {
      row.td()._("N/A")._();
    } else {
      String httpAddress = info.getNodeHTTPAddress();
      row.td().a("//" + httpAddress, httpAddress)._();
    }
    row.td().br().$title(String.valueOf(info.getLastHealthUpdate()))._()
        ._(Times.format(info.getLastHealthUpdate()))._()
        .td(info.getHealthReport())
        .td(String.valueOf(info.getNumContainers())).td().br()
        .$title(String.valueOf(usedMemory))._()
        ._(StringUtils.byteDesc(usedMemory * BYTES_IN_MB))._().td().br()
        .$title(String.valueOf(availableMemory))._()
        ._(StringUtils.byteDesc(availableMemory * BYTES_IN_MB))._()
        .td(String.valueOf(info.getUsedVirtualCores()))
        .td(String.valueOf(info.getAvailableVirtualCores()))
        .td(ni.getNodeManagerVersion())._();
  }
  tbody._()._();
}
 

@SuppressWarnings({ "rawtypes", "unchecked" })
public AppInfo(ResourceManager rm, RMApp app, Boolean hasAccess,
    String schemePrefix) {
  this.schemePrefix = schemePrefix;
  if (app != null) {
    String trackingUrl = app.getTrackingUrl();
    this.state = app.createApplicationState();
    this.trackingUrlIsNotReady = trackingUrl == null || trackingUrl.isEmpty()
        || YarnApplicationState.NEW == this.state
        || YarnApplicationState.NEW_SAVING == this.state
        || YarnApplicationState.SUBMITTED == this.state
        || YarnApplicationState.ACCEPTED == this.state;
    this.trackingUI = this.trackingUrlIsNotReady ? "UNASSIGNED" : (app
        .getFinishTime() == 0 ? "ApplicationMaster" : "History");
    if (!trackingUrlIsNotReady) {
      this.trackingUrl =
          WebAppUtils.getURLWithScheme(schemePrefix,
              trackingUrl);
      this.trackingUrlPretty = this.trackingUrl;
    } else {
      this.trackingUrlPretty = "UNASSIGNED";
    }
    this.applicationId = app.getApplicationId();
    this.applicationType = app.getApplicationType();
    this.appIdNum = String.valueOf(app.getApplicationId().getId());
    this.id = app.getApplicationId().toString();
    this.user = app.getUser().toString();
    this.name = app.getName().toString();
    this.queue = app.getQueue().toString();
    this.progress = app.getProgress() * 100;
    this.diagnostics = app.getDiagnostics().toString();
    if (diagnostics == null || diagnostics.isEmpty()) {
      this.diagnostics = "";
    }
    if (app.getApplicationTags() != null && !app.getApplicationTags().isEmpty()) {
      this.applicationTags = Joiner.on(',').join(app.getApplicationTags());
    }
    this.finalStatus = app.getFinalApplicationStatus();
    this.clusterId = ResourceManager.getClusterTimeStamp();
    if (hasAccess) {
      this.startedTime = app.getStartTime();
      this.finishedTime = app.getFinishTime();
      this.elapsedTime = Times.elapsed(app.getStartTime(),
          app.getFinishTime());

      RMAppAttempt attempt = app.getCurrentAppAttempt();
      if (attempt != null) {
        Container masterContainer = attempt.getMasterContainer();
        if (masterContainer != null) {
          this.amContainerLogsExist = true;
          this.amContainerLogs = WebAppUtils.getRunningLogURL(
              schemePrefix + masterContainer.getNodeHttpAddress(),
              ConverterUtils.toString(masterContainer.getId()),
              app.getUser());
          this.amHostHttpAddress = masterContainer.getNodeHttpAddress();
        }
        
        ApplicationResourceUsageReport resourceReport = attempt
            .getApplicationResourceUsageReport();
        if (resourceReport != null) {
          Resource usedResources = resourceReport.getUsedResources();
          allocatedMB = usedResources.getMemory();
          allocatedVCores = usedResources.getVirtualCores();
          runningContainers = resourceReport.getNumUsedContainers();
        }
        resourceRequests =
            ((AbstractYarnScheduler) rm.getRMContext().getScheduler())
              .getPendingResourceRequestsForAttempt(attempt.getAppAttemptId());
      }
    }

    // copy preemption info fields
    RMAppMetrics appMetrics = app.getRMAppMetrics();
    numAMContainerPreempted =
        appMetrics.getNumAMContainersPreempted();
    preemptedResourceMB =
        appMetrics.getResourcePreempted().getMemory();
    numNonAMContainerPreempted =
        appMetrics.getNumNonAMContainersPreempted();
    preemptedResourceVCores =
        appMetrics.getResourcePreempted().getVirtualCores();
    memorySeconds = appMetrics.getMemorySeconds();
    vcoreSeconds = appMetrics.getVcoreSeconds();
  }
}
 

private void testReadAcontainerLog(boolean logUploadedTime) throws Exception {
  Configuration conf = new Configuration();
  File workDir = new File(testWorkDir, "testReadAcontainerLogs1");
  Path remoteAppLogFile =
      new Path(workDir.getAbsolutePath(), "aggregatedLogFile");
  Path srcFileRoot = new Path(workDir.getAbsolutePath(), "srcFiles");
  ContainerId testContainerId = TestContainerId.newContainerId(1, 1, 1, 1);
  Path t =
      new Path(srcFileRoot, testContainerId.getApplicationAttemptId()
          .getApplicationId().toString());
  Path srcFilePath = new Path(t, testContainerId.toString());

  int numChars = 80000;

  // create a sub-folder under srcFilePath
  // and create file logs in this sub-folder.
  // We only aggregate top level files.
  // So, this log file should be ignored.
  Path subDir = new Path(srcFilePath, "subDir");
  fs.mkdirs(subDir);
  writeSrcFile(subDir, "logs", numChars);

  // create file stderr and stdout in containerLogDir
  writeSrcFile(srcFilePath, "stderr", numChars);
  writeSrcFile(srcFilePath, "stdout", numChars);

  UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
  LogWriter logWriter = new LogWriter(conf, remoteAppLogFile, ugi);

  LogKey logKey = new LogKey(testContainerId);
  LogValue logValue =
      new LogValue(Collections.singletonList(srcFileRoot.toString()),
          testContainerId, ugi.getShortUserName());

  // When we try to open FileInputStream for stderr, it will throw out an IOException.
  // Skip the log aggregation for stderr.
  LogValue spyLogValue = spy(logValue);
  File errorFile = new File((new Path(srcFilePath, "stderr")).toString());
  doThrow(new IOException("Mock can not open FileInputStream")).when(
    spyLogValue).secureOpenFile(errorFile);

  logWriter.append(logKey, spyLogValue);
  logWriter.close();

  // make sure permission are correct on the file
  FileStatus fsStatus =  fs.getFileStatus(remoteAppLogFile);
  Assert.assertEquals("permissions on log aggregation file are wrong",  
    FsPermission.createImmutable((short) 0640), fsStatus.getPermission()); 

  LogReader logReader = new LogReader(conf, remoteAppLogFile);
  LogKey rLogKey = new LogKey();
  DataInputStream dis = logReader.next(rLogKey);
  Writer writer = new StringWriter();

  if (logUploadedTime) {
    LogReader.readAcontainerLogs(dis, writer, System.currentTimeMillis());
  } else {
    LogReader.readAcontainerLogs(dis, writer);
  }

  // We should only do the log aggregation for stdout.
  // Since we could not open the fileInputStream for stderr, this file is not
  // aggregated.
  String s = writer.toString();
  int expectedLength =
      "LogType:stdout".length()
          + (logUploadedTime ? ("\nLog Upload Time:" + Times.format(System
            .currentTimeMillis())).length() : 0)
          + ("\nLogLength:" + numChars).length()
          + "\nLog Contents:\n".length() + numChars + "\n".length()
          + "End of LogType:stdout\n".length();
  Assert.assertTrue("LogType not matched", s.contains("LogType:stdout"));
  Assert.assertTrue("log file:stderr should not be aggregated.", !s.contains("LogType:stderr"));
  Assert.assertTrue("log file:logs should not be aggregated.", !s.contains("LogType:logs"));
  Assert.assertTrue("LogLength not matched", s.contains("LogLength:" + numChars));
  Assert.assertTrue("Log Contents not matched", s.contains("Log Contents"));
  
  StringBuilder sb = new StringBuilder();
  for (int i = 0 ; i < numChars ; i++) {
    sb.append(filler);
  }
  String expectedContent = sb.toString();
  Assert.assertTrue("Log content incorrect", s.contains(expectedContent));
  
  Assert.assertEquals(expectedLength, s.length());
}
 

@SuppressWarnings({ "rawtypes", "unchecked" })
public AppInfo(ResourceManager rm, RMApp app, Boolean hasAccess,
    String schemePrefix) {
  this.schemePrefix = schemePrefix;
  if (app != null) {
    String trackingUrl = app.getTrackingUrl();
    this.state = app.createApplicationState();
    this.trackingUrlIsNotReady = trackingUrl == null || trackingUrl.isEmpty()
        || YarnApplicationState.NEW == this.state
        || YarnApplicationState.NEW_SAVING == this.state
        || YarnApplicationState.SUBMITTED == this.state
        || YarnApplicationState.ACCEPTED == this.state;
    this.trackingUI = this.trackingUrlIsNotReady ? "UNASSIGNED" : (app
        .getFinishTime() == 0 ? "ApplicationMaster" : "History");
    if (!trackingUrlIsNotReady) {
      this.trackingUrl =
          WebAppUtils.getURLWithScheme(schemePrefix,
              trackingUrl);
      this.trackingUrlPretty = this.trackingUrl;
    } else {
      this.trackingUrlPretty = "UNASSIGNED";
    }
    this.applicationId = app.getApplicationId();
    this.applicationType = app.getApplicationType();
    this.appIdNum = String.valueOf(app.getApplicationId().getId());
    this.id = app.getApplicationId().toString();
    this.user = app.getUser().toString();
    this.name = app.getName().toString();
    this.queue = app.getQueue().toString();
    this.progress = app.getProgress() * 100;
    this.diagnostics = app.getDiagnostics().toString();
    if (diagnostics == null || diagnostics.isEmpty()) {
      this.diagnostics = "";
    }
    if (app.getApplicationTags() != null && !app.getApplicationTags().isEmpty()) {
      this.applicationTags = Joiner.on(',').join(app.getApplicationTags());
    }
    this.finalStatus = app.getFinalApplicationStatus();
    this.clusterId = ResourceManager.getClusterTimeStamp();
    if (hasAccess) {
      this.startedTime = app.getStartTime();
      this.finishedTime = app.getFinishTime();
      this.elapsedTime = Times.elapsed(app.getStartTime(),
          app.getFinishTime());

      RMAppAttempt attempt = app.getCurrentAppAttempt();
      if (attempt != null) {
        Container masterContainer = attempt.getMasterContainer();
        if (masterContainer != null) {
          this.amContainerLogsExist = true;
          this.amContainerLogs = WebAppUtils.getRunningLogURL(
              schemePrefix + masterContainer.getNodeHttpAddress(),
              ConverterUtils.toString(masterContainer.getId()),
              app.getUser());
          this.amHostHttpAddress = masterContainer.getNodeHttpAddress();
        }
        
        ApplicationResourceUsageReport resourceReport = attempt
            .getApplicationResourceUsageReport();
        if (resourceReport != null) {
          Resource usedResources = resourceReport.getUsedResources();
          allocatedMB = usedResources.getMemory();
          allocatedVCores = usedResources.getVirtualCores();
          allocatedGCores = usedResources.getGpuCores();
          runningContainers = resourceReport.getNumUsedContainers();
        }
        resourceRequests =
            ((AbstractYarnScheduler) rm.getRMContext().getScheduler())
              .getPendingResourceRequestsForAttempt(attempt.getAppAttemptId());
      }
    }

    // copy preemption info fields
    RMAppMetrics appMetrics = app.getRMAppMetrics();
    numAMContainerPreempted =
        appMetrics.getNumAMContainersPreempted();
    preemptedResourceMB =
        appMetrics.getResourcePreempted().getMemory();
    numNonAMContainerPreempted =
        appMetrics.getNumNonAMContainersPreempted();
    preemptedResourceVCores =
        appMetrics.getResourcePreempted().getVirtualCores();
    preemptedResourceGCores =
        appMetrics.getResourcePreempted().getGpuCores();
    memorySeconds = appMetrics.getMemorySeconds();
    vcoreSeconds = appMetrics.getVcoreSeconds();
    gcoreSeconds = appMetrics.getGcoreSeconds();
  }
}