org.apache.hadoop.metrics.MetricsContext Java Examples

ShuffleClientMetrics(String dagName, String vertexName, int taskIndex, Configuration conf, 
    String user) {
  this.numCopiers = 
      conf.getInt(
          TezJobConfig.TEZ_RUNTIME_SHUFFLE_PARALLEL_COPIES, 
          TezJobConfig.TEZ_RUNTIME_SHUFFLE_PARALLEL_COPIES_DEFAULT);

  MetricsContext metricsContext = MetricsUtil.getContext(Constants.TEZ);
  this.shuffleMetrics = 
    MetricsUtil.createRecord(metricsContext, "shuffleInput");
  this.shuffleMetrics.setTag("user", user);
  this.shuffleMetrics.setTag("dagName", dagName);
  this.shuffleMetrics.setTag("taskId", TezRuntimeUtils.getTaskIdentifier(vertexName, taskIndex));
  this.shuffleMetrics.setTag("sessionId", 
      conf.get(
          TezRuntimeFrameworkConfigs.TEZ_RUNTIME_METRICS_SESSION_ID,
          TezRuntimeFrameworkConfigs.TEZ_RUNTIME_METRICS_SESSION_ID_DEFAULT));
  metricsContext.registerUpdater(this);
}
 

public void doUpdates(MetricsContext unused) {
  synchronized (this) {
    if (workerThreads != 0) {
      shuffleMetricsRecord.setMetric("shuffle_handler_busy_percent", 
          100*((float)serverHandlerBusy/workerThreads));
    } else {
      shuffleMetricsRecord.setMetric("shuffle_handler_busy_percent", 0);
    }
    shuffleMetricsRecord.incrMetric("shuffle_output_bytes", 
                                    outputBytes);
    shuffleMetricsRecord.incrMetric("shuffle_failed_outputs", 
                                    failedOutputs);
    shuffleMetricsRecord.incrMetric("shuffle_success_outputs", 
                                    successOutputs);
    outputBytes = 0;
    failedOutputs = 0;
    successOutputs = 0;
  }
  shuffleMetricsRecord.update();
}
 

@InterfaceAudience.Private
public void init(String contextName, ContextFactory factory) {
  super.init(contextName, factory);
  int nKids;
  try {
    String sKids = getAttribute(ARITY_LABEL);
    nKids = Integer.parseInt(sKids);
  } catch (Exception e) {
    LOG.error("Unable to initialize composite metric " + contextName +
              ": could not init arity", e);
    return;
  }
  for (int i = 0; i < nKids; ++i) {
    MetricsContext ctxt = MetricsUtil.getContext(
        String.format(SUB_FMT, contextName, i), contextName);
    if (null != ctxt) {
      subctxt.add(ctxt);
    }
  }
}
 

@InterfaceAudience.Private
@Override
protected void emitRecord(String contextName, String recordName,
    OutputRecord outRec) throws IOException {
  for (MetricsContext ctxt : subctxt) {
    try {
      ((AbstractMetricsContext)ctxt).emitRecord(
        contextName, recordName, outRec);
      if (contextName == null || recordName == null || outRec == null) {
        throw new IOException(contextName + ":" + recordName + ":" + outRec);
      }
    } catch (IOException e) {
      LOG.warn("emitRecord failed: " + ctxt.getContextName(), e);
    }
  }
}
 

@Override
public void doUpdates(MetricsContext unused) {
  synchronized (aggregateJobStats) {
    // Update metrics with aggregate job stats and reset the aggregate.
    aggregateJobStats.incrementMetricsAndReset(metricsRecord);

    incrementMetricsAndReset(metricsRecord, aggregateCounters);

    for (Map.Entry<String, MetricsRecord> entry :
      poolToMetricsRecord.entrySet()) {
      String pool = entry.getKey();

      JobStats poolJobStats = poolToJobStats.get(pool);
      poolJobStats.incrementMetricsAndReset(entry.getValue());

      Counters poolCounters = poolToJobCounters.get(pool);
      incrementMetricsAndReset(entry.getValue(), poolCounters);
    }
  }
}
 

public void init(String contextName, ContextFactory factory) {
  super.init(contextName, factory);
  int nKids;
  try {
    String sKids = getAttribute(ARITY_LABEL);
    nKids = Integer.valueOf(sKids);
  } catch (Exception e) {
    LOG.error("Unable to initialize composite metric " + contextName +
              ": could not init arity", e);
    return;
  }
  for (int i = 0; i < nKids; ++i) {
    MetricsContext ctxt = MetricsUtil.getContext(
        String.format(SUB_FMT, contextName, i), contextName);
    if (null != ctxt) {
      subctxt.add(ctxt);
    }
  }
}
 

public void doUpdates(MetricsContext unused) {
  synchronized (this) {
    shuffleMetrics.incrMetric("shuffle_input_bytes", numBytes);
    shuffleMetrics.incrMetric("shuffle_failed_fetches", 
                              numFailedFetches);
    shuffleMetrics.incrMetric("shuffle_success_fetches", 
                              numSuccessFetches);
    if (numCopiers != 0) {
      shuffleMetrics.setMetric("shuffle_fetchers_busy_percent",
          100*((float)numThreadsBusy/numCopiers));
    } else {
      shuffleMetrics.setMetric("shuffle_fetchers_busy_percent", 0);
    }
    numBytes = 0;
    numSuccessFetches = 0;
    numFailedFetches = 0;
  }
  shuffleMetrics.update();
}
 

public HighTideNodeMetrics(Configuration conf, HighTideNode hightideNode) {
  String sessionId = conf.get("session.id");
  // Initiate Java VM metrics
  JvmMetrics.init("HighTideNode", sessionId);


  // Now the Mbean for the name node - this also registers the MBean
  hightidenodeActivityMBean = new HighTideNodeActivityMBean(registry);

  // Create a record for HighTideNode metrics
  MetricsContext metricsContext = MetricsUtil.getContext("dfs");
  metricsRecord = MetricsUtil.createRecord(metricsContext, "hightidenode");
  metricsRecord.setTag("sessionId", sessionId);
  metricsContext.registerUpdater(this);
  LOG.info("Initializing HighTideNodeMetrics using context object:" +
            metricsContext.getClass().getName());
}
 

public void init(String contextName, ContextFactory factory) {
  super.init(contextName, factory);
  int nKids;
  try {
    String sKids = getAttribute(ARITY_LABEL);
    nKids = Integer.valueOf(sKids);
  } catch (Exception e) {
    LOG.error("Unable to initialize composite metric " + contextName +
              ": could not init arity", e);
    return;
  }
  for (int i = 0; i < nKids; ++i) {
    MetricsContext ctxt = MetricsUtil.getContext(
        String.format(SUB_FMT, contextName, i), contextName);
    if (null != ctxt) {
      subctxt.add(ctxt);
    }
  }
}
 

@InterfaceAudience.Private
@Override
protected void emitRecord(String contextName, String recordName,
    OutputRecord outRec) throws IOException {
  for (MetricsContext ctxt : subctxt) {
    try {
      ((AbstractMetricsContext)ctxt).emitRecord(
        contextName, recordName, outRec);
      if (contextName == null || recordName == null || outRec == null) {
        throw new IOException(contextName + ":" + recordName + ":" + outRec);
      }
    } catch (IOException e) {
      LOG.warn("emitRecord failed: " + ctxt.getContextName(), e);
    }
  }
}
 

@InterfaceAudience.Private
public void init(String contextName, ContextFactory factory) {
  super.init(contextName, factory);
  int nKids;
  try {
    String sKids = getAttribute(ARITY_LABEL);
    nKids = Integer.parseInt(sKids);
  } catch (Exception e) {
    LOG.error("Unable to initialize composite metric " + contextName +
              ": could not init arity", e);
    return;
  }
  for (int i = 0; i < nKids; ++i) {
    MetricsContext ctxt = MetricsUtil.getContext(
        String.format(SUB_FMT, contextName, i), contextName);
    if (null != ctxt) {
      subctxt.add(ctxt);
    }
  }
}
 

public void doUpdates(MetricsContext unused) {
  synchronized (this) {
    shuffleMetrics.incrMetric("shuffle_input_bytes", numBytes);
    shuffleMetrics.incrMetric("shuffle_failed_fetches",
                              numFailedFetches);
    shuffleMetrics.incrMetric("shuffle_success_fetches",
                              numSuccessFetches);
    if (numCopiers != 0) {
      shuffleMetrics.setMetric("shuffle_fetchers_busy_percent",
          100*((float)numThreadsBusy/numCopiers));
    } else {
      shuffleMetrics.setMetric("shuffle_fetchers_busy_percent", 0);
    }
    numBytes = 0;
    numSuccessFetches = 0;
    numFailedFetches = 0;
  }
  shuffleMetrics.update();
}
 

/**
 * Since this object is a registered updater, this method will be called
 * periodically, e.g. every 5 seconds.
 */
@Override
public void doUpdates(MetricsContext unused) {
  synchronized (this) {
    metricsRecord.setMetric("maps_running", tt.mapTotal);
    metricsRecord.setMetric("reduces_running", tt.reduceTotal);
    metricsRecord.setMetric("mapTaskSlots", (short)tt.getMaxCurrentMapTasks());
    metricsRecord.setMetric("reduceTaskSlots", 
                                 (short)tt.getMaxCurrentReduceTasks());
    metricsRecord.incrMetric("tasks_completed", numCompletedTasks);
    metricsRecord.incrMetric("tasks_failed_timeout", timedoutTasks);
    metricsRecord.incrMetric("tasks_failed_ping", tasksFailedPing);
    
    numCompletedTasks = 0;
    timedoutTasks = 0;
    tasksFailedPing = 0;
  }
    metricsRecord.update();
}
 

/**
 * Since this object is a registered updater, this method will be called
 * periodically, e.g. every 5 seconds.
 */
public void doUpdates(MetricsContext unused) {
  synchronized (this) {
    metricsRecord.incrMetric("maps_launched", numMapTasksLaunched);
    metricsRecord.incrMetric("maps_completed", numMapTasksCompleted);
    metricsRecord.incrMetric("reduces_launched", numReduceTasksLaunched);
    metricsRecord.incrMetric("reduces_completed", numReduceTasksCompleted);
    metricsRecord.incrMetric("waiting_maps", numWaitingMaps);
    metricsRecord.incrMetric("waiting_reduces", numWaitingReduces);

    numMapTasksLaunched = 0;
    numMapTasksCompleted = 0;
    numReduceTasksLaunched = 0;
    numReduceTasksCompleted = 0;
    numWaitingMaps = 0;
    numWaitingReduces = 0;
  }
  metricsRecord.update();
}
 

public DataNodeMetrics(Configuration conf, String storageId) {
  String sessionId = conf.get("session.id"); 
  // Initiate reporting of Java VM metrics
  JvmMetrics.init("DataNode", sessionId);
  

  // Now the MBean for the data node
  datanodeActivityMBean = new DataNodeActivityMBean(registry, storageId);
  
  // Create record for DataNode metrics
  MetricsContext context = MetricsUtil.getContext("dfs");
  metricsRecord = MetricsUtil.createRecord(context, "datanode");
  metricsRecord.setTag("sessionId", sessionId);
  context.registerUpdater(this);
}
 

public LookasideMetrics() {
  // Create a record for LookasideCache metrics
  MetricsContext metricsContext = MetricsUtil.getContext("lookasideCache");
  metricsRecord = MetricsUtil.createRecord(metricsContext,
                                           "LookasideFileSystem");
  metricsContext.registerUpdater(this);

}
 

/**
 * Since this object is a registered updater, this method will be called
 * periodically, e.g. every 5 seconds.
 */
public void doUpdates(MetricsContext unused) {
  synchronized (this) {
    for (MetricsBase m : registry.getMetricsList()) {
      m.pushMetric(metricsRecord);
    }
  }
  metricsRecord.update();
}
 

public JobTrackerMetricsInst(JobTracker tracker, JobConf conf) {
  super(tracker, conf);
  String sessionId = conf.getSessionId();
  // Initiate JVM Metrics
  JvmMetrics.init("JobTracker", sessionId);
  // Create a record for map-reduce metrics
  MetricsContext context = MetricsUtil.getContext("mapred");
  metricsRecord = MetricsUtil.createRecord(context, "jobtracker");
  metricsRecord.setTag("sessionId", sessionId);
  context.registerUpdater(this);
}
 

@Override
public void doUpdates(MetricsContext unused) {
    synchronized (this) {
        for (MetricsBase m : metricsRegistry.getMetricsList()) {
            m.pushMetric(metricsRecord);
        }
    }
    metricsRecord.update();
}
 

ShuffleClientMetrics(JobConf conf) {
  MetricsContext metricsContext = MetricsUtil.getContext("mapred");
  this.shuffleMetrics =
    MetricsUtil.createRecord(metricsContext, "shuffleInput");
  this.shuffleMetrics.setTag("user", conf.getUser());
  this.shuffleMetrics.setTag("jobName", conf.getJobName());
  this.shuffleMetrics.setTag("jobId", ReduceTask.this.getJobID().toString());
  this.shuffleMetrics.setTag("taskId", getTaskID().toString());
  this.shuffleMetrics.setTag("sessionId", conf.getSessionId());
  metricsContext.registerUpdater(this);
}
 

/**
 * This will be called periodically (with the period being configuration
 * dependent).
 */
public void doUpdates(MetricsContext context) {
    doMemoryUpdates();
    doGarbageCollectionUpdates();
    doThreadUpdates();
    doEventCountUpdates();
    metrics.update();
}
 

/** Creates a new instance of JvmMetrics */
private JvmMetrics(String processName, String sessionId,
  String recordName) {
    MetricsContext context = MetricsUtil.getContext("jvm");
    metrics = MetricsUtil.createRecord(context, recordName);
    metrics.setTag("processName", processName);
    metrics.setTag("sessionId", sessionId);
    context.registerUpdater(this);
}
 

/**
 * Since this object is a registered updater, this method will be called
 * periodically, e.g. every 5 seconds.
 */
public void doUpdates(MetricsContext unused) {
  synchronized (this) {
    for (MetricsBase m : registry.getMetricsList()) {
      m.pushMetric(metricsRecord);
    }
  }
  metricsRecord.update();
}
 

MetricsRecordDelegator(String recordName, ArrayList<MetricsContext> ctxts) {
  this.recordName = recordName;
  this.subrecs = new ArrayList<MetricsRecord>(ctxts.size());
  for (MetricsContext ctxt : ctxts) {
    subrecs.add(ctxt.createRecord(recordName));
  }
}
 

ShuffleServerMetrics(JobConf conf) {
  MetricsContext context = MetricsUtil.getContext("mapred");
  shuffleMetricsRecord = 
                       MetricsUtil.createRecord(context, "shuffleOutput");
  this.shuffleMetricsRecord.setTag("sessionId", conf.getSessionId());
  context.registerUpdater(this);
}
 

@Override
public void doUpdates(MetricsContext context) {
  synchronized (this) {
    for (MetricsBase m : registry.getMetricsList()) {
      m.pushMetric(metricsRecord);
    }
  }
  metricsRecord.update();
}
 

public RpcMetrics(String hostName, String port, Server server) {
  myServer = server;
  MetricsContext context = MetricsUtil.getContext("rpc");
  metricsRecord = MetricsUtil.createRecord(context, "metrics");

  metricsRecord.setTag("port", port);

  LOG.info("Initializing RPC Metrics with hostName=" 
      + hostName + ", port=" + port);

  context.registerUpdater(this);
  
  // Need to clean up the interface to RpcMgt - don't need both metrics and server params
  rpcMBean = new RpcActivityMBean(registry, hostName, port);
}
 

public void doUpdates(MetricsContext unused) {
  synchronized (this) {
    if (workerThreads != 0) {
      shuffleMetricsRecord.setMetric("shuffle_handler_busy_percent",
          100*((float)serverHandlerBusy/workerThreads));
    } else {
      shuffleMetricsRecord.setMetric("shuffle_handler_busy_percent", 0);
    }
    shuffleMetricsRecord.setMetric("shuffle_queue_len", httpQueueLen);
    shuffleMetricsRecord.incrMetric("shuffle_output_bytes",
                                    outputBytes);
    shuffleMetricsRecord.incrMetric("shuffle_failed_outputs",
                                    failedOutputs);
    shuffleMetricsRecord.incrMetric("shuffle_success_outputs",
                                    successOutputs);
    // Netty map output metrics
    if (nettyWorkerThreadPool != null) {
      shuffleMetricsRecord.setMetric("netty_mapoutput_activecount",
          nettyWorkerThreadPool.getActiveCount());
      shuffleMetricsRecord.setMetric("netty_mapoutput_poolsize",
          nettyWorkerThreadPool.getPoolSize());
      shuffleMetricsRecord.setMetric("netty_mapoutput_maximumpoolsize",
          nettyWorkerThreadPool.getMaximumPoolSize());
      shuffleMetricsRecord.setMetric("netty_mapoutput_largestpoolsize",
          nettyWorkerThreadPool.getLargestPoolSize());
      shuffleMetricsRecord.setMetric("netty_mapoutput_taskcount",
          nettyWorkerThreadPool.getTaskCount());
    }
    outputBytes = 0;
    failedOutputs = 0;
    successOutputs = 0;
  }
  shuffleMetricsRecord.update();
}
 

public LocalJobRunnerMetrics(JobConf conf) {
  String sessionId = conf.getSessionId();
  // Initiate JVM Metrics
  JvmMetrics.init("JobTracker", sessionId);
  // Create a record for map-reduce metrics
  MetricsContext context = MetricsUtil.getContext("mapred");
  // record name is jobtracker for compatibility 
  metricsRecord = MetricsUtil.createRecord(context, "jobtracker");
  metricsRecord.setTag("sessionId", sessionId);
  context.registerUpdater(this);
}
 

ShuffleServerMetrics(JobConf conf) {
  MetricsContext context = MetricsUtil.getContext("mapred");
  shuffleMetricsRecord =
                       MetricsUtil.createRecord(context, "shuffleOutput");
  this.shuffleMetricsRecord.setTag("sessionId", conf.getSessionId());
  context.registerUpdater(this);
}