org.apache.hadoop.metrics2.lib.MutableQuantiles Java Examples

The following examples show how to use org.apache.hadoop.metrics2.lib.MutableQuantiles. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: QueryMetrics.java    From kylin with Apache License 2.0 6 votes vote down vote up 
public QueryMetrics(int[] intervals) {
    queryLatencyTimeMillisQuantiles = new MutableQuantiles[intervals.length];
    scanRowCountQuantiles = new MutableQuantiles[intervals.length];
    resultRowCountQuantiles = new MutableQuantiles[intervals.length];
    cacheHitCountQuantiles = new MutableQuantiles[intervals.length];

    for (int i = 0; i < intervals.length; i++) {
        int interval = intervals[i];

        queryLatencyTimeMillisQuantiles[i] = registry.newQuantiles("QueryLatency" + interval + "s", "Query queue time in milli second", "ops", "", interval);
        scanRowCountQuantiles[i] = registry.newQuantiles("ScanRowCount" + interval + "s", "Scan row count in milli second", "ops", "", interval);
        resultRowCountQuantiles[i] = registry.newQuantiles("ResultRowCount" + interval + "s", "Result row count in milli second", "ops", "", interval);
        cacheHitCountQuantiles[i] = registry.newQuantiles("CacheHitCount" + interval + "s", "Cache Hit Count in milli second", "ops", "", interval);
    }

    queryLatency = registry.newRate("QueryLatency", "", true);
    scanRowCount = registry.newRate("ScanRowCount", "", true);
    resultRowCount = registry.newRate("ResultRowCount", "", true);
}
Example #2
Source File: IPCLoggerChannelMetrics.java    From hadoop with Apache License 2.0 6 votes vote down vote up 
private IPCLoggerChannelMetrics(IPCLoggerChannel ch) {
  this.ch = ch;
  
  Configuration conf = new HdfsConfiguration();
  int[] intervals = 
      conf.getInts(DFSConfigKeys.DFS_METRICS_PERCENTILES_INTERVALS_KEY);
  if (intervals != null) {
    writeEndToEndLatencyQuantiles = new MutableQuantiles[intervals.length];
    writeRpcLatencyQuantiles = new MutableQuantiles[intervals.length];
    for (int i = 0; i < writeEndToEndLatencyQuantiles.length; i++) {
      int interval = intervals[i];
      writeEndToEndLatencyQuantiles[i] = registry.newQuantiles(
          "writesE2E" + interval + "s",
          "End-to-end time for write operations", "ops", "LatencyMicros", interval);
      writeRpcLatencyQuantiles[i] = registry.newQuantiles(
          "writesRpc" + interval + "s",
          "RPC RTT for write operations", "ops", "LatencyMicros", interval);
    }
  } else {
    writeEndToEndLatencyQuantiles = null;
    writeRpcLatencyQuantiles = null;
  }
}
Example #3
Source File: Nfs3Metrics.java    From big-c with Apache License 2.0 6 votes vote down vote up 
public Nfs3Metrics(String name, String sessionId, int[] intervals,
    final JvmMetrics jvmMetrics) {
  this.name = name;
  this.jvmMetrics = jvmMetrics;
  registry.tag(SessionId, sessionId);

  final int len = intervals.length;
  readNanosQuantiles = new MutableQuantiles[len];
  writeNanosQuantiles = new MutableQuantiles[len];
  commitNanosQuantiles = new MutableQuantiles[len];

  for (int i = 0; i < len; i++) {
    int interval = intervals[i];
    readNanosQuantiles[i] = registry.newQuantiles("readProcessNanos"
        + interval + "s", "Read process in ns", "ops", "latency", interval);
    writeNanosQuantiles[i] = registry.newQuantiles("writeProcessNanos"
        + interval + "s", "Write process in ns", "ops", "latency", interval);
    commitNanosQuantiles[i] = registry.newQuantiles("commitProcessNanos"
        + interval + "s", "Commit process in ns", "ops", "latency", interval);
  }
}
Example #4
Source File: NameNodeMetrics.java    From big-c with Apache License 2.0 6 votes vote down vote up 
NameNodeMetrics(String processName, String sessionId, int[] intervals,
    final JvmMetrics jvmMetrics) {
  this.jvmMetrics = jvmMetrics;
  registry.tag(ProcessName, processName).tag(SessionId, sessionId);
  
  final int len = intervals.length;
  syncsQuantiles = new MutableQuantiles[len];
  blockReportQuantiles = new MutableQuantiles[len];
  cacheReportQuantiles = new MutableQuantiles[len];
  
  for (int i = 0; i < len; i++) {
    int interval = intervals[i];
    syncsQuantiles[i] = registry.newQuantiles(
        "syncs" + interval + "s",
        "Journal syncs", "ops", "latency", interval);
    blockReportQuantiles[i] = registry.newQuantiles(
        "blockReport" + interval + "s", 
        "Block report", "ops", "latency", interval);
    cacheReportQuantiles[i] = registry.newQuantiles(
        "cacheReport" + interval + "s",
        "Cache report", "ops", "latency", interval);
  }
}
Example #5
Source File: IPCLoggerChannelMetrics.java    From big-c with Apache License 2.0 6 votes vote down vote up 
private IPCLoggerChannelMetrics(IPCLoggerChannel ch) {
  this.ch = ch;
  
  Configuration conf = new HdfsConfiguration();
  int[] intervals = 
      conf.getInts(DFSConfigKeys.DFS_METRICS_PERCENTILES_INTERVALS_KEY);
  if (intervals != null) {
    writeEndToEndLatencyQuantiles = new MutableQuantiles[intervals.length];
    writeRpcLatencyQuantiles = new MutableQuantiles[intervals.length];
    for (int i = 0; i < writeEndToEndLatencyQuantiles.length; i++) {
      int interval = intervals[i];
      writeEndToEndLatencyQuantiles[i] = registry.newQuantiles(
          "writesE2E" + interval + "s",
          "End-to-end time for write operations", "ops", "LatencyMicros", interval);
      writeRpcLatencyQuantiles[i] = registry.newQuantiles(
          "writesRpc" + interval + "s",
          "RPC RTT for write operations", "ops", "LatencyMicros", interval);
    }
  } else {
    writeEndToEndLatencyQuantiles = null;
    writeRpcLatencyQuantiles = null;
  }
}
Example #6
Source File: NameNodeMetrics.java    From hadoop with Apache License 2.0 6 votes vote down vote up 
NameNodeMetrics(String processName, String sessionId, int[] intervals,
    final JvmMetrics jvmMetrics) {
  this.jvmMetrics = jvmMetrics;
  registry.tag(ProcessName, processName).tag(SessionId, sessionId);
  
  final int len = intervals.length;
  syncsQuantiles = new MutableQuantiles[len];
  blockReportQuantiles = new MutableQuantiles[len];
  cacheReportQuantiles = new MutableQuantiles[len];
  
  for (int i = 0; i < len; i++) {
    int interval = intervals[i];
    syncsQuantiles[i] = registry.newQuantiles(
        "syncs" + interval + "s",
        "Journal syncs", "ops", "latency", interval);
    blockReportQuantiles[i] = registry.newQuantiles(
        "blockReport" + interval + "s", 
        "Block report", "ops", "latency", interval);
    cacheReportQuantiles[i] = registry.newQuantiles(
        "cacheReport" + interval + "s",
        "Cache report", "ops", "latency", interval);
  }
}
Example #7
Source File: Nfs3Metrics.java    From hadoop with Apache License 2.0 6 votes vote down vote up 
public Nfs3Metrics(String name, String sessionId, int[] intervals,
    final JvmMetrics jvmMetrics) {
  this.name = name;
  this.jvmMetrics = jvmMetrics;
  registry.tag(SessionId, sessionId);

  final int len = intervals.length;
  readNanosQuantiles = new MutableQuantiles[len];
  writeNanosQuantiles = new MutableQuantiles[len];
  commitNanosQuantiles = new MutableQuantiles[len];

  for (int i = 0; i < len; i++) {
    int interval = intervals[i];
    readNanosQuantiles[i] = registry.newQuantiles("readProcessNanos"
        + interval + "s", "Read process in ns", "ops", "latency", interval);
    writeNanosQuantiles[i] = registry.newQuantiles("writeProcessNanos"
        + interval + "s", "Write process in ns", "ops", "latency", interval);
    commitNanosQuantiles[i] = registry.newQuantiles("commitProcessNanos"
        + interval + "s", "Commit process in ns", "ops", "latency", interval);
  }
}
Example #8
Source File: QueryMetrics.java    From kylin-on-parquet-v2 with Apache License 2.0 6 votes vote down vote up 
public QueryMetrics(int[] intervals) {
    queryLatencyTimeMillisQuantiles = new MutableQuantiles[intervals.length];
    scanRowCountQuantiles = new MutableQuantiles[intervals.length];
    resultRowCountQuantiles = new MutableQuantiles[intervals.length];
    cacheHitCountQuantiles = new MutableQuantiles[intervals.length];

    for (int i = 0; i < intervals.length; i++) {
        int interval = intervals[i];

        queryLatencyTimeMillisQuantiles[i] = registry.newQuantiles("QueryLatency" + interval + "s", "Query queue time in milli second", "ops", "", interval);
        scanRowCountQuantiles[i] = registry.newQuantiles("ScanRowCount" + interval + "s", "Scan row count in milli second", "ops", "", interval);
        resultRowCountQuantiles[i] = registry.newQuantiles("ResultRowCount" + interval + "s", "Result row count in milli second", "ops", "", interval);
        cacheHitCountQuantiles[i] = registry.newQuantiles("CacheHitCount" + interval + "s", "Cache Hit Count in milli second", "ops", "", interval);
    }

    queryLatency = registry.newRate("QueryLatency", "", true);
    scanRowCount = registry.newRate("ScanRowCount", "", true);
    resultRowCount = registry.newRate("ResultRowCount", "", true);
}
Example #9
Source File: UserGroupInformation.java    From big-c with Apache License 2.0 5 votes vote down vote up 
void addGetGroups(long latency) {
  getGroups.add(latency);
  if (getGroupsQuantiles != null) {
    for (MutableQuantiles q : getGroupsQuantiles) {
      q.add(latency);
    }
  }
}
Example #10
Source File: ContainerMetrics.java    From hadoop-ozone with Apache License 2.0 5 votes vote down vote up 
public ContainerMetrics(int[] intervals) {
  int numEnumEntries = ContainerProtos.Type.values().length;
  final int len = intervals.length;
  this.numOpsArray = new MutableCounterLong[numEnumEntries];
  this.opsBytesArray = new MutableCounterLong[numEnumEntries];
  this.opsLatency = new MutableRate[numEnumEntries];
  this.opsLatQuantiles = new MutableQuantiles[numEnumEntries][len];
  this.registry = new MetricsRegistry("StorageContainerMetrics");
  for (int i = 0; i < numEnumEntries; i++) {
    numOpsArray[i] = registry.newCounter(
        "num" + ContainerProtos.Type.forNumber(i + 1),
        "number of " + ContainerProtos.Type.forNumber(i + 1) + " ops",
        (long) 0);
    opsBytesArray[i] = registry.newCounter(
        "bytes" + ContainerProtos.Type.forNumber(i + 1),
        "bytes used by " + ContainerProtos.Type.forNumber(i + 1) + "op",
        (long) 0);
    opsLatency[i] = registry.newRate(
        "latency" + ContainerProtos.Type.forNumber(i + 1),
        ContainerProtos.Type.forNumber(i + 1) + " op");

    for (int j = 0; j < len; j++) {
      int interval = intervals[j];
      String quantileName = ContainerProtos.Type.forNumber(i + 1) + "Nanos"
          + interval + "s";
      opsLatQuantiles[i][j] = registry.newQuantiles(quantileName,
          "latency of Container ops", "ops", "latency", interval);
    }
  }
}
Example #11
Source File: IPCLoggerChannelMetrics.java    From hadoop with Apache License 2.0 5 votes vote down vote up 
public void addWriteEndToEndLatency(long micros) {
  if (writeEndToEndLatencyQuantiles != null) {
    for (MutableQuantiles q : writeEndToEndLatencyQuantiles) {
      q.add(micros);
    }
  }
}
Example #12
Source File: ContainerMetrics.java    From hadoop-ozone with Apache License 2.0 5 votes vote down vote up 
public void incContainerOpsLatencies(ContainerProtos.Type type,
                                     long latencyNanos) {
  opsLatency[type.ordinal()].add(latencyNanos);
  for (MutableQuantiles q: opsLatQuantiles[type.ordinal()]) {
    q.add(latencyNanos);
  }
}
Example #13
Source File: IPCLoggerChannelMetrics.java    From hadoop with Apache License 2.0 5 votes vote down vote up 
public void addWriteRpcLatency(long micros) {
  if (writeRpcLatencyQuantiles != null) {
    for (MutableQuantiles q : writeRpcLatencyQuantiles) {
      q.add(micros);
    }
  }
}
Example #14
Source File: JournalMetrics.java    From hadoop with Apache License 2.0 5 votes vote down vote up 
JournalMetrics(Journal journal) {
  this.journal = journal;
  
  syncsQuantiles = new MutableQuantiles[QUANTILE_INTERVALS.length];
  for (int i = 0; i < syncsQuantiles.length; i++) {
    int interval = QUANTILE_INTERVALS[i];
    syncsQuantiles[i] = registry.newQuantiles(
        "syncs" + interval + "s",
        "Journal sync time", "ops", "latencyMicros", interval);
  }
}
Example #15
Source File: UserGroupInformation.java    From hadoop with Apache License 2.0 5 votes vote down vote up 
void addGetGroups(long latency) {
  getGroups.add(latency);
  if (getGroupsQuantiles != null) {
    for (MutableQuantiles q : getGroupsQuantiles) {
      q.add(latency);
    }
  }
}
Example #16
Source File: RpcMetrics.java    From hadoop with Apache License 2.0 5 votes vote down vote up 
RpcMetrics(Server server, Configuration conf) {
  String port = String.valueOf(server.getListenerAddress().getPort());
  name = "RpcActivityForPort" + port;
  this.server = server;
  registry = new MetricsRegistry("rpc").tag("port", "RPC port", port);
  int[] intervals = conf.getInts(
      CommonConfigurationKeys.RPC_METRICS_PERCENTILES_INTERVALS_KEY);
  rpcQuantileEnable = (intervals.length > 0) && conf.getBoolean(
      CommonConfigurationKeys.RPC_METRICS_QUANTILE_ENABLE,
      CommonConfigurationKeys.RPC_METRICS_QUANTILE_ENABLE_DEFAULT);
  if (rpcQuantileEnable) {
    rpcQueueTimeMillisQuantiles =
        new MutableQuantiles[intervals.length];
    rpcProcessingTimeMillisQuantiles =
        new MutableQuantiles[intervals.length];
    for (int i = 0; i < intervals.length; i++) {
      int interval = intervals[i];
      rpcQueueTimeMillisQuantiles[i] = registry.newQuantiles("rpcQueueTime"
          + interval + "s", "rpc queue time in milli second", "ops",
          "latency", interval);
      rpcProcessingTimeMillisQuantiles[i] = registry.newQuantiles(
          "rpcProcessingTime" + interval + "s",
          "rpc processing time in milli second", "ops", "latency", interval);
    }
  }
  LOG.debug("Initialized " + registry);
}
Example #17
Source File: RpcMetrics.java    From hadoop with Apache License 2.0 5 votes vote down vote up 
/**
 * Add an RPC queue time sample
 * @param qTime the queue time
 */
//@Override
public void addRpcQueueTime(int qTime) {
  rpcQueueTime.add(qTime);
  if (rpcQuantileEnable) {
    for (MutableQuantiles q : rpcQueueTimeMillisQuantiles) {
      q.add(qTime);
    }
  }
}
Example #18
Source File: RpcMetrics.java    From hadoop with Apache License 2.0 5 votes vote down vote up 
/**
 * Add an RPC processing time sample
 * @param processingTime the processing time
 */
//@Override
public void addRpcProcessingTime(int processingTime) {
  rpcProcessingTime.add(processingTime);
  if (rpcQuantileEnable) {
    for (MutableQuantiles q : rpcProcessingTimeMillisQuantiles) {
      q.add(processingTime);
    }
  }
}
Example #19
Source File: MetricsAsserts.java    From hadoop with Apache License 2.0 5 votes vote down vote up 
/**
 * Asserts that the NumOps and quantiles for a metric have been changed at
 * some point to a non-zero value.
 * 
 * @param prefix of the metric
 * @param rb MetricsRecordBuilder with the metric
 */
public static void assertQuantileGauges(String prefix, 
    MetricsRecordBuilder rb) {
  verify(rb).addGauge(eqName(info(prefix + "NumOps", "")), geq(0l));
  for (Quantile q : MutableQuantiles.quantiles) {
    String nameTemplate = prefix + "%dthPercentileLatency";
    int percentile = (int) (100 * q.quantile);
    verify(rb).addGauge(
        eqName(info(String.format(nameTemplate, percentile), "")),
        geq(0l));
  }
}
Example #20
Source File: MetricsAsserts.java    From big-c with Apache License 2.0 5 votes vote down vote up 
/**
 * Asserts that the NumOps and quantiles for a metric have been changed at
 * some point to a non-zero value.
 * 
 * @param prefix of the metric
 * @param rb MetricsRecordBuilder with the metric
 */
public static void assertQuantileGauges(String prefix, 
    MetricsRecordBuilder rb) {
  verify(rb).addGauge(eqName(info(prefix + "NumOps", "")), geq(0l));
  for (Quantile q : MutableQuantiles.quantiles) {
    String nameTemplate = prefix + "%dthPercentileLatency";
    int percentile = (int) (100 * q.quantile);
    verify(rb).addGauge(
        eqName(info(String.format(nameTemplate, percentile), "")),
        geq(0l));
  }
}
Example #21
Source File: RpcMetrics.java    From big-c with Apache License 2.0 5 votes vote down vote up 
/**
 * Add an RPC processing time sample
 * @param processingTime the processing time
 */
//@Override
public void addRpcProcessingTime(int processingTime) {
  rpcProcessingTime.add(processingTime);
  if (rpcQuantileEnable) {
    for (MutableQuantiles q : rpcProcessingTimeMillisQuantiles) {
      q.add(processingTime);
    }
  }
}
Example #22
Source File: IPCLoggerChannelMetrics.java    From big-c with Apache License 2.0 5 votes vote down vote up 
public void addWriteEndToEndLatency(long micros) {
  if (writeEndToEndLatencyQuantiles != null) {
    for (MutableQuantiles q : writeEndToEndLatencyQuantiles) {
      q.add(micros);
    }
  }
}
Example #23
Source File: RpcMetrics.java    From big-c with Apache License 2.0 5 votes vote down vote up 
/**
 * Add an RPC queue time sample
 * @param qTime the queue time
 */
//@Override
public void addRpcQueueTime(int qTime) {
  rpcQueueTime.add(qTime);
  if (rpcQuantileEnable) {
    for (MutableQuantiles q : rpcQueueTimeMillisQuantiles) {
      q.add(qTime);
    }
  }
}
Example #24
Source File: IPCLoggerChannelMetrics.java    From big-c with Apache License 2.0 5 votes vote down vote up 
public void addWriteRpcLatency(long micros) {
  if (writeRpcLatencyQuantiles != null) {
    for (MutableQuantiles q : writeRpcLatencyQuantiles) {
      q.add(micros);
    }
  }
}
Example #25
Source File: JournalMetrics.java    From big-c with Apache License 2.0 5 votes vote down vote up 
JournalMetrics(Journal journal) {
  this.journal = journal;
  
  syncsQuantiles = new MutableQuantiles[QUANTILE_INTERVALS.length];
  for (int i = 0; i < syncsQuantiles.length; i++) {
    int interval = QUANTILE_INTERVALS[i];
    syncsQuantiles[i] = registry.newQuantiles(
        "syncs" + interval + "s",
        "Journal sync time", "ops", "latencyMicros", interval);
  }
}
Example #26
Source File: RpcMetrics.java    From big-c with Apache License 2.0 5 votes vote down vote up 
RpcMetrics(Server server, Configuration conf) {
  String port = String.valueOf(server.getListenerAddress().getPort());
  name = "RpcActivityForPort" + port;
  this.server = server;
  registry = new MetricsRegistry("rpc").tag("port", "RPC port", port);
  int[] intervals = conf.getInts(
      CommonConfigurationKeys.RPC_METRICS_PERCENTILES_INTERVALS_KEY);
  rpcQuantileEnable = (intervals.length > 0) && conf.getBoolean(
      CommonConfigurationKeys.RPC_METRICS_QUANTILE_ENABLE,
      CommonConfigurationKeys.RPC_METRICS_QUANTILE_ENABLE_DEFAULT);
  if (rpcQuantileEnable) {
    rpcQueueTimeMillisQuantiles =
        new MutableQuantiles[intervals.length];
    rpcProcessingTimeMillisQuantiles =
        new MutableQuantiles[intervals.length];
    for (int i = 0; i < intervals.length; i++) {
      int interval = intervals[i];
      rpcQueueTimeMillisQuantiles[i] = registry.newQuantiles("rpcQueueTime"
          + interval + "s", "rpc queue time in milli second", "ops",
          "latency", interval);
      rpcProcessingTimeMillisQuantiles[i] = registry.newQuantiles(
          "rpcProcessingTime" + interval + "s",
          "rpc processing time in milli second", "ops", "latency", interval);
    }
  }
  LOG.debug("Initialized " + registry);
}
Example #27
Source File: Nfs3Metrics.java    From hadoop with Apache License 2.0 4 votes vote down vote up 
public void addCommit(long latencyNanos) {
  commit.add(latencyNanos);
  for (MutableQuantiles q : commitNanosQuantiles) {
    q.add(latencyNanos);
  }
}
Example #28
Source File: NameNodeMetrics.java    From hadoop with Apache License 2.0 4 votes vote down vote up 
public void addBlockReport(long latency) {
  blockReport.add(latency);
  for (MutableQuantiles q : blockReportQuantiles) {
    q.add(latency);
  }
}
Example #29
Source File: NameNodeMetrics.java    From hadoop with Apache License 2.0 4 votes vote down vote up 
public void addSync(long elapsed) {
  syncs.add(elapsed);
  for (MutableQuantiles q : syncsQuantiles) {
    q.add(elapsed);
  }
}
Example #30
Source File: DataNodeMetrics.java    From big-c with Apache License 2.0 4 votes vote down vote up 
public void addRamDiskBlocksLazyPersistWindowMs(long latencyMs) {
  ramDiskBlocksLazyPersistWindowMs.add(latencyMs);
  for (MutableQuantiles q : ramDiskBlocksLazyPersistWindowMsQuantiles) {
    q.add(latencyMs);
  }
}