org.apache.hadoop.metrics2.MetricsCollector Java Examples

@Override
public void getMetrics(MetricsCollector collector, boolean all) {
  StartupProgressView prog = startupProgress.createView();
  MetricsRecordBuilder builder = collector.addRecord(
    STARTUP_PROGRESS_METRICS_INFO);

  builder.addCounter(info("ElapsedTime", "overall elapsed time"),
    prog.getElapsedTime());
  builder.addGauge(info("PercentComplete", "overall percent complete"),
    prog.getPercentComplete());

  for (Phase phase: prog.getPhases()) {
    addCounter(builder, phase, "Count", " count", prog.getCount(phase));
    addCounter(builder, phase, "ElapsedTime", " elapsed time",
      prog.getElapsedTime(phase));
    addCounter(builder, phase, "Total", " total", prog.getTotal(phase));
    addGauge(builder, phase, "PercentComplete", " percent complete",
      prog.getPercentComplete(phase));
  }
}
 

@Override
public void getMetrics(MetricsCollector metricsCollector, boolean all) {
  MetricsRecordBuilder record = metricsCollector.addRecord(metricsRegistry.info());
  if (wrapper != null) {
    // Summarize the tables
    Map<String,Entry<Long,Long>> tableUsages = wrapper.getTableSpaceUtilization();
    String tableSummary = "[]";
    if (tableUsages != null && !tableUsages.isEmpty()) {
      tableSummary = generateJsonQuotaSummary(tableUsages.entrySet(), "table");
    }
    record.tag(Interns.info(TABLE_QUOTA_USAGE_NAME, TABLE_QUOTA_USAGE_DESC), tableSummary);

    // Summarize the namespaces
    String nsSummary = "[]";
    Map<String,Entry<Long,Long>> namespaceUsages = wrapper.getNamespaceSpaceUtilization();
    if (namespaceUsages != null && !namespaceUsages.isEmpty()) {
      nsSummary = generateJsonQuotaSummary(namespaceUsages.entrySet(), "namespace");
    }
    record.tag(Interns.info(NS_QUOTA_USAGE_NAME, NS_QUOTA_USAGE_DESC), nsSummary);
  }
  metricsRegistry.snapshot(record, all);
}
 

@Override
public void getMetrics(MetricsCollector collector, boolean all) {
  counters.forEach((key, value) -> {
    MetricsRecordBuilder builder =
        collector.addRecord(name);
    builder.add(
        new MetricsTag(Interns.info("type", "Message type"), key.toString()));
    builder.addCounter(new MetricName("counter", "Number of distinct calls"),
        value.longValue());
    builder.addCounter(
        new MetricName("time", "Sum of the duration of the calls"),
        elapsedTimes.get(key).longValue());
    builder.endRecord();

  });
}
 

@Override
public void getMetrics(MetricsCollector metricsCollector, boolean all) {
  MetricsRecordBuilder metricsRecordBuilder = metricsCollector.addRecord(metricsName);

  // masterWrapper can be null because this function is called inside of init.
  if (masterWrapper != null) {
    metricsRecordBuilder
        .addGauge(Interns.info(NUM_MASTER_WALS_NAME, NUM_MASTER_WALS_DESC),
            masterWrapper.getNumWALFiles());
  }

  metricsRegistry.snapshot(metricsRecordBuilder, all);
  if(metricsAdapter != null) {
    metricsAdapter.snapshotAllMetrics(registry, metricsRecordBuilder);
  }
}
 

@Test
public void testCounterReporting() {
    final Counter counter = new Counter();

    TreeMap<String, Counter> counters = new TreeMap<>();
    counters.put("my_counter", counter);
    // Add the metrics objects to the internal "queues" by hand
    metrics2Reporter.setDropwizardCounters(counters);

    // Set some values
    counter.inc(5L);

    MetricsCollector collector = mock(MetricsCollector.class);
    MetricsRecordBuilder recordBuilder = mock(MetricsRecordBuilder.class);

    Mockito.when(collector.addRecord(recordName)).thenReturn(recordBuilder);

    metrics2Reporter.getMetrics(collector, true);

    verify(recordBuilder).addCounter(Interns.info("my_counter", ""), 5L);
    verifyRecordBuilderUnits(recordBuilder);

    // Should not be the same instance we gave before. Our map should have gotten swapped out.
    assertTrue("Should not be the same map instance after collection",
            counters != metrics2Reporter.getDropwizardCounters());
}
 

@Override
public void getMetrics(MetricsCollector metricsCollector, boolean all) {
  MetricsRecordBuilder metricsRecordBuilder = metricsCollector.addRecord(metricsName);

  if (stochasticCosts != null) {
    synchronized (stochasticCosts) {
      for (Map.Entry<String, Map<String, Double>> tableEntry : stochasticCosts.entrySet()) {
        for (Map.Entry<String, Double> costEntry : tableEntry.getValue().entrySet()) {
          String attrName = tableEntry.getKey() + TABLE_FUNCTION_SEP + costEntry.getKey();
          Double cost = costEntry.getValue();
          String functionDesc = costFunctionDescs.get(costEntry.getKey());

          if (functionDesc == null) {
            functionDesc = costEntry.getKey();
          }

          metricsRecordBuilder.addGauge(Interns.info(attrName, functionDesc), cost);
        }
      }
    }
  }
  metricsRegistry.snapshot(metricsRecordBuilder, all);
}
 

@Override
@SuppressWarnings("SuspiciousMethodCalls")
public void getMetrics(MetricsCollector collector, boolean all) {
  Map<String, Integer> stateCount = scmmxBean.getContainerStateCount();

  collector.addRecord(SOURCE)
      .addGauge(Interns.info("OpenContainers",
          "Number of open containers"),
          stateCount.get(OPEN.toString()))
      .addGauge(Interns.info("ClosingContainers",
          "Number of containers in closing state"),
          stateCount.get(CLOSING.toString()))
      .addGauge(Interns.info("QuasiClosedContainers",
          "Number of containers in quasi closed state"),
          stateCount.get(QUASI_CLOSED.toString()))
      .addGauge(Interns.info("ClosedContainers",
          "Number of containers in closed state"),
          stateCount.get(CLOSED.toString()))
      .addGauge(Interns.info("DeletingContainers",
          "Number of containers in deleting state"),
          stateCount.get(DELETING.toString()))
      .addGauge(Interns.info("DeletedContainers",
          "Number of containers in deleted state"),
          stateCount.get(DELETED.toString()));
}
 

@SuppressWarnings("rawtypes")
@Test
public void metrics2CycleIsNonDestructive() {
    metrics2Reporter.setDropwizardCounters(Collections.unmodifiableSortedMap(new TreeMap<String, Counter>()));
    metrics2Reporter.setDropwizardGauges(Collections.unmodifiableSortedMap(new TreeMap<String, Gauge>()));
    metrics2Reporter.setDropwizardHistograms(Collections.unmodifiableSortedMap(new TreeMap<String, Histogram>()));
    metrics2Reporter.setDropwizardMeters(Collections.unmodifiableSortedMap(new TreeMap<String, Meter>()));
    metrics2Reporter.setDropwizardTimers(Collections.unmodifiableSortedMap(new TreeMap<String, Timer>()));

    MetricsCollector collector = mock(MetricsCollector.class);
    MetricsRecordBuilder recordBuilder = mock(MetricsRecordBuilder.class);

    Mockito.when(collector.addRecord(recordName)).thenReturn(recordBuilder);

    metrics2Reporter.getMetrics(collector, true);
}
 

@SuppressWarnings("rawtypes")
@Test
public void metrics2CycleIsNonDestructive() {
    metrics2Reporter.setDropwizardCounters(Collections.unmodifiableSortedMap(new TreeMap<String, Counter>()));
    metrics2Reporter.setDropwizardGauges(Collections.unmodifiableSortedMap(new TreeMap<String, Gauge>()));
    metrics2Reporter.setDropwizardHistograms(Collections.unmodifiableSortedMap(new TreeMap<String, Histogram>()));
    metrics2Reporter.setDropwizardMeters(Collections.unmodifiableSortedMap(new TreeMap<String, Meter>()));
    metrics2Reporter.setDropwizardTimers(Collections.unmodifiableSortedMap(new TreeMap<String, Timer>()));

    MetricsCollector collector = mock(MetricsCollector.class);
    MetricsRecordBuilder recordBuilder = mock(MetricsRecordBuilder.class);

    Mockito.when(collector.addRecord(recordName)).thenReturn(recordBuilder);

    metrics2Reporter.getMetrics(collector, true);
}
 

@Override
public synchronized void getMetrics(MetricsCollector collector, boolean all) {
  //Container goes through registered -> finished -> unregistered.
  if (unregister) {
    metricsSystem.unregisterSource(recordInfo.name());
    usageMetrics.remove(containerId);
    return;
  }

  if (finished || flushOnPeriod) {
    registry.snapshot(collector.addRecord(registry.info()), all);
  }

  if (finished) {
    this.unregister = true;
  } else if (flushOnPeriod) {
    flushOnPeriod = false;
    scheduleTimerTaskIfRequired();
  }
}
 

@Test
public void testCounterReporting() {
    final Counter counter = new Counter();

    TreeMap<String, Counter> counters = new TreeMap<>();
    counters.put("my_counter", counter);
    // Add the metrics objects to the internal "queues" by hand
    metrics2Reporter.setDropwizardCounters(counters);

    // Set some values
    counter.inc(5L);

    MetricsCollector collector = mock(MetricsCollector.class);
    MetricsRecordBuilder recordBuilder = mock(MetricsRecordBuilder.class);

    Mockito.when(collector.addRecord(recordName)).thenReturn(recordBuilder);

    metrics2Reporter.getMetrics(collector, true);

    verify(recordBuilder).addCounter(Interns.info("my_counter", ""), 5L);
    verifyRecordBuilderUnits(recordBuilder);

    // Should not be the same instance we gave before. Our map should have gotten swapped out.
    assertTrue("Should not be the same map instance after collection",
            counters != metrics2Reporter.getDropwizardCounters());
}
 

@Override
public void getMetrics(MetricsCollector collector, boolean all) {
  StartupProgressView prog = startupProgress.createView();
  MetricsRecordBuilder builder = collector.addRecord(
    STARTUP_PROGRESS_METRICS_INFO);

  builder.addCounter(info("ElapsedTime", "overall elapsed time"),
    prog.getElapsedTime());
  builder.addGauge(info("PercentComplete", "overall percent complete"),
    prog.getPercentComplete());

  for (Phase phase: prog.getPhases()) {
    addCounter(builder, phase, "Count", " count", prog.getCount(phase));
    addCounter(builder, phase, "ElapsedTime", " elapsed time",
      prog.getElapsedTime(phase));
    addCounter(builder, phase, "Total", " total", prog.getTotal(phase));
    addGauge(builder, phase, "PercentComplete", " percent complete",
      prog.getPercentComplete(phase));
  }
}
 

public MetricsSource build() {
  if (source instanceof MetricsSource) {
    if (hasAtMetric && !hasRegistry) {
      throw new MetricsException("Hybrid metrics: registry required.");
    }
    return (MetricsSource) source;
  }
  else if (!hasAtMetric) {
    throw new MetricsException("No valid @Metric annotation found.");
  }
  return new MetricsSource() {
    @Override
    public void getMetrics(MetricsCollector builder, boolean all) {
      registry.snapshot(builder.addRecord(registry.info()), all);
    }
  };
}
 

public static MetricsRecordBuilder mockMetricsRecordBuilder() {
  final MetricsCollector mc = mock(MetricsCollector.class);
  MetricsRecordBuilder rb = mock(MetricsRecordBuilder.class,
      new Answer<Object>() {
    @Override
    public Object answer(InvocationOnMock invocation) {
      Object[] args = invocation.getArguments();
      StringBuilder sb = new StringBuilder();
      for (Object o : args) {
        if (sb.length() > 0) sb.append(", ");
        sb.append(String.valueOf(o));
      }
      String methodName = invocation.getMethod().getName();
      LOG.debug(methodName +": "+ sb);
      return methodName.equals("parent") || methodName.equals("endRecord") ?
             mc : invocation.getMock();
    }
  });
  when(mc.addRecord(anyString())).thenReturn(rb);
  when(mc.addRecord(anyInfo())).thenReturn(rb);
  return rb;
}
 

@Test public void testHybrid() {
  HybridMetrics metrics = new HybridMetrics();
  MetricsSource source = MetricsAnnotations.makeSource(metrics);

  assertSame(metrics, source);
  metrics.C0.incr();
  MetricsRecordBuilder rb = getMetrics(source);
  MetricsCollector collector = rb.parent();

  verify(collector).addRecord("foo");
  verify(collector).addRecord("bar");
  verify(collector).addRecord(info("HybridMetrics", "HybridMetrics"));
  verify(rb).setContext("foocontext");
  verify(rb).addCounter(info("C1", "C1 desc"), 1);
  verify(rb).setContext("barcontext");
  verify(rb).addGauge(info("G1", "G1 desc"), 1);
  verify(rb).add(tag(MsInfo.Context, "hybrid"));
  verify(rb).addCounter(info("C0", "C0 desc"), 1);
  verify(rb).addGauge(info("G0", "G0"), 0);
}
 

@Test public void testPresence() {
  JvmPauseMonitor pauseMonitor = new JvmPauseMonitor(new Configuration());
  JvmMetrics jvmMetrics = new JvmMetrics("test", "test");
  jvmMetrics.setPauseMonitor(pauseMonitor);
  MetricsRecordBuilder rb = getMetrics(jvmMetrics);
  MetricsCollector mc = rb.parent();

  verify(mc).addRecord(JvmMetrics);
  verify(rb).tag(ProcessName, "test");
  verify(rb).tag(SessionId, "test");
  for (JvmMetricsInfo info : JvmMetricsInfo.values()) {
    if (info.name().startsWith("Mem"))
      verify(rb).addGauge(eq(info), anyFloat());
    else if (info.name().startsWith("Gc"))
      verify(rb).addCounter(eq(info), anyLong());
    else if (info.name().startsWith("Threads"))
      verify(rb).addGauge(eq(info), anyInt());
    else if (info.name().startsWith("Log"))
      verify(rb).addCounter(eq(info), anyLong());
  }
}
 

public static MetricsRecordBuilder mockMetricsRecordBuilder() {
  final MetricsCollector mc = mock(MetricsCollector.class);
  MetricsRecordBuilder rb = mock(MetricsRecordBuilder.class,
      new Answer<Object>() {
    @Override
    public Object answer(InvocationOnMock invocation) {
      Object[] args = invocation.getArguments();
      StringBuilder sb = new StringBuilder();
      for (Object o : args) {
        if (sb.length() > 0) sb.append(", ");
        sb.append(String.valueOf(o));
      }
      String methodName = invocation.getMethod().getName();
      LOG.debug(methodName +": "+ sb);
      return methodName.equals("parent") || methodName.equals("endRecord") ?
             mc : invocation.getMock();
    }
  });
  when(mc.addRecord(anyString())).thenReturn(rb);
  when(mc.addRecord(anyInfo())).thenReturn(rb);
  return rb;
}
 

public MetricsSource build() {
  if (source instanceof MetricsSource) {
    if (hasAtMetric && !hasRegistry) {
      throw new MetricsException("Hybrid metrics: registry required.");
    }
    return (MetricsSource) source;
  }
  else if (!hasAtMetric) {
    throw new MetricsException("No valid @Metric annotation found.");
  }
  return new MetricsSource() {
    @Override
    public void getMetrics(MetricsCollector builder, boolean all) {
      registry.snapshot(builder.addRecord(registry.info()), all);
    }
  };
}
 

@Override
public synchronized void getMetrics(MetricsCollector collector, boolean all) {
  //Container goes through registered -> finished -> unregistered.
  if (unregister) {
    metricsSystem.unregisterSource(recordInfo.name());
    usageMetrics.remove(containerId);
    return;
  }

  if (finished || flushOnPeriod) {
    registry.snapshot(collector.addRecord(registry.info()), all);
  }

  if (finished) {
    this.unregister = true;
  } else if (flushOnPeriod) {
    flushOnPeriod = false;
    scheduleTimerTaskIfRequired();
  }
}
 

@Override
@SuppressWarnings("SuspiciousMethodCalls")
public void getMetrics(MetricsCollector collector, boolean all) {
  MetricsRecordBuilder recordBuilder = collector.addRecord(SOURCE_NAME);
  numPipelineAllocated.snapshot(recordBuilder, true);
  numPipelineCreated.snapshot(recordBuilder, true);
  numPipelineCreationFailed.snapshot(recordBuilder, true);
  numPipelineDestroyed.snapshot(recordBuilder, true);
  numPipelineDestroyFailed.snapshot(recordBuilder, true);
  numPipelineReportProcessed.snapshot(recordBuilder, true);
  numPipelineReportProcessingFailed.snapshot(recordBuilder, true);
  numPipelineContainSameDatanodes.snapshot(recordBuilder, true);
  numBytesWritten
      .forEach((pid, metric) -> metric.snapshot(recordBuilder, true));
  numBlocksAllocated
      .forEach((pid, metric) -> metric.snapshot(recordBuilder, true));
}
 

@Override
public void getMetrics(MetricsCollector metricsCollector, boolean all) {
  MetricsRecordBuilder mrb = metricsCollector.addRecord(metricsName);
  // source is registered in supers constructor, sometimes called before the whole initialization.
  metricsRegistry.snapshot(mrb, all);
  if (metricsAdapter != null) {
    // snapshot MetricRegistry as well
    metricsAdapter.snapshotAllMetrics(registry, mrb);
  }
}
 

@Override
public void getMetrics(MetricsCollector builder, boolean all) {
  averageBlockDownloadLatencyMs.set(
      currentBlockDownloadLatency.getCurrentAverage());
  averageBlockUploadLatencyMs.set(
      currentBlockUploadLatency.getCurrentAverage());
  registry.snapshot(builder.addRecord(registry.info().name()), true);
}
 

@SuppressWarnings("rawtypes")
@Test
public void cachedMetricsAreClearedAfterCycle() {
    // After we perform a metrics2 reporting cycle, the maps should be reset to avoid double-reporting
    TreeMap<String, Counter> counters = new TreeMap<>();
    TreeMap<String, Gauge> gauges = new TreeMap<>();
    TreeMap<String, Histogram> histograms = new TreeMap<>();
    TreeMap<String, Meter> meters = new TreeMap<>();
    TreeMap<String, Timer> timers = new TreeMap<>();

    metrics2Reporter.setDropwizardCounters(counters);
    metrics2Reporter.setDropwizardGauges(gauges);
    metrics2Reporter.setDropwizardHistograms(histograms);
    metrics2Reporter.setDropwizardMeters(meters);
    metrics2Reporter.setDropwizardTimers(timers);

    MetricsCollector collector = mock(MetricsCollector.class);
    MetricsRecordBuilder recordBuilder = mock(MetricsRecordBuilder.class);

    Mockito.when(collector.addRecord(recordName)).thenReturn(recordBuilder);

    metrics2Reporter.getMetrics(collector, true);

    assertTrue(counters != metrics2Reporter.getDropwizardCounters());
    assertEquals(0, metrics2Reporter.getDropwizardCounters().size());
    assertTrue(gauges != metrics2Reporter.getDropwizardGauges());
    assertEquals(0, metrics2Reporter.getDropwizardGauges().size());
    assertTrue(histograms != metrics2Reporter.getDropwizardHistograms());
    assertEquals(0, metrics2Reporter.getDropwizardHistograms().size());
    assertTrue(meters != metrics2Reporter.getDropwizardMeters());
    assertEquals(0, metrics2Reporter.getDropwizardMeters().size());
    assertTrue(timers != metrics2Reporter.getDropwizardTimers());
    assertEquals(0, metrics2Reporter.getDropwizardTimers().size());
}
 

@Override
public void getMetrics(MetricsCollector collector, boolean all) {
  MetricsRecordBuilder rb = collector.addRecord(JvmMetrics)
      .setContext("jvm").tag(ProcessName, processName)
      .tag(SessionId, sessionId);
  getMemoryUsage(rb);
  getGcUsage(rb);
  getThreadUsage(rb);
  getEventCounters(rb);
}
 

@Override
public void getMetrics(MetricsCollector metricsCollector, boolean all) {
  MetricsRecordBuilder mrb = metricsCollector.addRecord(metricsName);

  // wrapper can be null because this function is called inside of init.
  if (wrapper != null) {
    mrb.addCounter(Interns.info(CHECKSUM_FAILURES_KEY, CHECKSUM_FAILURES_DESC),
      wrapper.getChecksumFailures());
  }

  metricsRegistry.snapshot(mrb, all);
}
 

/**
 * Yes this is a get function that doesn't return anything.  Thanks Hadoop for breaking all
 * expectations of java programmers.  Instead of returning anything Hadoop metrics expects
 * getMetrics to push the metrics into the collector.
 *
 * @param collector the collector
 * @param all       get all the metrics regardless of when they last changed.
 */
@Override
public void getMetrics(MetricsCollector collector, boolean all) {
  MetricsRecordBuilder mrb = collector.addRecord(metricsName);

  if (regionSources != null) {
    for (MetricsRegionSource regionMetricSource : regionSources) {
      if (regionMetricSource instanceof MetricsRegionSourceImpl) {
        ((MetricsRegionSourceImpl) regionMetricSource).snapshot(mrb, all);
      }
    }
    mrb.addGauge(Interns.info(NUM_REGIONS, NUMBER_OF_REGIONS_DESC), regionSources.size());
    metricsRegistry.snapshot(mrb, all);
  }
}
 

@Override
public synchronized void getMetrics(MetricsCollector builder, boolean all) {
  MetricsRecordBuilder rb = builder.addRecord(MS_NAME)
      .addGauge(MsInfo.NumActiveSources, sources.size())
      .addGauge(MsInfo.NumAllSources, allSources.size())
      .addGauge(MsInfo.NumActiveSinks, sinks.size())
      .addGauge(MsInfo.NumAllSinks, allSinks.size());

  for (MetricsSinkAdapter sa : sinks.values()) {
    sa.snapshot(rb, all);
  }
  registry.snapshot(rb, all);
}
 

MetricsRecordBuilderImpl(MetricsCollector parent, MetricsInfo info,
                         MetricsFilter rf, MetricsFilter mf,
                         boolean acceptable) {
  this.parent = parent;
  timestamp = Time.now();
  recInfo = info;
  metrics = Lists.newArrayList();
  tags = Lists.newArrayList();
  recordFilter = rf;
  metricFilter = mf;
  this.acceptable = acceptable;
}
 

/**
 * Yes this is a get function that doesn't return anything.  Thanks Hadoop for breaking all
 * expectations of java programmers.  Instead of returning anything Hadoop metrics expects
 * getMetrics to push the metrics into the collector.
 *
 * @param collector the collector
 * @param all       get all the metrics regardless of when they last changed.
 */
@Override
public void getMetrics(MetricsCollector collector, boolean all) {
  MetricsRecordBuilder mrb = collector.addRecord(metricsName);
  if (tableSources != null) {
    for (MetricsTableSource tableMetricSource : tableSources.values()) {
      if (tableMetricSource instanceof MetricsTableSourceImpl) {
        ((MetricsTableSourceImpl) tableMetricSource).snapshot(mrb, all);
      }
    }
    mrb.addGauge(Interns.info(NUM_TABLES, NUMBER_OF_TABLES_DESC), tableSources.size());
    metricsRegistry.snapshot(mrb, all);
  }
}
 

@Override
public void getMetrics(MetricsCollector collector, boolean all) {
  MetricsRecordBuilder mrb = collector.addRecord(metricsName);

  if (userSources != null) {
    for (MetricsUserSource userMetricSource : userSources.values()) {
      if (userMetricSource instanceof MetricsUserSourceImpl) {
        ((MetricsUserSourceImpl) userMetricSource).snapshot(mrb, all);
      }
    }
    mrb.addGauge(Interns.info(NUM_USERS, NUMBER_OF_USERS_DESC), userSources.size());
    metricsRegistry.snapshot(mrb, all);
  }
}