org.apache.kafka.common.Metric Java Examples

@Test
void shouldKeepMetersWhenMetricsDoNotChange() {
    //Given
    Supplier<Map<MetricName, ? extends Metric>> supplier = () -> {
        MetricName metricName = new MetricName("a", "b", "c", new LinkedHashMap<>());
        KafkaMetric metric = new KafkaMetric(this, metricName, new Value(), new MetricConfig(), Time.SYSTEM);
        return Collections.singletonMap(metricName, metric);
    };
    kafkaMetrics = new KafkaMetrics(supplier);
    MeterRegistry registry = new SimpleMeterRegistry();

    kafkaMetrics.bindTo(registry);
    assertThat(registry.getMeters()).hasSize(1);

    kafkaMetrics.checkAndBindMetrics(registry);
    assertThat(registry.getMeters()).hasSize(1);
}
 

/**
 * kafka doesnt have an API for getting the client id from a client (WTH?!)
 * relying on reflection is tricky because we may be dealing with various
 * wrappers/decorators, but it does leak through kafka's metrics tags ...
 * @param metrics kafka client metrics
 * @return best guess for the client id
 */
private static String fishForClientId(Map<MetricName, ? extends Metric> metrics) {
  Set<String> candidates = new HashSet<>();
  metrics.forEach((metricName, metric) -> {
    Map<String, String> tags = metricName.tags();
    if (tags == null) {
      return;
    }
    String clientId = tags.get("client-id");
    if (clientId != null) {
      candidates.add(clientId);
    }
  });
  if (candidates.isEmpty()) {
    return null;
  }
  if (candidates.size() > 1) {
    throw new IllegalArgumentException("ambiguous client id from client: " + candidates);
  }
  return candidates.iterator().next();
}
 

/**
 * Define common tags and meters before binding metrics
 */
void prepareToBindMetrics(MeterRegistry registry) {
    Map<MetricName, ? extends Metric> metrics = metricsSupplier.get();
    // Collect static metrics and tags
    Metric startTime = null;

    for (Map.Entry<MetricName, ? extends Metric> entry : metrics.entrySet()) {
        MetricName name = entry.getKey();
        if (clientId.equals(DEFAULT_VALUE) && name.tags().get(CLIENT_ID_TAG_NAME) != null)
            clientId = name.tags().get(CLIENT_ID_TAG_NAME);
        if (METRIC_GROUP_APP_INFO.equals(name.group()))
            if (VERSION_METRIC_NAME.equals(name.name())) {
                kafkaVersion = (String) entry.getValue().metricValue();
            } else if (START_TIME_METRIC_NAME.equals(name.name())) {
                startTime = entry.getValue();
            }
    }

    if (startTime != null) {
        bindMeter(registry, startTime, meterName(startTime), meterTags(startTime));
    }
}
 

@Test
public final void sendDoubleGauge() throws Exception {
  final double value = 10.11;
  Metric metric = new Metric() {
    @Override
    public MetricName metricName() {
      return new MetricName("test-metric", "group");
    }

    @Override
    public double value() {
      return value;
    }
  };

  addMetricAndRunReporter("foo", metric, "bar");
  verify(statsD).gauge(Matchers.eq("foo"), Matchers.eq(value), Matchers.eq("bar"));
}
 

@Test
void shouldRemoveOlderMeterWithLessTags() {
    Map<String, String> tags = new LinkedHashMap<>();
    Supplier<Map<MetricName, ? extends Metric>> supplier = () -> {
        MetricName metricName = new MetricName("a", "b", "c", tags);
        KafkaMetric metric = new KafkaMetric(this, metricName, new Value(), new MetricConfig(), Time.SYSTEM);
        return Collections.singletonMap(metricName, metric);
    };
    kafkaMetrics = new KafkaMetrics(supplier);
    MeterRegistry registry = new SimpleMeterRegistry();

    kafkaMetrics.bindTo(registry);
    assertThat(registry.getMeters()).hasSize(1);
    assertThat(registry.getMeters().get(0).getId().getTags()).hasSize(1); //only version

    tags.put("key0", "value0");
    kafkaMetrics.checkAndBindMetrics(registry);
    assertThat(registry.getMeters()).hasSize(1);
    assertThat(registry.getMeters().get(0).getId().getTags()).hasSize(2);
}
 

@Test
void shouldRemoveMeterWithLessTags() {
    Supplier<Map<MetricName, ? extends Metric>> supplier = () -> {
        MetricName firstName = new MetricName("a", "b", "c", Collections.emptyMap());
        KafkaMetric firstMetric = new KafkaMetric(this, firstName, new Value(), new MetricConfig(), Time.SYSTEM);
        Map<String, String> tags = new LinkedHashMap<>();
        tags.put("key0", "value0");
        MetricName secondName = new MetricName("a", "b", "c", tags);
        KafkaMetric secondMetric = new KafkaMetric(this, secondName, new Value(), new MetricConfig(), Time.SYSTEM);
        Map<MetricName, KafkaMetric> metrics = new LinkedHashMap<>();
        metrics.put(firstName, firstMetric);
        metrics.put(secondName, secondMetric);
        return metrics;
    };
    kafkaMetrics = new KafkaMetrics(supplier);
    MeterRegistry registry = new SimpleMeterRegistry();

    kafkaMetrics.bindTo(registry);
    assertThat(registry.getMeters()).hasSize(1);
    assertThat(registry.getMeters().get(0).getId().getTags()).hasSize(2); // version + key0
}
 

@Test
void shouldBindMetersWithSameTags() {
    Supplier<Map<MetricName, ? extends Metric>> supplier = () -> {
        Map<String, String> firstTags = new LinkedHashMap<>();
        firstTags.put("key0", "value0");
        MetricName firstName = new MetricName("a", "b", "c", firstTags);
        KafkaMetric firstMetric = new KafkaMetric(this, firstName, new Value(), new MetricConfig(), Time.SYSTEM);
        Map<String, String> secondTags = new LinkedHashMap<>();
        secondTags.put("key0", "value1");
        MetricName secondName = new MetricName("a", "b", "c", secondTags);
        KafkaMetric secondMetric = new KafkaMetric(this, secondName, new Value(), new MetricConfig(), Time.SYSTEM);

        Map<MetricName, KafkaMetric> metrics = new LinkedHashMap<>();
        metrics.put(firstName, firstMetric);
        metrics.put(secondName, secondMetric);
        return metrics;
    };

    kafkaMetrics = new KafkaMetrics(supplier);
    MeterRegistry registry = new SimpleMeterRegistry();

    kafkaMetrics.bindTo(registry);
    assertThat(registry.getMeters()).hasSize(2);
    assertThat(registry.getMeters().get(0).getId().getTags()).hasSize(2); // version + key0
}
 

@Issue("#1968")
@Test
void shouldRemoveOlderMeterWithLessTagsWhenCommonTagsConfigured() {
    //Given
    Map<String, String> tags = new LinkedHashMap<>();
    Supplier<Map<MetricName, ? extends Metric>> supplier = () -> {
        MetricName metricName = new MetricName("a", "b", "c", tags);
        KafkaMetric metric = new KafkaMetric(this, metricName, new Value(), new MetricConfig(), Time.SYSTEM);
        return Collections.singletonMap(metricName, metric);
    };

    kafkaMetrics = new KafkaMetrics(supplier);
    MeterRegistry registry = new SimpleMeterRegistry();
    registry.config().commonTags("common", "value");

    kafkaMetrics.bindTo(registry);
    assertThat(registry.getMeters()).hasSize(1);
    assertThat(registry.getMeters().get(0).getId().getTags()).containsExactlyInAnyOrder(Tag.of("kafka-version", "unknown"), Tag.of("common", "value")); // only version

    tags.put("key0", "value0");
    kafkaMetrics.checkAndBindMetrics(registry);
    assertThat(registry.getMeters()).hasSize(1);
    assertThat(registry.getMeters().get(0).getId().getTags()).containsExactlyInAnyOrder(Tag.of("kafka-version", "unknown"), Tag.of("key0", "value0"), Tag.of("common", "value"));
}
 

private synchronized void checkAssignments() {
    logger.debug("Checking partition assignments");

    try {
        KafkaConsumer<String, String> consumer = consumers.iterator().next();
        Map<MetricName, ? extends Metric> metrics = consumer.metrics();
        for (MetricName name : metrics.keySet()) {
            if ("assigned-partitions".equals(name.name())) {
                Metric metric = metrics.get(name);
                Map<String, String> tags = name.tags();
                String clientId = tags.get("client-id");
                int partitionCount = ((Double)metric.metricValue()).intValue();
                processDataPoint(clientId, partitionCount, Instant.now());
            }
        }
    } catch (NoSuchElementException ex) {
    }
}
 

@SuppressWarnings({ "deprecation" })
protected void publishKafkaProducerMetricsToOstrich() {
  Map<KafkaProducerConfig, KafkaProducer<byte[], byte[]>> producers = KafkaProducerManager
      .getInstance().getProducers();
  for (Entry<KafkaProducerConfig, KafkaProducer<byte[], byte[]>> kafkaProducerEntry : producers
      .entrySet()) {
    KafkaProducerConfig key = kafkaProducerEntry.getKey();
    String signature = convertSignatureToTag(key);
    Map<MetricName, ? extends Metric> metrics = kafkaProducerEntry.getValue().metrics();
    for (Entry<MetricName, ? extends Metric> entry : metrics.entrySet()) {
      if (PRODUCER_METRICS_WHITELIST.contains(entry.getKey().name())) {
        OpenTsdbMetricConverter.gauge("kafkaproducer." + entry.getKey().name(),
            entry.getValue().value(), "cluster=" + signature);
      }
    }
  }
}
 

void add(Map<MetricName, ? extends Metric> kafkaMetrics) {
    for (var entry : kafkaMetrics.entrySet()) {
        MetricName name = entry.getKey();
        if ("consumer-fetch-manager-metrics".equals(name.group())) {
            if ("records-lag-max".equals(name.name())) recordsLagMax.add(entry.getValue());
            else if ("records-consumed-rate".equals(name.name())) recordsConsumedRate.add(entry.getValue());
            else if ("bytes-consumed-rate".equals(name.name())) bytesConsumedRate.add(entry.getValue());
            else if ("fetch-rate".equals(name.name())) fetchRate.add(entry.getValue());
        }
    }
}
 

public void set(Map<MetricName, ? extends Metric> kafkaMetrics) {
    for (Map.Entry<MetricName, ? extends Metric> entry : kafkaMetrics.entrySet()) {
        MetricName metricName = entry.getKey();
        if ("producer-metrics".equals(metricName.group())) {
            String name = metricName.name();
            if ("request-rate".equals(name)) requestRate = entry.getValue();
            else if ("request-size-avg".equals(name)) requestSizeAvg = entry.getValue();
            else if ("outgoing-byte-rate".equals(name)) outgoingByteRate = entry.getValue();
        }
    }
}
 

@Test
void sum() {
    Metric metric1 = mock(Metric.class);
    when(metric1.metricValue()).thenReturn(1d);
    Metric metric2 = mock(Metric.class);
    when(metric2.metricValue()).thenReturn(Double.NEGATIVE_INFINITY);

    var metrics = new ConsumerMetrics(null);
    assertThat(metrics.sum(List.of(metric1, metric1, metric2))).isEqualTo(2);
}
 

@BeforeEach
void createProducerMetrics() {
    requestSizeAvg = mock(Metric.class);

    metrics = new ProducerMetrics(null);
    metrics.requestSizeAvg = requestSizeAvg;
}
 

void updateMetrics(String[] clusters, Map<String, Map<MetricName, ? extends Metric>> metricsMap)
{
  long current = System.currentTimeMillis();
  if (current - lastMetricSampleTime < metricsRefreshInterval) {
    return;
  }

  lastMetricSampleTime = current;

  if (stats == null) {
    stats = new KafkaConsumerStats[clusters.length];
  }

  for (int i = 0; i < clusters.length; i++) {
    if (stats[i] == null) {
      stats[i] = new KafkaConsumerStats();
      stats[i].cluster = clusters[i];
    }
    Map<MetricName, ? extends Metric> cMetrics = metricsMap.get(clusters[i]);
    if (cMetrics == null || cMetrics.isEmpty()) {
      stats[i].bytesPerSec = 0;
      stats[i].msgsPerSec = 0;
      continue;
    }
    if (stats[i].bytePerSecMK == null || stats[i].msgPerSecMK == null) {
      for (MetricName mn : cMetrics.keySet()) {
        if (mn.name().equals("bytes-consumed-rate")) {
          stats[i].bytePerSecMK = mn;
        } else if (mn.name().equals("records-consumed-rate")) {
          stats[i].msgPerSecMK = mn;
        }
      }
    }
    stats[i].bytesPerSec = cMetrics.get(stats[i].bytePerSecMK).value();
    stats[i].msgsPerSec = cMetrics.get(stats[i].msgPerSecMK).value();
  }
}
 

@Test
void shouldNotAddAppInfoMetrics() {
    Supplier<Map<MetricName, ? extends Metric>> supplier = () -> {
        Map<MetricName, KafkaMetric> metrics = new LinkedHashMap<>();
        MetricName metricName = new MetricName("a0", "b0", "c0", new LinkedHashMap<>());
        KafkaMetric metric = new KafkaMetric(this, metricName, new Value(), new MetricConfig(), Time.SYSTEM);
        metrics.put(metricName, metric);
        MetricName appInfoMetricName =
                new MetricName("a1", KafkaMetrics.METRIC_GROUP_APP_INFO, "c0",
                        new LinkedHashMap<>());
        KafkaMetric appInfoMetric =
                new KafkaMetric(this, appInfoMetricName, new Value(), new MetricConfig(), Time.SYSTEM);
        metrics.put(appInfoMetricName, appInfoMetric);
        return metrics;
    };
    kafkaMetrics = new KafkaMetrics(supplier);
    MeterRegistry registry = new SimpleMeterRegistry();

    kafkaMetrics.bindTo(registry);
    assertThat(registry.getMeters()).hasSize(1);

    kafkaMetrics.checkAndBindMetrics(registry);
    assertThat(registry.getMeters()).hasSize(1);
}
 

private void bindMeter(MeterRegistry registry, Metric metric, String name, Iterable<Tag> tags) {
    if (name.endsWith("total") || name.endsWith("count")) {
        registerCounter(registry, metric, name, tags);
    } else {
        registerGauge(registry, metric, name, tags);
    }
}
 

private List<Tag> meterTags(Metric metric, boolean includeCommonTags) {
    List<Tag> tags = new ArrayList<>();
    metric.metricName().tags().forEach((key, value) -> tags.add(Tag.of(key, value)));
    tags.add(Tag.of(KAFKA_VERSION_TAG_NAME, kafkaVersion));
    extraTags.forEach(tags::add);
    if (includeCommonTags) {
        commonTags.forEach(tags::add);
    }
    return tags;
}
 

public <T extends com.codahale.metrics.Metric> void registerMetric(String metricName, T metric) {
  checkState();
  if (_registry == null) {
    return;
  }
  if (!_registry.getNames().contains(metricName)) {
    _registry.register(metricName, metric);
  } else {
    LOGGER.warn("Failed to register an existed metric: {}", metricName);
  }
}
 

@Override
public KeyValueStats getStats(Map<String, Object> stats) {
	super.getStats(stats);
	if (isOpen()) {
		Map<MetricName, ? extends Metric> kMetrics = producer.metrics();
		for (Map.Entry<MetricName, ? extends Metric> entry : kMetrics.entrySet()) {
			MetricName kMetric = entry.getKey();
			stats.put(Utils.qualify(this, kMetric.group() + "/" + kMetric.name()), entry.getValue().metricValue());
		}
	}
	return this;
}
 

public void register(
  String metricName,
  Metric metric,
  String tag
) {
  metrics.put(metricName, metric);
  tags.put(metricName, tag);
}
 

public void removeKafkaMetrics(String prefix, Map<MetricName, ? extends Metric> metrics) {
  checkState();
  for (MetricName metricName : metrics.keySet()) {
    String kafkaMetricName = String.format("%s.%s", prefix, metricName.name());
    removeMetric(kafkaMetricName);
  }
}
 

public void registerKafkaMetrics(String prefix, Map<MetricName, ? extends Metric> metrics) {
  checkState();
  for (MetricName metricName : metrics.keySet()) {
    String kafkaMetricName = String.format("%s.%s", prefix, metricName.name());
    registerMetric(kafkaMetricName, new GraphiteKafkaGauge(metrics.get(metricName)));
  }
}
 

public Map<String, Map<MetricName, ? extends Metric>> getAllConsumerMetrics()
{
  Map<String, Map<MetricName, ? extends Metric>> val = new HashMap<>();
  for (Map.Entry<String, AbstractKafkaConsumer> e : consumers.entrySet()) {
    val.put(e.getKey(), e.getValue().metrics());
  }
  return val;
}
 

@Override
public Metric put(Object key, Object value) {
  throw new UnsupportedOperationException(); //this collection is immutable in vanilla kafka anyway
}
 

@Override
public Map<MetricName, ? extends Metric> metrics() {
  throw new UnsupportedOperationException("Not implemented yet");
}
 

public KafkaMetricMutableWrapper(org.apache.kafka.common.Metric metric) {
	this.kafkaMetric = metric;
}
 

public void setKafkaMetric(Metric kafkaMetric) {
	this.kafkaMetric = kafkaMetric;
}
 

@Override
public Map<MetricName, Metric> metrics() {
  return null;
}
 

@Override
public Map<MetricName, ? extends Metric> metrics() {
	throw new UnsupportedOperationException();
}