org.apache.kafka.common.metrics.Sensor Java Examples

@Test
public void testMetricChange() throws Exception {
    Metrics metrics = new Metrics();
    DropwizardReporter reporter = new DropwizardReporter();
    reporter.configure(new HashMap<String, Object>());
    metrics.addReporter(reporter);
    Sensor sensor = metrics.sensor("kafka.requests");
    sensor.add(new MetricName("pack.bean1.avg", "grp1"), new Avg());

    Map<String, Gauge> gauges = SharedMetricRegistries.getOrCreate("default").getGauges();
    String expectedName = "org.apache.kafka.common.metrics.grp1.pack.bean1.avg";
    Assert.assertEquals(1, gauges.size());
    Assert.assertEquals(expectedName, gauges.keySet().toArray()[0]);

    sensor.record(2.1);
    sensor.record(2.2);
    sensor.record(2.6);
    Assert.assertEquals(2.3, (Double)gauges.get(expectedName).getValue(), 0.001);
}
 

@Override
public synchronized void start() {
  if (_running.compareAndSet(false, true)) {
    _consumeThread.start();
    LOG.info("{}/ConsumeService started.", _name);

    Sensor topicPartitionCount = metrics.sensor("topic-partitions");
    DescribeTopicsResult describeTopicsResult = _adminClient.describeTopics(Collections.singleton(_topic));
    Map<String, KafkaFuture<TopicDescription>> topicResultValues = describeTopicsResult.values();
    KafkaFuture<TopicDescription> topicDescriptionKafkaFuture = topicResultValues.get(_topic);
    TopicDescription topicDescription = null;
    try {
      topicDescription = topicDescriptionKafkaFuture.get();
    } catch (InterruptedException | ExecutionException e) {
      LOG.error("Exception occurred while getting the topicDescriptionKafkaFuture for topic: {}", _topic, e);
    }
    @SuppressWarnings("ConstantConditions")
    double partitionCount = topicDescription.partitions().size();
    topicPartitionCount.add(
        new MetricName("topic-partitions-count", METRIC_GROUP_NAME, "The total number of partitions for the topic.", tags), new Total(partitionCount));
  }
}
 

@Override
@SuppressWarnings("unchecked")
public void init(final ProcessorContext processorContext) {
    keyValueStore = (KeyValueStore) processorContext.getStateStore(stocksStateStore);
    this.processorContext = processorContext;
    
    this.processorContext.schedule(5000, PunctuationType.WALL_CLOCK_TIME, this::doPunctuate);


    final String tagKey = "task-id";
    final String tagValue = processorContext.taskId().toString();
    final String nodeName = "StockPerformanceProcessor_"+count.getAndIncrement();
   metricsSensor = processorContext.metrics().addLatencyAndThroughputSensor("transformer-node",
            nodeName, "stock-performance-calculation",
            Sensor.RecordingLevel.DEBUG,
            tagKey,
            tagValue);
}
 

private void ensureMetricsCreated(ConnectorTaskId taskId) {
  Map<String, String> tags = getTaskLevelTags(taskId);
  MetricName taskMetric =
      getMetric(
          FAILED_TASK_ATTEMPTS_METRIC_NAME + "-count",
          TASK_CONNECTOR_JMX_GROUP_NAME,
          "count of restart attempts to a failed task",
          taskLevelJmxTags,
          tags);

  if (!metrics.metrics().containsKey(taskMetric)) {
    Sensor sensor = getSensor(taskId.toString());
    sensor.add(taskMetric, new Total());
    logger.info("Added the task {} to the list of JMX metrics", taskId);
    logger.debug("Updated set of JMX metrics is {}", metrics.metrics());
  }
}
 

private Sensor configureErrorRate(Metrics metrics) {
  Sensor sensor = createSensor(metrics, metricGroupName + "-error-rate");
  sensor.add(
      metrics.metricName("error-rate", this.metricGroupName,
                         "The number of messages which were consumed but not processed. "
                         + "Messages may not be processed if, for instance, the message "
                         + "contents could not be deserialized due to an incompatible schema. "
                         + "Alternately, a consumed messages may not have been produced, hence "
                         + "being effectively dropped. Such messages would also be counted "
                         + "toward the error rate."),
      new Value());
  return sensor;
}
 

private Sensor configureMessagesOut(Metrics metrics) {
  Sensor sensor = createSensor(metrics, metricGroupName + "-messages-produced");
  sensor.add(
      metrics.metricName("messages-produced-per-sec", this.metricGroupName,
                         "The number of messages produced per second across all queries"),
      new Value());

  return sensor;
}
 

private Sensor configureMessagesIn(Metrics metrics) {
  Sensor sensor = createSensor(metrics, metricGroupName + "-messages-consumed");
  sensor.add(
      metrics.metricName("messages-consumed-per-sec", this.metricGroupName,
                         "The number of messages consumed per second across all queries"),
      new Value());
  return sensor;
}
 

private Sensor configureTotalMessagesIn(Metrics metrics) {
  Sensor sensor = createSensor(metrics, metricGroupName + "-total-messages-consumed");
  sensor.add(
      metrics.metricName("messages-consumed-total", this.metricGroupName,
          "The total number of messages consumed across all queries"),
      new Value());
  return sensor;
}
 

private Sensor configureTotalBytesIn(Metrics metrics) {
  Sensor sensor = createSensor(metrics, metricGroupName + "-total-bytes-consumed");
  sensor.add(
      metrics.metricName("bytes-consumed-total", this.metricGroupName,
          "The total number of bytes consumed across all queries"),
      new Value());
  return sensor;
}
 

public void addPartitionSensors(int partition) {
  Sensor recordsProducedSensor = _metrics.sensor("records-produced-partition-" + partition);
  recordsProducedSensor.add(new MetricName("records-produced-rate-partition-" + partition,
      XinfraMonitorConstants.METRIC_GROUP_NAME_PRODUCE_SERVICE,
      "The average number of records per second that are produced to this partition", _tags), new Rate());
  _recordsProducedPerPartition.put(partition, recordsProducedSensor);

  Sensor errorsSensor = _metrics.sensor("produce-error-partition-" + partition);
  errorsSensor.add(new MetricName("produce-error-rate-partition-" + partition,
      XinfraMonitorConstants.METRIC_GROUP_NAME_PRODUCE_SERVICE,
      "The average number of errors per second when producing to this partition", _tags), new Rate());
  _produceErrorPerPartition.put(partition, errorsSensor);
}
 

private Sensor configureMessageConsumptionByQuerySensor(Metrics metrics) {
  Sensor sensor = createSensor(metrics, "message-consumption-by-query");
  sensor.add(metrics.metricName("messages-consumed-max", this.metricGroupName), new Max());
  sensor.add(metrics.metricName("messages-consumed-min", this.metricGroupName), new Min());
  sensor.add(metrics.metricName("messages-consumed-avg", this.metricGroupName), new Avg());
  return sensor;
}
 

MissingPartitionsJmxReporter(Metrics metrics) {
  super(metrics);
  Sensor missingPartsSensor = metrics.sensor(MISSING_DEST_PARTITIONS);
  MetricName missingPartsName = metrics.metricName(MISSING_DEST_PARTITIONS + "-count", "mirus");
  missingPartsSensor.add(missingPartsName, new Value());
  this.missingPartsSensor = missingPartsSensor;
}
 

private void updateTaskMetrics(ConnectorTaskId taskId, TaskState taskStatus) {

    if (taskStatus.state().equalsIgnoreCase(TaskStatus.State.FAILED.toString())) {
      Sensor sensor = getSensor(taskId.toString());
      sensor.record(1, Time.SYSTEM.milliseconds());
    }
  }
 

private Sensor createSensor(Metrics metrics, String sensorName) {
  Sensor sensor = metrics.sensor(sensorName);
  sensors.add(sensor);
  return sensor;
}
 

private Sensor getSensor(String taskId) {
  return metrics.sensor(taskId);
}
 

SensorMetric(Sensor sensor, KafkaMetric metric, Time time, boolean errorMetric) {
  this.sensor = sensor;
  this.metric = metric;
  this.time = time;
  this.errorMetric = errorMetric;
}
 

public void addSensor(String name) {
    Sensor sensor = metrics.sensor(name);
    sensor.add(metrics.metricName("value", name), new Value());
}
 

private Sensor configureIdleQueriesSensor(Metrics metrics) {
  Sensor sensor = createSensor(metrics, "num-idle-queries");
  sensor.add(metrics.metricName("num-idle-queries", this.metricGroupName), new Value());
  return sensor;
}
 

List<Sensor> registeredSensors() {
  return sensors;
}