com.spotify.metrics.core.SemanticMetricRegistry Java Examples

public SemanticFutureReporter(SemanticMetricRegistry registry, MetricId id) {
    final String what = id.getTags().get("what");

    if (what == null) {
        throw new IllegalArgumentException("id does not provide the tag 'what'");
    }

    this.timer = new SemanticHeroicTimer(registry.timer(id.tagged("what", what + "-latency")));
    this.failed =
        registry.counter(id.tagged("what", what + "-failed", "unit", Units.COUNT));
    this.resolved =
        registry.counter(id.tagged("what", what + "-resolved", "unit", Units.COUNT));
    this.cancelled =
        registry.counter(id.tagged("what", what + "-cancelled", "unit", Units.COUNT));
    this.pending =
        registry.counter(id.tagged("what", what + "-pending", "unit", Units.COUNT));
}
 

public SemanticSuggestBackendReporter(SemanticMetricRegistry registry) {
    final MetricId base = MetricId.build().tagged("component", COMPONENT);

    tagValuesSuggest = new SemanticFutureReporter(registry,
        base.tagged("what", "tag-values-suggest", "unit", Units.QUERY));
    tagKeyCount = new SemanticFutureReporter(registry,
        base.tagged("what", "tag-key-count", "unit", Units.QUERY));
    tagSuggest = new SemanticFutureReporter(registry,
        base.tagged("what", "tag-suggest", "unit", Units.QUERY));
    keySuggest = new SemanticFutureReporter(registry,
        base.tagged("what", "key-suggest", "unit", Units.QUERY));
    tagValueSuggest = new SemanticFutureReporter(registry,
        base.tagged("what", "tag-value-suggest", "unit", Units.QUERY));
    write =
        new SemanticFutureReporter(registry, base.tagged("what", "write", "unit", Units.WRITE));
    backendWrite = new SemanticFutureReporter(registry,
        base.tagged("what", "backend-write", "unit", Units.WRITE));

    writesDroppedByCacheHit = registry.counter(
        base.tagged("what", "writes-dropped-by-cache-hit", "unit", Units.COUNT));
    writesDroppedByDuplicate = registry.counter(
        base.tagged("what", "writes-dropped-by-duplicate", "unit", Units.COUNT));
}
 

private FastForwardHttpReporter(
        SemanticMetricRegistry registry, MetricId prefix, TimeUnit unit, long duration,
        HttpClient client, Set<Percentile> histogramPercentiles, Clock clock,
        TagExtractor tagExtractor,
        ScheduledExecutorService executorService, boolean executorOwner
) {
    this.registry = registry;
    this.prefix = prefix;
    this.unit = unit;
    this.duration = duration;
    this.client = client;
    this.histogramPercentiles = new HashSet<>(histogramPercentiles);
    this.clock = clock;
    this.tagExtractor = tagExtractor;
    this.executorService = executorService;
    this.executorOwner = executorOwner;
}
 

private static Statistics setupStatistics() throws IOException {
    final String key = System.getProperty("ffwd.key", "ffwd-java");

    final SemanticMetricRegistry registry = new SemanticMetricRegistry();
    final SemanticCoreStatistics statistics = new SemanticCoreStatistics(registry);

    final MetricId gauges = MetricId.build();

    registry.register(gauges, new ThreadStatesMetricSet());
    registry.register(gauges, new GarbageCollectorMetricSet());
    registry.register(gauges, new MemoryUsageGaugeSet());

    final MetricId metric = MetricId.build(key);

    final FastForwardReporter ffwd = FastForwardReporter
        .forRegistry(registry)
        .schedule(TimeUnit.SECONDS, 30)
        .prefix(metric)
        .build();

    ffwd.start();

    return new Statistics(ffwd, statistics);
}
 

@Before
public void setUp() throws Exception {
    registry = new SemanticMetricRegistry();
    fixedClock = new Clock.Fixed(0L);

    commonTags = of("foo", "bar");
    executorService = new DeterministicScheduler();

    reporter = FastForwardHttpReporter
        .forRegistry(registry, httpClient)
        .schedule(REPORTING_PERIOD, TimeUnit.MILLISECONDS)
        .prefix(MetricId.build("prefix").tagged(commonTags))
        .clock(fixedClock)
        .executorService(executorService)
        .build();
}
 

static Metrics create(final SemanticMetricRegistry registry, final MetricId metricId) {
  final MetricId predictDurationId = metricId.tagged("what", PREDICT_DURATION.tag());
  final MetricId predictRateId = metricId.tagged("what", PREDICT_RATE.tag());
  final MetricId extractDuration = metricId.tagged("what", FEATURE_EXTRACT_DURATION.tag());
  final MetricId extractRate = metricId.tagged("what", FEATURE_EXTRACT_RATE.tag());

  final Timer predictTimer = registry.timer(predictDurationId);
  final Meter predictMeter = registry.meter(predictRateId);
  final Timer extractTimer = registry.timer(extractDuration);
  final Meter extractMeter = registry.meter(extractRate);

  return new AutoValue_SemanticPredictorMetrics_Metrics(
      predictTimer, predictMeter, extractTimer, extractMeter);
}
 

public SemanticMetricBackendReporter(SemanticMetricRegistry registry) {
    final MetricId base = MetricId.build().tagged("component", COMPONENT);

    this.write =
        new SemanticFutureReporter(registry, base.tagged("what", "write", "unit", Units.WRITE));
    this.fetch =
        new SemanticFutureReporter(registry, base.tagged("what", "fetch", "unit", Units.QUERY));
    this.deleteKey = new SemanticFutureReporter(registry,
        base.tagged("what", "delete-key", "unit", Units.DELETE));
    this.countKey = new SemanticFutureReporter(registry,
        base.tagged("what", "count-key", "unit", Units.QUERY));
    this.fetchRow = new SemanticFutureReporter(registry,
        base.tagged("what", "fetch-row", "unit", Units.QUERY));

    this.findSeries = new SemanticFutureReporter(registry,
        base.tagged("what", "find-series", "unit", Units.QUERY));
    this.queryMetrics = new SemanticFutureReporter(registry,
        base.tagged("what", "query-metrics", "unit", Units.QUERY));

    sampleSizeLive = registry.counter(
        base.tagged("what", "sample-size-live", "unit", Units.SAMPLE));
    sampleSizeAccumulated = registry.counter(
        base.tagged("what", "sample-size-accumulated", "unit", Units.SAMPLE));

    querySamplesRead = registry.histogram(
        base.tagged("what", "query-metrics-samples-read", "unit", Units.COUNT));
    queryRowsAccessed = registry.histogram(
        base.tagged("what", "query-metrics-rows-accessed", "unit", Units.COUNT));
    queryMaxLiveSamples = registry.histogram(
        base.tagged("what", "query-metrics-max-live-samples", "unit", Units.COUNT));
    queryReadRate = registry.histogram(
        base.tagged("what", "query-metrics-read-rate", "unit", Units.COUNT));
    queryRowMsBetweenSamples = registry.histogram(
        base.tagged("what", "query-metrics-row-metric-distance", "unit", Units.MILLISECOND));
    queryRowDensity = registry.histogram(
        base.tagged("what", "query-metrics-row-density", "unit", Units.COUNT));
}
 

public SemanticAnalyticsReporter(SemanticMetricRegistry registry) {
    final MetricId id = MetricId.build().tagged("component", COMPONENT);

    this.droppedFetchSeries =
        registry.counter(id.tagged("what", "dropped-fetch-series", "unit", Units.DROP));
    this.failedFetchSeries =
        registry.counter(id.tagged("what", "failed-fetch-series", "unit", Units.FAILURE));
}
 

public SemanticIngestionManagerReporter(SemanticMetricRegistry registry) {
    final MetricId id = MetricId.build().tagged("component", COMPONENT);
    this.concurrentWritesCounter =
        registry.counter(id.tagged("what", "concurrent-writes", "unit", Units.WRITE));
    this.droppedByFilter =
        registry.counter(id.tagged("what", "dropped-by-filter", "unit", Units.COUNT));
}
 

public SemanticMetadataBackendReporter(SemanticMetricRegistry registry) {
    final MetricId base = MetricId.build().tagged("component", COMPONENT);

    findTags = new SemanticFutureReporter(registry,
        base.tagged("what", "find-tags", "unit", Units.QUERY));
    findSeries = new SemanticFutureReporter(registry,
        base.tagged("what", "find-series", "unit", Units.QUERY));
    findSeriesIds = new SemanticFutureReporter(registry,
        base.tagged("what", "find-series-ids", "unit", Units.QUERY));
    countSeries = new SemanticFutureReporter(registry,
        base.tagged("what", "count-series", "unit", Units.QUERY));
    deleteSeries = new SemanticFutureReporter(registry,
        base.tagged("what", "delete-series", "unit", Units.QUERY));
    findKeys = new SemanticFutureReporter(registry,
        base.tagged("what", "find-keys", "unit", Units.QUERY));
    write = new SemanticFutureReporter(registry,
        base.tagged("what", "write", "unit", Units.WRITE));
    backendWrite = new SemanticFutureReporter(registry,
        base.tagged("what", "backend-write", "unit", Units.WRITE));

    entries = registry.counter(base.tagged("what", "entries", "unit", Units.COUNT));

    writesDroppedByCacheHit = registry.counter(
        base.tagged("what", "writes-dropped-by-cache-hit", "unit", Units.COUNT));
    writesDroppedByDuplicate = registry.counter(
        base.tagged("what", "writes-dropped-by-duplicate", "unit", Units.COUNT));

    // only relevant to es backend.
    failedShards = registry.counter(
        base.tagged("what", "failed-es-shards", "unit", Units.COUNT));
}
 

public SemanticQueryReporter(final SemanticMetricRegistry registry) {
    final MetricId base = MetricId.build().tagged("component", COMPONENT);

    query =
        new SemanticFutureReporter(registry, base.tagged("what", "query", "unit", Units.QUERY));
    smallQueryLatency = registry.histogram(
        base.tagged("what", "small-query-latency", "unit", Units.MILLISECOND));
    queryReadRate =
        registry.histogram(base.tagged("what", "query-read-rate", "unit", Units.COUNT));

    rpcError = registry.counter(base.tagged("what", "cluster-rpc-error", "unit", Units.COUNT));
    rpcCancellation =
        registry.counter(base.tagged("what", "cluster-rpc-cancellation", "unit", Units.COUNT));
}
 

public SemanticConsumerReporter(SemanticMetricRegistry registry, String id) {
    this.base = MetricId.build().tagged("component", COMPONENT, "id", id);

    messageIn = registry.counter(base.tagged("what", "message-in", "unit", Units.COUNT));
    metricsIn = registry.counter(base.tagged("what", "metrics-in", "unit", Units.COUNT));
    messageError = registry.counter(base.tagged("what", "message-error", "unit", Units.COUNT));
    messageRetry = registry.counter(base.tagged("what", "message-retry", "unit", Units.COUNT));
    consumerSchemaError =
        registry.counter(base.tagged("what", "consumer-schema-error", "unit", Units.COUNT));
    consumerThreadsLiveRatio = new SemanticRatioGauge();
    registry.register(base.tagged("what", "consumer-threads-live-ratio", "unit", Units.RATIO),
        consumerThreadsLiveRatio);

    messageSize = registry.histogram(base.tagged("what", "message-size", "unit", Units.BYTE));
    messageDrift =
        registry.histogram(base.tagged("what", "message-drift", "unit", Units.MILLISECOND));

    consumer = new SemanticFutureReporter(registry,
        base.tagged("what", "consumer", "unit", Units.WRITE));

    consumerCommitWholeOperationTimer =
        registry.register(base.tagged("what", "consumer-commit-latency"),
            new SemanticHeroicTimerGauge());
    consumerCommitPhase1Timer =
        registry.register(base.tagged("what", "consumer-commit-phase1-latency"),
            new SemanticHeroicTimerGauge());
    consumerCommitPhase2Timer =
        registry.register(base.tagged("what", "consumer-commit-phase2-latency"),
            new SemanticHeroicTimerGauge());
}
 

public SemanticMemcachedReporter(SemanticMetricRegistry registry, final String consumerType) {
  final MetricId id = MetricId.build().tagged("component", COMPONENT, "consumer", consumerType);

  memcachedHit = registry.counter(id.tagged("what", "memcached-performance", "result",
      "hit"));
  memcachedMiss = registry.counter(id.tagged("what", "memcached-performance", "result",
      "miss"));
  memcachedTimeout = registry.counter(id.tagged("what", "memcached-performance", "result",
      "timeout"));

  memcachedError = registry.counter(id.tagged("what", "memcached-performance", "result",
      "error"));


}
 

public static void main(String[] args) throws LoadingException, IOException {
  StackdriverTraceExporter.createAndRegister(
      StackdriverTraceConfiguration.builder().build());

  final AppInit init = (env) -> {
    final MetricsStats stats =
        new MetricsStats(env.resolve(SemanticMetricRegistry.class), Instant::now);
    final StatsFactory statsFactory = (ignored) -> stats;

    final AuthenticatorFactory authenticatorFactory =
        Function1.of(AuthenticatorFactory.DEFAULT::apply).memoized()::apply;

    final StyxScheduler scheduler = StyxScheduler.newBuilder()
        .setServiceName(SERVICE_NAME)
        .setStatsFactory(statsFactory)
        .setAuthenticatorFactory(authenticatorFactory)
        .build();
    final StyxApi api = StyxApi.newBuilder()
        .setServiceName(SERVICE_NAME)
        .setStatsFactory(statsFactory)
        .setAuthenticatorFactory(authenticatorFactory)
        .build();

    scheduler.create(env);
    api.create(env);
  };

  HttpService.boot(init, SERVICE_NAME, args);
}
 

public MetricsStats(SemanticMetricRegistry registry, Time time) {
  this.registry = Objects.requireNonNull(registry);
  this.time = Objects.requireNonNull(time, "time");

  this.submitToRunning = registry.getOrAdd(TRANSITIONING_DURATION, HISTOGRAM);
  this.pullImageErrorMeter = registry.meter(PULL_IMAGE_ERROR_RATE);
  this.naturalTrigger = registry.meter(NATURAL_TRIGGER_RATE);
  this.terminationLogMissing = registry.meter(TERMINATION_LOG_MISSING);
  this.terminationLogInvalid = registry.meter(TERMINATION_LOG_INVALID);
  this.exitCodeMismatch = registry.meter(EXIT_CODE_MISMATCH);
  this.workflowConsumerErrorMeter = registry.meter(WORKFLOW_CONSUMER_ERROR_RATE);
  this.counterCacheHitMeter = registry.meter(COUNTER_CACHE_RATE.tagged(COUNTER_CACHE_RESULT, COUNTER_CACHE_HIT));
  this.counterCacheMissMeter = registry.meter(COUNTER_CACHE_RATE.tagged(COUNTER_CACHE_RESULT, COUNTER_CACHE_MISS));
  this.serviceAccountCleanupMeter = registry.meter(SERVICE_ACCOUNT_CLEANUP_RATE);
  this.storageOperationHistograms = new ConcurrentHashMap<>();
  this.storageOperationMeters = new ConcurrentHashMap<>();
  this.dockerOperationHistograms = new ConcurrentHashMap<>();
  this.dockerOperationMeters = new ConcurrentHashMap<>();
  this.dockerOperationErrorMeters = new ConcurrentHashMap<>();
  this.kubernetesOperationHistograms = new ConcurrentHashMap<>();
  this.kubernetesOperationMeters = new ConcurrentHashMap<>();
  this.kubernetesOperationErrorMeters = new ConcurrentHashMap<>();
  this.exitCodeMeters = new ConcurrentHashMap<>();
  this.resourceConfiguredHistograms = new ConcurrentHashMap<>();
  this.resourceUsedHistograms = new ConcurrentHashMap<>();
  this.resourceDemandedHistograms = new ConcurrentHashMap<>();
  this.eventConsumerErrorMeters = new ConcurrentHashMap<>();
  this.eventConsumerMeters = new ConcurrentHashMap<>();
  this.publishingMeters = new ConcurrentHashMap<>();
  this.publishingErrorMeters = new ConcurrentHashMap<>();
  this.workflowConsumerMeters = new ConcurrentHashMap<>();
  this.tickHistograms = new ConcurrentHashMap<>();
  this.datastoreOperationMeters = new ConcurrentHashMap<>();
}
 

/** Creates a new @{link SemanticPredictorMetrics}. */
public static <InputT, VectorT, ValueT> SemanticPredictorMetrics<InputT, VectorT, ValueT> create(
    final SemanticMetricRegistry registry, final MetricId metricId) {
  final LoadingCache<Id, Metrics> metersCache =
      CacheBuilder.newBuilder()
          .build(
              new CacheLoader<Id, Metrics>() {
                @Override
                public Metrics load(final Id id) {
                  return Metrics.create(registry, metricId.tagged("model", id.value()));
                }
              });

  return new AutoValue_SemanticPredictorMetrics<>(metersCache);
}
 

static CustomMetrics create(final SemanticMetricRegistry registry, final MetricId metricId) {
  final MetricId predictCountId = metricId.tagged("what", "negativePredictCount");
  final MetricId extractCountId = metricId.tagged("what", "negativeExtractCount");
  final Counter negativePredictCount = registry.counter(predictCountId);
  final Counter negativeExtractCount = registry.counter(extractCountId);

  return new AutoValue_CustomMetricsExample_CustomMetrics(
      negativePredictCount, negativeExtractCount);
}
 

static CustomPredictorMetrics create(
    final SemanticMetricRegistry registry, final MetricId metricId) {
  final LoadingCache<Model.Id, CustomMetrics> metersCache =
      CacheBuilder.newBuilder()
          .build(
              new CacheLoader<Model.Id, CustomMetrics>() {
                @Override
                public CustomMetrics load(final Model.Id id) {
                  return CustomMetrics.create(registry, metricId.tagged("model", id.value()));
                }
              });

  return new AutoValue_CustomMetricsExample_CustomPredictorMetrics(metersCache);
}
 

CustomMetricsExample(final SemanticMetricRegistry metricRegistry, final MetricId metricId)
    throws InterruptedException, ExecutionException, TimeoutException {
  final ModelLoader<DummyModel> modelLoader =
      ModelLoader.preload(ModelLoader.loaded(new DummyModel()), Duration.ofMinutes(1));
  final ExtractFn<Integer, Float> extractFn = ExtractFn.lift(input -> (float) input / 10);
  final PredictFn<DummyModel, Integer, Float, Float> predictFn =
      (model, vectors) -> {
        return vectors
            .stream()
            .map(vector -> Prediction.create(vector.input(), vector.value() * 2))
            .collect(Collectors.toList());
      };
  final FeatureExtractor<DummyModel, Integer, Float> featureExtractor =
      FeatureExtractor.create(extractFn);

  // We build the PredictorBuilder as usual, compose with the built-in metrics, and then compose
  // with our custom metrics.
  // #PredictorMetrics
  final PredictorMetrics<Integer, Float, Float> predictorMetrics =
      SemanticPredictorMetrics.create(metricRegistry, metricId);
  // #PredictorMetrics

  final PredictorMetrics<Integer, Float, Float> customMetrics =
      CustomPredictorMetrics.create(metricRegistry, metricId);

  predictorBuilder =
      // #PredictorBuilderWithMetrics
      Predictors.newBuilder(modelLoader, featureExtractor, predictFn, predictorMetrics)
          // #PredictorBuilderWithMetrics
          .with(Instrumentations.predictor(customMetrics));
}
 

@Test
public void testCustomMetrics()
    throws InterruptedException, ExecutionException, TimeoutException {
  // #SemanticMetricRegistry
  final SemanticMetricRegistry registry = new SemanticMetricRegistry();
  final MetricId metricId = MetricId.build().tagged("service", "my-application");
  // #SemanticMetricRegistry

  final CustomMetricsExample example = new CustomMetricsExample(registry, metricId);

  example.predict(3, 1, -4, -42, 42, -10).toCompletableFuture().join();

  registry.getCounters().values().forEach(counter -> Assert.assertEquals(3, counter.getCount()));
}
 

private FastForwardReporter(
        SemanticMetricRegistry registry, MetricId prefix, TimeUnit unit, long duration,
        FastForward client, Set<Percentile> histogramPercentiles, TagExtractor tagExtractor,
        ScheduledExecutorService executorService,
        boolean executorOwner) {
    this.registry = registry;
    this.prefix = prefix;
    this.unit = unit;
    this.duration = duration;
    this.client = client;
    this.histogramPercentiles = new HashSet<>(histogramPercentiles);
    this.tagExtractor = tagExtractor;
    this.executorService = executorService;
    this.executorOwner = executorOwner;
}
 

@Before
public void setUp() throws Exception {
    registry = new SemanticMetricRegistry();
    fastForward = mock(FastForward.class);
    executorService = new DeterministicScheduler();
    reporter = FastForwardReporter
        .forRegistry(registry)
        .schedule(TimeUnit.MILLISECONDS, REPORTING_PERIOD)
        .fastForward(fastForward)
        .executorService(executorService)
        .build();

    registry.counter(MetricId.build("hi"));
}
 

@java.beans.ConstructorProperties({ "registry" })
public SemanticHeroicReporter(final SemanticMetricRegistry registry) {
    this.registry = registry;
}
 

private static Stats stats(Environment environment) {
  return new MetricsStats(environment.resolve(SemanticMetricRegistry.class), Instant::now);
}
 

public static Builder forRegistry(SemanticMetricRegistry registry) {
    return new Builder(registry);
}
 

public Builder(SemanticMetricRegistry registry) {
    this.registry = registry;
}
 

@Provides
@SemanticStatisticsScope
public HeroicReporter reporter(SemanticMetricRegistry registry) {
    return new SemanticHeroicReporter(registry);
}
 

@Provides
@SemanticStatisticsScope
public LifeCycle life(
    AsyncFramework async, SemanticMetricRegistry registry, ServiceInfo info,
    LifeCycleRegistry lifeCycleRegistry
) {
    final MetricId gauges = MetricId.build();

    registry.register(gauges, new ThreadStatesMetricSet());
    registry.register(gauges, new GarbageCollectorMetricSet());
    registry.register(gauges, new MemoryUsageGaugeSet());
    registry.register(gauges, CpuGaugeSet.create());

    final MetricId metric =
        MetricId.build("heroic").tagged("service", "heroic", "heroic_id", info.getId());

    final FastForwardReporter reporter;

    try {
        reporter = FastForwardReporter
            .forRegistry(registry)
            .schedule(TimeUnit.SECONDS, 30)
            .prefix(metric)
            .build();
    } catch (IOException e) {
        throw new RuntimeException("Failed to configure reporter", e);
    }

    return () -> {
        final LifeCycleRegistry scope = lifeCycleRegistry.scoped("ffwd-reporter");

        scope.start(() -> async.call(() -> {
            reporter.start();
            return null;
        }));

        scope.stop(() -> async.call(() -> {
            reporter.stop();
            return null;
        }));
    };
}
 

@Provides
@SemanticStatisticsScope
public SemanticMetricRegistry registry() {
    return new SemanticMetricRegistry(
        () -> new SlidingTimeWindowArrayReservoir(60, TimeUnit.SECONDS));
}
 

public SemanticCoreStatistics(SemanticMetricRegistry registry) {
    this.registry = registry;
}