io.opencensus.exporter.stats.stackdriver.StackdriverStatsExporter Java Examples

private void ensureStackdriver(Gradle gradle) {
  // make sure we only initialize stackdriver once as gradle daemon is not guaranteed to restart
  // across gradle invocations.
  if (!STACKDRIVER_INITIALIZED.compareAndSet(false, true)) {
    logger.lifecycle("Stackdriver exporter already initialized.");
    return;
  }
  logger.lifecycle("Initializing Stackdriver exporter.");

  try {
    StackdriverStatsExporter.createAndRegister(
        StackdriverStatsConfiguration.builder()
            .setExportInterval(Duration.fromMillis(STACKDRIVER_UPLOAD_PERIOD_MS))
            .build());

    // make sure gradle does not exit before metrics get uploaded to stackdriver.
    gradle.addBuildListener(new DrainingBuildListener(STACKDRIVER_UPLOAD_PERIOD_MS, logger));
  } catch (IOException e) {
    throw new GradleException("Could not configure metrics exporter", e);
  }
}
 

public AsyncFuture<Void> start() {
  // NB using Application Default Credentials here:
  // See https://developers.google.com/identity/protocols/application-default-credentials
  try {
    StackdriverStatsConfiguration.Builder builder = StackdriverStatsConfiguration.builder();

    // We clear out the constant labels because otherwise they are populated with
    // values from this VM which is unlikely to be what we want.
    builder.setConstantLabels(Collections.emptyMap());

    // This can also be done by setting enviroment variables but it'll be frequently
    // be used so let's make it easy.
    if (gcpProject.isPresent()) {
      builder.setProjectId(gcpProject.get());
    }

    StackdriverStatsExporter.createAndRegister(builder.build());
  } catch (IOException ex) {
    log.error("Couldn't connect to Stackdriver");
    return async.failed(ex);
  }
  return async.resolved();
}
 

@Test
public void canFileLoad() throws Exception {
  createTablesAndPublishInputs(ImmutableList.of(SIMPLE_MESSAGE));

  Map<String, String> sinkEnv = ImmutableMap.<String, String>builder()
      .put("BATCH_MAX_DELAY", "0s") //
      .put("BIG_QUERY_OUTPUT_MODE", "file_loads") //
      .put("INPUT_SUBSCRIPTION", pubsub.getSubscription(0)) //
      .put("OUTPUT_BUCKET", gcs.bucket) //
      .put("OUTPUT_TABLE",
          bq.project + "." + bq.dataset + ".${document_type}_v${document_version}") //
      .put("OUTPUT_TOPIC", pubsub.getTopic(1)) //
      .build();
  sinkEnv.forEach(environmentVariables::set);
  BoundedSink.run(1, 30);
  checkResult(ImmutableList.of());

  sinkEnv.keySet().forEach(environmentVariables::clear);
  environmentVariables.set("INPUT_SUBSCRIPTION", pubsub.getSubscription(1));
  environmentVariables.set("LOAD_MAX_DELAY", "0s");
  // unregister stackdriver stats exporter so BoundedSink can register another one
  StackdriverStatsExporter.unregister();
  BoundedSink.run(1, 30);
  checkResult(ImmutableList.of(SIMPLE_MESSAGE_ROW));
}
 

public static boolean setupStackdriverStatsExporter() {
  if (enableTelemetry()) {
    try {
      registerAllViews();
      URL url = Resources.getResource(CREDENTIALS_JSON);
      StackdriverStatsExporter.createAndRegister(
          StackdriverStatsConfiguration.builder()
              .setProjectId(PROJECT_ID)
              .setCredentials(ServiceAccountCredentials.fromStream(url.openStream()))
              .build());
      log.info("enable stackdriver stats exporter");
      return true;
    } catch (Throwable e) {
      log.warn("{}", e.getMessage());
    }
  }
  return false;
}
 

private void configureTracing() {
  try {
    // Installs a handler for /tracez page.
    ZPageHandlers.startHttpServerAndRegisterAll(8080);
    // Installs an exporter for stack driver traces.
    StackdriverExporter.createAndRegister();
    Tracing.getExportComponent()
        .getSampledSpanStore()
        .registerSpanNamesForCollection(Arrays.asList(SAMPLE_SPAN));

    // Installs an exporter for stack driver stats.
    StackdriverStatsExporter.createAndRegister();
    RpcViews.registerAllCumulativeViews();
  } catch (IOException e) {
    log.error("Could not start the tracing server", e);
  }
}
 

@Override
public void configureAndStart(String name, MetricRegistry registry, MetricFilter filter) {
  this.producer = new DropWizardMetrics(Collections.singletonList(registry), filter);

  Metrics.getExportComponent().getMetricProducerManager().add(producer);

  try {
    StackdriverStatsExporter.createAndRegister();
  } catch (IOException e) {
    logger.warn("Could not setup stackdriver stats", e);
  }
}
 

public static void main(String[] args) throws IOException, InterruptedException {
  // Register the view. It is imperative that this step exists,
  // otherwise recorded metrics will be dropped and never exported.
  View view =
      View.create(
          Name.create("task_latency_distribution"),
          "The distribution of the task latencies.",
          LATENCY_MS,
          Aggregation.Distribution.create(LATENCY_BOUNDARIES),
          Collections.emptyList());

  ViewManager viewManager = Stats.getViewManager();
  viewManager.registerView(view);

  // [START setup_exporter]
  // Enable OpenCensus exporters to export metrics to Stackdriver Monitoring.
  // Exporters use Application Default Credentials to authenticate.
  // See https://developers.google.com/identity/protocols/application-default-credentials
  // for more details.
  StackdriverStatsExporter.createAndRegister();
  // [END setup_exporter]

  // Record 100 fake latency values between 0 and 5 seconds.
  Random rand = new Random();
  for (int i = 0; i < 100; i++) {
    long ms = (long) (TimeUnit.MILLISECONDS.convert(5, TimeUnit.SECONDS) * rand.nextDouble());
    System.out.println(String.format("Latency %d: %d", i, ms));
    STATS_RECORDER.newMeasureMap().put(LATENCY_MS, ms).record();
  }

  // The default export interval is 60 seconds. The thread with the StackdriverStatsExporter must
  // live for at least the interval past any metrics that must be collected, or some risk being
  // lost if they are recorded after the last export.

  System.out.println(
      String.format(
          "Sleeping %d seconds before shutdown to ensure all records are flushed.",
          EXPORT_INTERVAL));
  Thread.sleep(TimeUnit.MILLISECONDS.convert(EXPORT_INTERVAL, TimeUnit.SECONDS));
}
 

/** Main launches the server from the command line. */
public static void main(String[] args) throws IOException, InterruptedException {
  // Add final keyword to pass checkStyle.
  final int serverPort = getPortOrDefaultFromArgs(args, 0, 50051);
  final String cloudProjectId = getStringOrDefaultFromArgs(args, 1, null);
  final int zPagePort = getPortOrDefaultFromArgs(args, 2, 3000);
  final int prometheusPort = getPortOrDefaultFromArgs(args, 3, 9090);

  // Registers all RPC views. For demonstration all views are registered. You may want to
  // start with registering basic views and register other views as needed for your application.
  RpcViews.registerAllViews();

  // Registers logging trace exporter.
  LoggingTraceExporter.register();

  // Starts a HTTP server and registers all Zpages to it.
  ZPageHandlers.startHttpServerAndRegisterAll(zPagePort);
  logger.info("ZPages server starts at localhost:" + zPagePort);

  // Registers Stackdriver exporters.
  if (cloudProjectId != null) {
    StackdriverTraceExporter.createAndRegister(
        StackdriverTraceConfiguration.builder().setProjectId(cloudProjectId).build());
    StackdriverStatsExporter.createAndRegister(
        StackdriverStatsConfiguration.builder()
            .setProjectId(cloudProjectId)
            .setExportInterval(Duration.create(60, 0))
            .build());
  }

  // Register Prometheus exporters and export metrics to a Prometheus HTTPServer.
  PrometheusStatsCollector.createAndRegister();
  HTTPServer prometheusServer = new HTTPServer(prometheusPort, true);

  // Start the RPC server. You shouldn't see any output from gRPC before this.
  logger.info("gRPC starting.");
  final HelloWorldServer server = new HelloWorldServer(serverPort);
  server.start();
  server.blockUntilShutdown();
}
 

private GoogleDtpInternalMetricRecorder(GoogleCredentials credentials, String projectId)
    throws IOException {
  // Enable OpenCensus exporters to export metrics to Stackdriver Monitoring.
  // Exporters use Application Default Credentials to authenticate.
  // See https://developers.google.com/identity/protocols/application-default-credentials
  // for more details.
  StackdriverStatsConfiguration configuration = StackdriverStatsConfiguration.builder()
      .setCredentials(credentials)
      .setProjectId(projectId)
      .setExportInterval(io.opencensus.common.Duration.create(EXPORT_INTERVAL_SECONDS, 0))
      .build();
  StackdriverStatsExporter.createAndRegister(configuration);
  this.viewManager = Stats.getViewManager();
  setupViews();
}
 

/**
 * Run async until {@code messageCount} messages have been delivered.
 */
private static CompletableFuture<Void> runAsync(int messageCount) throws IOException {
  final Output output = getOutput();
  final AtomicInteger counter = new AtomicInteger(0);
  final AtomicReference<Pubsub.Read> input = new AtomicReference<>();
  input.set(getInput(output.via(message -> output.apply(message).thenApplyAsync(v -> {
    final int currentMessages = counter.incrementAndGet();
    if (currentMessages >= messageCount) {
      input.get().subscriber.stopAsync();
      StackdriverStatsExporter.unregister();
    }
    return v;
  }))));
  return CompletableFuture.runAsync(input.get()::run);
}
 

/** Return a configured input transform. */
public static Pubsub.Read getInput(Output output) throws IOException {
  // read pubsub messages from INPUT_SUBSCRIPTION
  Pubsub.Read input = new Pubsub.Read(output.env.getString(INPUT_SUBSCRIPTION), output,
      builder -> builder
          .setFlowControlSettings(FlowControlSettings.newBuilder()
              .setMaxOutstandingElementCount(
                  output.type.getMaxOutstandingElementCount(output.env))
              .setMaxOutstandingRequestBytes(
                  output.type.getMaxOutstandingRequestBytes(output.env))
              .build())
          // The number of streaming subscriber connections for reading from Pub/Sub.
          // https://github.com/googleapis/java-pubsub/blob/v1.105.0/google-cloud-pubsub/src/main/java/com/google/cloud/pubsub/v1/Subscriber.java#L141
          // https://github.com/googleapis/java-pubsub/blob/v1.105.0/google-cloud-pubsub/src/main/java/com/google/cloud/pubsub/v1/Subscriber.java#L318-L320
          // The default number of executor threads is max(6, 2*parallelPullCount).
          // https://github.com/googleapis/java-pubsub/blob/v1.105.0/google-cloud-pubsub/src/main/java/com/google/cloud/pubsub/v1/Subscriber.java#L566-L568
          // Subscriber connections are expected to be CPU bound until flow control thresholds are
          // reached, so parallelism should be no less than the number of available processors.
          .setParallelPullCount(
              output.env.getInt(INPUT_PARALLELISM, Runtime.getRuntime().availableProcessors())),
      getInputCompression(output.env));
  output.env.requireAllVarsUsed();
  // Setup OpenCensus stackdriver exporter after all measurement views have been registered,
  // as seen in https://opencensus.io/exporters/supported-exporters/java/stackdriver-stats/
  StackdriverStatsExporter.createAndRegister();
  return input;
}
 

@Before
public void unregisterStackdriver() {
  // unregister stackdriver stats exporter in case a previous test already registered one.
  StackdriverStatsExporter.unregister();
}
 

@Override
public void close() {
  Metrics.getExportComponent().getMetricProducerManager().remove(producer);
  StackdriverStatsExporter.unregister();
}
 

@Test
public void canStreamSpecificDoctypes() throws Exception {
  final List<PubsubMessage> streamingInput = ImmutableList.of(
      SIMPLE_MESSAGE.toBuilder().putAttributes("document_namespace", "namespace_0").build(),
      SIMPLE_MESSAGE.toBuilder().putAttributes("document_namespace", "namespace_0")
          .putAttributes("document_type", "bar").build(),
      SIMPLE_MESSAGE.toBuilder().putAttributes("document_namespace", "namespace_1")
          .putAttributes("document_version", "2").build());
  final List<List<String>> streamingExpected = ImmutableList.of(
      ImmutableList.of("foo", "1", SUBMISSION_TIMESTAMP, "x", "foo_v1"),
      ImmutableList.of("bar", "1", SUBMISSION_TIMESTAMP, "x", "bar_v1"),
      ImmutableList.of("foo", "2", SUBMISSION_TIMESTAMP, "x", "foo_v2"));
  createTablesAndPublishInputs(streamingInput);

  final List<PubsubMessage> nonStreamingInput = ImmutableList.of(
      SIMPLE_MESSAGE.toBuilder().putAttributes("document_namespace", "namespace_1")
          .putAttributes("document_type", "bar").putAttributes("document_version", "2").build(),
      SIMPLE_MESSAGE.toBuilder().putAttributes("document_namespace", "namespace_2")
          .putAttributes("document_version", "3").build(),
      SIMPLE_MESSAGE.toBuilder().putAttributes("document_namespace", "namespace_2")
          .putAttributes("document_type", "bar").putAttributes("document_version", "3").build());
  final List<List<String>> nonStreamingExpected = ImmutableList.of(
      ImmutableList.of("bar", "2", SUBMISSION_TIMESTAMP, "x", "bar_v2"),
      ImmutableList.of("foo", "3", SUBMISSION_TIMESTAMP, "x", "foo_v3"),
      ImmutableList.of("bar", "3", SUBMISSION_TIMESTAMP, "x", "bar_v3"));
  createTablesAndPublishInputs(nonStreamingInput);

  Map<String, String> sinkEnv = ImmutableMap.<String, String>builder()
      .put("BATCH_MAX_DELAY", "0s") //
      .put("BIG_QUERY_OUTPUT_MODE", "mixed") //
      .put("INPUT_SUBSCRIPTION", pubsub.getSubscription(0)) //
      .put("OUTPUT_BUCKET", gcs.bucket) //
      .put("OUTPUT_TABLE",
          bq.project + "." + bq.dataset + ".${document_type}_v${document_version}") //
      .put("OUTPUT_TOPIC", pubsub.getTopic(1)) //
      .put("STREAMING_BATCH_MAX_DELAY", "0s") //
      .put("STREAMING_DOCTYPES", "namespace-0/.*|namespace-1/foo") //
      .build();
  sinkEnv.forEach(environmentVariables::set);
  BoundedSink.run(streamingInput.size() + nonStreamingInput.size(), 30);
  checkResult(streamingExpected);

  sinkEnv.keySet().forEach(environmentVariables::clear);
  environmentVariables.set("INPUT_SUBSCRIPTION", pubsub.getSubscription(1));
  environmentVariables.set("LOAD_MAX_DELAY", "0s");
  // unregister stackdriver stats exporter so BoundedSink can register another one
  StackdriverStatsExporter.unregister();
  BoundedSink.run(nonStreamingInput.size(), 30);
  checkResult(Iterables.concat(streamingExpected, nonStreamingExpected));
}
 

/** Main launcher for the Stackdriver example. */
public static void main(String[] args) throws IOException, InterruptedException {
  // Register the view. It is imperative that this step exists,
  // otherwise recorded metrics will be dropped and never exported.
  View view =
      View.create(
          Name.create("task_latency_distribution"),
          "The distribution of the task latencies.",
          LATENCY_MS,
          Aggregation.Distribution.create(LATENCY_BOUNDARIES),
          Collections.<TagKey>emptyList());

  // Create the view manager
  ViewManager viewManager = Stats.getViewManager();

  // Then finally register the views
  viewManager.registerView(view);

  // [START setup_exporter]
  // Enable OpenCensus exporters to export metrics to Stackdriver Monitoring.
  // Exporters use Application Default Credentials to authenticate.
  // See https://developers.google.com/identity/protocols/application-default-credentials
  // for more details.
  StackdriverStatsExporter.createAndRegister();
  // [END setup_exporter]

  // Record 100 fake latency values between 0 and 5 seconds.
  Random rand = new Random();
  for (int i = 0; i < 100; i++) {
    long ms = (long) (TimeUnit.MILLISECONDS.convert(5, TimeUnit.SECONDS) * rand.nextDouble());
    System.out.println(String.format("Latency %d: %d", i, ms));
    STATS_RECORDER.newMeasureMap().put(LATENCY_MS, ms).record();
  }

  // The default export interval is 60 seconds. The thread with the StackdriverStatsExporter must
  // live for at least the interval past any metrics that must be collected, or some risk being
  // lost if they are recorded after the last export.

  System.out.println(
      String.format(
          "Sleeping %d seconds before shutdown to ensure all records are flushed.",
          EXPORT_INTERVAL));
  Thread.sleep(TimeUnit.MILLISECONDS.convert(EXPORT_INTERVAL, TimeUnit.SECONDS));
}
 

@Before
public void unregisterStackdriver() {
  // unregister stackdriver stats exporter in case a previous test already registered one.
  StackdriverStatsExporter.unregister();
}
 

/**
 * Greet server. If provided, the first element of {@code args} is the name to use in the
 * greeting.
 */
public static void main(String[] args) throws IOException, InterruptedException {
  // Add final keyword to pass checkStyle.
  final String user = getStringOrDefaultFromArgs(args, 0, "world");
  final String host = getStringOrDefaultFromArgs(args, 1, "localhost");
  final int serverPort = getPortOrDefaultFromArgs(args, 2, 50051);
  final String cloudProjectId = getStringOrDefaultFromArgs(args, 3, null);
  final int zPagePort = getPortOrDefaultFromArgs(args, 4, 3001);

  // Registers all RPC views. For demonstration all views are registered. You may want to
  // start with registering basic views and register other views as needed for your application.
  RpcViews.registerAllViews();

  // Starts a HTTP server and registers all Zpages to it.
  ZPageHandlers.startHttpServerAndRegisterAll(zPagePort);
  logger.info("ZPages server starts at localhost:" + zPagePort);

  // Registers logging trace exporter.
  LoggingTraceExporter.register();

  // Registers Stackdriver exporters.
  if (cloudProjectId != null) {
    StackdriverTraceExporter.createAndRegister(
        StackdriverTraceConfiguration.builder().setProjectId(cloudProjectId).build());
    StackdriverStatsExporter.createAndRegister(
        StackdriverStatsConfiguration.builder()
            .setProjectId(cloudProjectId)
            .setExportInterval(Duration.create(60, 0))
            .build());
  }

  // Register Prometheus exporters and export metrics to a Prometheus HTTPServer.
  PrometheusStatsCollector.createAndRegister();

  HelloWorldClient client = new HelloWorldClient(host, serverPort);
  try {
    client.greet(user);
  } finally {
    client.shutdown();
  }

  logger.info("Client sleeping, ^C to exit. Meanwhile you can view stats and spans on zpages.");
  while (true) {
    try {
      Thread.sleep(10000);
    } catch (InterruptedException e) {
      logger.info("Exiting HelloWorldClient...");
    }
  }
}
 

@Before
public void unregisterStackdriver() {
  // unregister stackdriver stats exporter in case a previous test already registered one.
  StackdriverStatsExporter.unregister();
}
 

public static void main(String[] args) throws Exception {
  if (args.length != 2) {
    System.err.println("Usage: TracingSample <instance_id> <database_id>");
    return;
  }
  SpannerOptions options = SpannerOptions.newBuilder().build();
  Spanner spanner = options.getService();

  // Installs a handler for /tracez page.
  ZPageHandlers.startHttpServerAndRegisterAll(8080);
  // Installs an exporter for stack driver traces.
  StackdriverExporter.createAndRegister();
  Tracing.getExportComponent()
      .getSampledSpanStore()
      .registerSpanNamesForCollection(Arrays.asList(SAMPLE_SPAN));

  // Installs an exporter for stack driver stats.
  StackdriverStatsExporter.createAndRegister();
  RpcViews.registerAllCumulativeViews();

  // Name of your instance & database.
  String instanceId = args[0];
  String databaseId = args[1];
  try {
    // Creates a database client
    DatabaseClient dbClient =
        spanner.getDatabaseClient(DatabaseId.of(options.getProjectId(), instanceId, databaseId));
    // Queries the database
    try (Scope ss =
        Tracing.getTracer()
            .spanBuilderWithExplicitParent(SAMPLE_SPAN, null)
            .setSampler(Samplers.alwaysSample())
            .startScopedSpan()) {
      ResultSet resultSet = dbClient.singleUse().executeQuery(Statement.of("SELECT 1"));

      System.out.println("\n\nResults:");
      // Prints the results
      while (resultSet.next()) {
        System.out.printf("%d\n\n", resultSet.getLong(0));
      }
    }
  } finally {
    // Closes the client which will free up the resources used
    spanner.close();
  }
}
 

public static void main(String[] args) throws IOException, InterruptedException {
  // Consult system properties to get project/instance
  String projectId = requiredProperty("bigtable.projectID");
  String instanceId = requiredProperty("bigtable.instanceID");

  try {
    // Force tracing for every request for demo purposes.  Use the default settings
    // in most cases.
    Tracing.getTraceConfig().updateActiveTraceParams(
        TraceParams.DEFAULT.toBuilder().setSampler(Samplers.probabilitySampler(1)).build());

    StackdriverTraceExporter.createAndRegister(
        StackdriverTraceConfiguration.builder()
            .setProjectId(projectId)
            .build());

    // Enable stats exporter to Stackdriver with a 5 second export time.
    // Production settings may vary.
    StackdriverStatsExporter.createAndRegister(
          StackdriverStatsConfiguration.builder()
              .setProjectId(projectId)
              .setExportInterval(Duration.create(5, 0))
              .build());

    RpcViews.registerAllGrpcViews();

    // HBase Bigtable specific setup for zpages
    HBaseTracingUtilities.setupTracingConfig();

    // Start a web server on port 8080 for tracing data
    ZPageHandlers.startHttpServerAndRegisterAll(8080);

    doHelloWorld(projectId, instanceId);

  } finally {
    System.out.println("Sleeping for 1 minute so that you can view http://localhost:8080/tracez");

    // Sleep for 1 minute.
    TimeUnit.MINUTES.sleep(1);

    // Terminating this program manually as Http-server prevents it from auto shutdown
    System.exit(0);
  }
}