org.apache.beam.sdk.metrics.MetricQueryResults Java Examples

@Override
public MetricResults metrics() {
  return new MetricResults() {
    @Override
    public MetricQueryResults queryMetrics(@Nullable MetricsFilter filter) {
      return new MetricQueryResults() {
        @Override
        public Iterable<MetricResult<Long>> getCounters() {
          return Collections.emptyList();
        }

        @Override
        public Iterable<MetricResult<DistributionResult>> getDistributions() {
          return Collections.emptyList();
        }

        @Override
        public Iterable<MetricResult<GaugeResult>> getGauges() {
          return Collections.emptyList();
        }
      };
    }
  };
}
 

@Override
public MetricQueryResults queryMetrics(@Nullable MetricsFilter filter) {
  ImmutableList.Builder<MetricResult<Long>> counterResults = ImmutableList.builder();
  for (Entry<MetricKey, DirectMetric<Long, Long>> counter : counters.entries()) {
    maybeExtractResult(filter, counterResults, counter);
  }
  ImmutableList.Builder<MetricResult<DistributionResult>> distributionResults =
      ImmutableList.builder();
  for (Entry<MetricKey, DirectMetric<DistributionData, DistributionResult>> distribution :
      distributions.entries()) {
    maybeExtractResult(filter, distributionResults, distribution);
  }
  ImmutableList.Builder<MetricResult<GaugeResult>> gaugeResults = ImmutableList.builder();
  for (Entry<MetricKey, DirectMetric<GaugeData, GaugeResult>> gauge : gauges.entries()) {
    maybeExtractResult(filter, gaugeResults, gauge);
  }

  return MetricQueryResults.create(
      counterResults.build(), distributionResults.build(), gaugeResults.build());
}
 

@Test
public void testWriteMetricsWithCommittedUnSupported() throws Exception {
  MetricQueryResults metricQueryResults = new CustomMetricQueryResults(false);
  MetricsOptions pipelineOptions = PipelineOptionsFactory.create().as(MetricsOptions.class);
  pipelineOptions.setMetricsGraphitePort(port);
  pipelineOptions.setMetricsGraphiteHost("127.0.0.1");
  MetricsGraphiteSink metricsGraphiteSink = new MetricsGraphiteSink(pipelineOptions);
  CountDownLatch countDownLatch = new CountDownLatch(1);
  graphiteServer.setCountDownLatch(countDownLatch);
  metricsGraphiteSink.writeMetrics(metricQueryResults);
  countDownLatch.await();
  String join = String.join("\n", graphiteServer.getMessages());
  String regexpr =
      "beam.counter.ns1.n1.attempted.value 20 [0-9]+\\n"
          + "beam.gauge.ns1.n3.attempted.value 120 [0-9]+\\n"
          + "beam.distribution.ns1.n2.attempted.min 3 [0-9]+\\n"
          + "beam.distribution.ns1.n2.attempted.max 9 [0-9]+\\n"
          + "beam.distribution.ns1.n2.attempted.count 4 [0-9]+\\n"
          + "beam.distribution.ns1.n2.attempted.sum 25 [0-9]+\\n"
          + "beam.distribution.ns1.n2.attempted.mean 6.25 [0-9]+";
  assertTrue(join.matches(regexpr));
}
 

/**
 * Return the current value for a long counter, or -1 if can't be retrieved. Note this uses only
 * attempted metrics because some runners don't support committed metrics.
 */
public long getCounterMetric(String name) {
  MetricQueryResults metrics =
      result
          .metrics()
          .queryMetrics(
              MetricsFilter.builder()
                  .addNameFilter(MetricNameFilter.named(namespace, name))
                  .build());
  Iterable<MetricResult<Long>> counters = metrics.getCounters();

  checkIfMetricResultIsUnique(name, counters);

  try {
    MetricResult<Long> metricResult = counters.iterator().next();
    return metricResult.getAttempted();
  } catch (NoSuchElementException e) {
    LOG.error("Failed to get metric {}, from namespace {}", name, namespace);
  }
  return ERRONEOUS_METRIC_VALUE;
}
 

@Test
public void testWriteMetricsWithCommittedUnSupported() throws Exception {
  MetricQueryResults metricQueryResults = new CustomMetricQueryResults(false);
  MetricsOptions pipelineOptions = PipelineOptionsFactory.create().as(MetricsOptions.class);
  pipelineOptions.setMetricsHttpSinkUrl(String.format("http://localhost:%s", port));
  MetricsHttpSink metricsHttpSink = new MetricsHttpSink(pipelineOptions);
  countDownLatch = new CountDownLatch(1);
  metricsHttpSink.writeMetrics(metricQueryResults);
  countDownLatch.await();
  String expected =
      "{\"counters\":[{\"attempted\":20,\"name\":{\"name\":\"n1\","
          + "\"namespace\":\"ns1\"},\"step\":\"s1\"}],\"distributions\":[{\"attempted\":"
          + "{\"count\":4,\"max\":9,\"mean\":6.25,\"min\":3,\"sum\":25},\"name\":{\"name\":\"n2\""
          + ",\"namespace\":\"ns1\"},\"step\":\"s2\"}],\"gauges\":[{\"attempted\":{\"timestamp\":"
          + "\"1970-01-05T00:04:22.800Z\",\"value\":120},\"name\":{\"name\":\"n3\",\"namespace\":"
          + "\"ns1\"},\"step\":\"s3\"}]}";
  assertEquals("Wrong number of messages sent to HTTP server", 1, messages.size());
  assertEquals("Wrong messages sent to HTTP server", expected, messages.get(0));
}
 

@Test
public void testWriteMetricsWithCommittedSupported() throws Exception {
  MetricQueryResults metricQueryResults = new CustomMetricQueryResults(true);
  MetricsOptions pipelineOptions = PipelineOptionsFactory.create().as(MetricsOptions.class);
  pipelineOptions.setMetricsHttpSinkUrl(String.format("http://localhost:%s", port));
  MetricsHttpSink metricsHttpSink = new MetricsHttpSink(pipelineOptions);
  countDownLatch = new CountDownLatch(1);
  metricsHttpSink.writeMetrics(metricQueryResults);
  countDownLatch.await();
  String expected =
      "{\"counters\":[{\"attempted\":20,\"committed\":10,\"name\":{\"name\":\"n1\","
          + "\"namespace\":\"ns1\"},\"step\":\"s1\"}],\"distributions\":[{\"attempted\":"
          + "{\"count\":4,\"max\":9,\"mean\":6.25,\"min\":3,\"sum\":25},\"committed\":"
          + "{\"count\":2,\"max\":8,\"mean\":5.0,\"min\":5,\"sum\":10},\"name\":{\"name\":\"n2\","
          + "\"namespace\":\"ns1\"},\"step\":\"s2\"}],\"gauges\":[{\"attempted\":{\"timestamp\":"
          + "\"1970-01-05T00:04:22.800Z\",\"value\":120},\"committed\":{\"timestamp\":"
          + "\"1970-01-05T00:04:22.800Z\",\"value\":100},\"name\":{\"name\":\"n3\",\"namespace\":"
          + "\"ns1\"},\"step\":\"s3\"}]}";
  assertEquals("Wrong number of messages sent to HTTP server", 1, messages.size());
  assertEquals("Wrong messages sent to HTTP server", expected, messages.get(0));
}
 

@Override
public MetricQueryResults queryMetrics(@Nullable MetricsFilter filter) {
  List<MetricUpdate> metricUpdates;
  ImmutableList<MetricResult<Long>> counters = ImmutableList.of();
  ImmutableList<MetricResult<DistributionResult>> distributions = ImmutableList.of();
  ImmutableList<MetricResult<GaugeResult>> gauges = ImmutableList.of();
  JobMetrics jobMetrics;
  try {
    jobMetrics = getJobMetrics();
  } catch (IOException e) {
    LOG.warn("Unable to query job metrics.\n");
    return MetricQueryResults.create(counters, distributions, gauges);
  }
  metricUpdates = firstNonNull(jobMetrics.getMetrics(), Collections.emptyList());
  return populateMetricQueryResults(metricUpdates, filter);
}
 

public MetricQueryResults build() {
  buildMetricsIndex();

  DataflowMetricResultExtractor extractor =
      new DataflowMetricResultExtractor(dataflowPipelineJob.getDataflowOptions().isStreaming());
  for (MetricKey metricKey : metricHashKeys) {
    String metricName = metricKey.metricName().getName();
    if (metricName.endsWith("[MIN]")
        || metricName.endsWith("[MAX]")
        || metricName.endsWith("[MEAN]")
        || metricName.endsWith("[COUNT]")) {
      // Skip distribution metrics, as these are not yet properly supported.
      // TODO: remove this when distributions stop being broken up for the UI.
      continue;
    }

    extractor.addMetricResult(
        metricKey, committedByName.get(metricKey), tentativeByName.get(metricKey));
  }
  return MetricQueryResults.create(
      extractor.getCounterResults(),
      extractor.getDistributionResults(),
      extractor.getGaugeResults());
}
 

private void pushMetrics() {
  if (!(metricsSink instanceof NoOpMetricsSink)) {
    try {
      // merge metrics
      MetricResults metricResults = asAttemptedOnlyMetricResults(metricsContainerStepMap);
      MetricQueryResults metricQueryResults = metricResults.allMetrics();
      if ((Iterables.size(metricQueryResults.getDistributions()) != 0)
          || (Iterables.size(metricQueryResults.getGauges()) != 0)
          || (Iterables.size(metricQueryResults.getCounters()) != 0)) {
        metricsSink.writeMetrics(metricQueryResults);
      }

    } catch (Exception e) {
      MetricsPushException metricsPushException = new MetricsPushException(e);
      metricsPushException.printStackTrace();
    }
  }
}
 

/**
 * Writes the metricQueryResults via HTTP POST to metrics sink endpoint.
 *
 * @param metricQueryResults query results to write.
 * @throws Exception throws IOException for non-200 response from endpoint.
 */
@Override
public void writeMetrics(MetricQueryResults metricQueryResults) throws Exception {
  URL url = new URL(urlString);
  String metrics = serializeMetrics(metricQueryResults);
  byte[] postData = metrics.getBytes(StandardCharsets.UTF_8);
  HttpURLConnection connection = (HttpURLConnection) url.openConnection();
  connection.setDoOutput(true);
  connection.setInstanceFollowRedirects(false);
  connection.setRequestMethod("POST");
  connection.setRequestProperty("Content-Type", "application/json");
  connection.setRequestProperty("charset", "utf-8");
  connection.setRequestProperty("Content-Length", Integer.toString(postData.length));
  connection.setUseCaches(false);
  try (DataOutputStream connectionOuputStream =
      new DataOutputStream(connection.getOutputStream())) {
    connectionOuputStream.write(postData);
  }
  int responseCode = connection.getResponseCode();
  if (responseCode != 200) {
    throw new IOException(
        "Expected HTTP 200 OK response while writing metrics to MetricsHttpSink but received "
            + responseCode);
  }
}
 

@Test
public void testEmptyMetricUpdates() throws IOException {
  Job modelJob = new Job();
  modelJob.setCurrentState(State.RUNNING.toString());

  DataflowPipelineJob job = mock(DataflowPipelineJob.class);
  DataflowPipelineOptions options = mock(DataflowPipelineOptions.class);
  when(options.isStreaming()).thenReturn(false);
  when(job.getDataflowOptions()).thenReturn(options);
  when(job.getState()).thenReturn(State.RUNNING);
  job.jobId = JOB_ID;

  JobMetrics jobMetrics = new JobMetrics();
  jobMetrics.setMetrics(null /* this is how the APIs represent empty metrics */);
  DataflowClient dataflowClient = mock(DataflowClient.class);
  when(dataflowClient.getJobMetrics(JOB_ID)).thenReturn(jobMetrics);

  DataflowMetrics dataflowMetrics = new DataflowMetrics(job, dataflowClient);
  MetricQueryResults result = dataflowMetrics.allMetrics();
  assertThat(ImmutableList.copyOf(result.getCounters()), is(empty()));
  assertThat(ImmutableList.copyOf(result.getDistributions()), is(empty()));
}
 

private String serializeMetrics(MetricQueryResults metricQueryResults) throws Exception {
  SimpleModule module = new JodaModule();
  module.addSerializer(new MetricNameSerializer(MetricName.class));
  module.addSerializer(new MetricKeySerializer(MetricKey.class));
  module.addSerializer(new MetricResultSerializer(MetricResult.class));
  objectMapper.registerModule(module);
  objectMapper.disable(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS);
  objectMapper.configure(MapperFeature.SORT_PROPERTIES_ALPHABETICALLY, true);
  // need to register a filter as soon as @JsonFilter annotation is specified.
  // So specify an pass through filter
  SimpleBeanPropertyFilter filter = SimpleBeanPropertyFilter.serializeAll();
  SimpleFilterProvider filterProvider = new SimpleFilterProvider();
  filterProvider.addFilter("committedMetrics", filter);
  objectMapper.setFilterProvider(filterProvider);
  String result;
  try {
    result = objectMapper.writeValueAsString(metricQueryResults);
  } catch (JsonMappingException exception) {
    if ((exception.getCause() instanceof UnsupportedOperationException)
        && exception.getCause().getMessage().contains("committed metrics")) {
      filterProvider.removeFilter("committedMetrics");
      filter = SimpleBeanPropertyFilter.serializeAllExcept("committed");
      filterProvider.addFilter("committedMetrics", filter);
      result = objectMapper.writeValueAsString(metricQueryResults);
    } else {
      throw exception;
    }
  }
  return result;
}
 

@Override
public MetricQueryResults queryMetrics(MetricsFilter filter) {
  return MetricQueryResults.create(
      Iterables.filter(
          this.counters, (counter) -> MetricFiltering.matches(filter, counter.getKey())),
      Iterables.filter(
          this.distributions,
          (distribution) -> MetricFiltering.matches(filter, distribution.getKey())),
      Iterables.filter(this.gauges, (gauge) -> MetricFiltering.matches(filter, gauge.getKey())));
}
 

@Override
public synchronized MetricQueryResults queryMetrics(@Nullable MetricsFilter filter) {
  if (metricsAccumulator != null) {
    updateLocalMetrics(metricsAccumulator);
  }
  return new QueryResults(
      counters.filter(filter), distributions.filter(filter), gauges.filter(filter));
}
 

@Test
@Category(StreamingTest.class)
public void testUnboundedSourceMetrics() {
  final long minElements = 1000;

  // Use a GenerateSequence for the UnboundedSequence, but push the watermark to infinity at
  // minElements to let the test pipeline cleanly shut it down.  Shutdown will occur shortly
  // afterwards, but at least minElements will be reported in the metrics.
  PCollection<Long> pc =
      pipeline.apply(
          GenerateSequence.from(1)
              .withRate(minElements / 10, Duration.millis(500L))
              .withTimestampFn(
                  t -> t < minElements ? Instant.now() : BoundedWindow.TIMESTAMP_MAX_VALUE));
  assertThat(pc.isBounded(), is(PCollection.IsBounded.UNBOUNDED));

  PipelineResult pipelineResult = pipeline.run();

  MetricQueryResults metrics =
      pipelineResult
          .metrics()
          .queryMetrics(
              MetricsFilter.builder()
                  .addNameFilter(
                      MetricNameFilter.named(
                          ELEMENTS_READ.getNamespace(), ELEMENTS_READ.getName()))
                  .build());

  assertThat(
      metrics.getCounters(),
      hasItem(
          metricsResult(
              ELEMENTS_READ.getNamespace(),
              ELEMENTS_READ.getName(),
              "GenerateSequence/Read(UnboundedCountingSource)",
              greaterThanOrEqualTo(minElements),
              false)));
}
 

/**
 * Update Flink's internal metrics ({@link this#flinkCounterCache}) with the latest metrics for a
 * given step.
 */
void updateMetrics(String stepName) {
  MetricResults metricResults = asAttemptedOnlyMetricResults(metricsContainers);
  MetricQueryResults metricQueryResults =
      metricResults.queryMetrics(MetricsFilter.builder().addStep(stepName).build());
  updateCounters(metricQueryResults.getCounters());
  updateDistributions(metricQueryResults.getDistributions());
  updateGauge(metricQueryResults.getGauges());
}
 

@Test
public void testSingleCounterUpdates() throws IOException {
  JobMetrics jobMetrics = new JobMetrics();
  DataflowPipelineJob job = mock(DataflowPipelineJob.class);
  DataflowPipelineOptions options = mock(DataflowPipelineOptions.class);
  when(options.isStreaming()).thenReturn(false);
  when(job.getDataflowOptions()).thenReturn(options);
  when(job.getState()).thenReturn(State.RUNNING);
  job.jobId = JOB_ID;

  AppliedPTransform<?, ?, ?> myStep = mock(AppliedPTransform.class);
  when(myStep.getFullName()).thenReturn("myStepName");
  job.transformStepNames = HashBiMap.create();
  job.transformStepNames.put(myStep, "s2");

  MetricUpdate update = new MetricUpdate();
  long stepValue = 1234L;
  update.setScalar(new BigDecimal(stepValue));

  // The parser relies on the fact that one tentative and one committed metric update exist in
  // the job metrics results.
  MetricUpdate mu1 =
      makeCounterMetricUpdate("counterName", "counterNamespace", "s2", 1234L, false);
  MetricUpdate mu1Tentative =
      makeCounterMetricUpdate("counterName", "counterNamespace", "s2", 1233L, true);
  jobMetrics.setMetrics(ImmutableList.of(mu1, mu1Tentative));
  DataflowClient dataflowClient = mock(DataflowClient.class);
  when(dataflowClient.getJobMetrics(JOB_ID)).thenReturn(jobMetrics);

  DataflowMetrics dataflowMetrics = new DataflowMetrics(job, dataflowClient);
  MetricQueryResults result = dataflowMetrics.allMetrics();
  assertThat(
      result.getCounters(),
      containsInAnyOrder(
          attemptedMetricsResult("counterNamespace", "counterName", "myStepName", 1234L)));
  assertThat(
      result.getCounters(),
      containsInAnyOrder(
          committedMetricsResult("counterNamespace", "counterName", "myStepName", 1234L)));
}
 

@Test
public void testIgnoreDistributionButGetCounterUpdates() throws IOException {
  JobMetrics jobMetrics = new JobMetrics();
  DataflowClient dataflowClient = mock(DataflowClient.class);
  when(dataflowClient.getJobMetrics(JOB_ID)).thenReturn(jobMetrics);
  DataflowPipelineJob job = mock(DataflowPipelineJob.class);
  DataflowPipelineOptions options = mock(DataflowPipelineOptions.class);
  when(options.isStreaming()).thenReturn(false);
  when(job.getDataflowOptions()).thenReturn(options);
  when(job.getState()).thenReturn(State.RUNNING);
  job.jobId = JOB_ID;

  AppliedPTransform<?, ?, ?> myStep = mock(AppliedPTransform.class);
  when(myStep.getFullName()).thenReturn("myStepName");
  job.transformStepNames = HashBiMap.create();
  job.transformStepNames.put(myStep, "s2");

  // The parser relies on the fact that one tentative and one committed metric update exist in
  // the job metrics results.
  jobMetrics.setMetrics(
      ImmutableList.of(
          makeCounterMetricUpdate("counterName", "counterNamespace", "s2", 1233L, false),
          makeCounterMetricUpdate("counterName", "counterNamespace", "s2", 1233L, true),
          makeCounterMetricUpdate("otherCounter[MIN]", "otherNamespace", "s2", 0L, false),
          makeCounterMetricUpdate("otherCounter[MIN]", "otherNamespace", "s2", 0L, true)));

  DataflowMetrics dataflowMetrics = new DataflowMetrics(job, dataflowClient);
  MetricQueryResults result = dataflowMetrics.allMetrics();
  assertThat(
      result.getCounters(),
      containsInAnyOrder(
          attemptedMetricsResult("counterNamespace", "counterName", "myStepName", 1233L)));
  assertThat(
      result.getCounters(),
      containsInAnyOrder(
          committedMetricsResult("counterNamespace", "counterName", "myStepName", 1233L)));
}
 

@Test
public void extractsMetrics() throws Exception {
  JobApi.MetricResults metricResults = generateMetricResults();
  try (CloseableResource<Server> server = createJobServer(JobState.Enum.DONE, metricResults)) {
    PortableRunner runner =
        PortableRunner.create(options, InProcessManagedChannelFactory.create());
    PipelineResult result = runner.run(p);
    result.waitUntilFinish();
    MetricQueryResults metricQueryResults = result.metrics().allMetrics();
    assertThat(
        metricQueryResults.getCounters().iterator().next().getAttempted(), is(COUNTER_VALUE));
    assertThat(
        metricQueryResults.getDistributions().iterator().next().getAttempted().getCount(),
        is(DIST_COUNT));
    assertThat(
        metricQueryResults.getDistributions().iterator().next().getAttempted().getMax(),
        is(DIST_MAX));
    assertThat(
        metricQueryResults.getDistributions().iterator().next().getAttempted().getMin(),
        is(DIST_MIN));
    assertThat(
        metricQueryResults.getDistributions().iterator().next().getAttempted().getSum(),
        is(DIST_SUM));
    assertThat(
        metricQueryResults.getGauges().iterator().next().getAttempted().getValue(),
        is(GAUGE_VALUE));
  }
}
 

@Override
public void writeMetrics(MetricQueryResults metricQueryResults) throws Exception {
  final long metricTimestamp = System.currentTimeMillis() / 1000L;
  Socket socket = new Socket(InetAddress.getByName(address), port);
  BufferedWriter writer =
      new BufferedWriter(new OutputStreamWriter(socket.getOutputStream(), charset));
  StringBuilder messagePayload = new StringBuilder();
  Iterable<MetricResult<Long>> counters = metricQueryResults.getCounters();
  Iterable<MetricResult<GaugeResult>> gauges = metricQueryResults.getGauges();
  Iterable<MetricResult<DistributionResult>> distributions =
      metricQueryResults.getDistributions();

  for (MetricResult<Long> counter : counters) {
    messagePayload.append(new CounterMetricMessage(counter, "value", metricTimestamp).toString());
  }

  for (MetricResult<GaugeResult> gauge : gauges) {
    messagePayload.append(new GaugeMetricMessage(gauge, "value").toString());
  }

  for (MetricResult<DistributionResult> distribution : distributions) {
    messagePayload.append(
        new DistributionMetricMessage(distribution, "min", metricTimestamp).toString());
    messagePayload.append(
        new DistributionMetricMessage(distribution, "max", metricTimestamp).toString());
    messagePayload.append(
        new DistributionMetricMessage(distribution, "count", metricTimestamp).toString());
    messagePayload.append(
        new DistributionMetricMessage(distribution, "sum", metricTimestamp).toString());
    messagePayload.append(
        new DistributionMetricMessage(distribution, "mean", metricTimestamp).toString());
  }
  writer.write(messagePayload.toString());
  writer.flush();
  writer.close();
  socket.close();
}
 

@Test
public void testWriteMetricsWithCommittedSupported() throws Exception {
  MetricQueryResults metricQueryResults = new CustomMetricQueryResults(true);
  MetricsOptions pipelineOptions = PipelineOptionsFactory.create().as(MetricsOptions.class);
  pipelineOptions.setMetricsGraphitePort(port);
  pipelineOptions.setMetricsGraphiteHost("127.0.0.1");
  MetricsGraphiteSink metricsGraphiteSink = new MetricsGraphiteSink(pipelineOptions);
  CountDownLatch countDownLatch = new CountDownLatch(1);
  graphiteServer.setCountDownLatch(countDownLatch);
  metricsGraphiteSink.writeMetrics(metricQueryResults);
  countDownLatch.await();
  String join = String.join("\n", graphiteServer.getMessages());
  String regexpr =
      "beam.counter.ns1.n1.committed.value 10 [0-9]+\\n"
          + "beam.counter.ns1.n1.attempted.value 20 [0-9]+\\n"
          + "beam.gauge.ns1.n3.committed.value 100 [0-9]+\\n"
          + "beam.gauge.ns1.n3.attempted.value 120 [0-9]+\\n"
          + "beam.distribution.ns1.n2.committed.min 5 [0-9]+\\n"
          + "beam.distribution.ns1.n2.attempted.min 3 [0-9]+\\n"
          + "beam.distribution.ns1.n2.committed.max 8 [0-9]+\\n"
          + "beam.distribution.ns1.n2.attempted.max 9 [0-9]+\\n"
          + "beam.distribution.ns1.n2.committed.count 2 [0-9]+\\n"
          + "beam.distribution.ns1.n2.attempted.count 4 [0-9]+\\n"
          + "beam.distribution.ns1.n2.committed.sum 10 [0-9]+\\n"
          + "beam.distribution.ns1.n2.attempted.sum 25 [0-9]+\\n"
          + "beam.distribution.ns1.n2.committed.mean 5.0 [0-9]+\\n"
          + "beam.distribution.ns1.n2.attempted.mean 6.25 [0-9]+";
  assertTrue(join.matches(regexpr));
}
 

@SuppressWarnings("unchecked")
@Test
public void testApplyAttemptedCountersQueryOneNamespace() {
  metrics.updatePhysical(
      bundle1,
      MetricUpdates.create(
          ImmutableList.of(
              MetricUpdate.create(MetricKey.create("step1", NAME1), 5L),
              MetricUpdate.create(MetricKey.create("step1", NAME3), 8L)),
          ImmutableList.of(),
          ImmutableList.of()));
  metrics.updatePhysical(
      bundle1,
      MetricUpdates.create(
          ImmutableList.of(
              MetricUpdate.create(MetricKey.create("step2", NAME1), 7L),
              MetricUpdate.create(MetricKey.create("step1", NAME3), 4L)),
          ImmutableList.of(),
          ImmutableList.of()));

  MetricQueryResults results =
      metrics.queryMetrics(MetricsFilter.builder().addNameFilter(inNamespace("ns1")).build());

  assertThat(
      results.getCounters(),
      containsInAnyOrder(
          attemptedMetricsResult("ns1", "name1", "step1", 5L),
          attemptedMetricsResult("ns1", "name1", "step2", 7L)));

  assertThat(
      results.getCounters(),
      containsInAnyOrder(
          committedMetricsResult("ns1", "name1", "step1", 0L),
          committedMetricsResult("ns1", "name1", "step2", 0L)));
}
 

@SuppressWarnings("unchecked")
@Test
public void testApplyAttemptedQueryCompositeScope() {
  metrics.updatePhysical(
      bundle1,
      MetricUpdates.create(
          ImmutableList.of(
              MetricUpdate.create(MetricKey.create("Outer1/Inner1", NAME1), 5L),
              MetricUpdate.create(MetricKey.create("Outer1/Inner2", NAME1), 8L)),
          ImmutableList.of(),
          ImmutableList.of()));
  metrics.updatePhysical(
      bundle1,
      MetricUpdates.create(
          ImmutableList.of(
              MetricUpdate.create(MetricKey.create("Outer1/Inner1", NAME1), 12L),
              MetricUpdate.create(MetricKey.create("Outer2/Inner2", NAME1), 18L)),
          ImmutableList.of(),
          ImmutableList.of()));

  MetricQueryResults results =
      metrics.queryMetrics(MetricsFilter.builder().addStep("Outer1").build());

  assertThat(
      results.getCounters(),
      containsInAnyOrder(
          attemptedMetricsResult("ns1", "name1", "Outer1/Inner1", 12L),
          attemptedMetricsResult("ns1", "name1", "Outer1/Inner2", 8L)));

  assertThat(
      results.getCounters(),
      containsInAnyOrder(
          committedMetricsResult("ns1", "name1", "Outer1/Inner1", 0L),
          committedMetricsResult("ns1", "name1", "Outer1/Inner2", 0L)));
}
 

@SuppressWarnings("unchecked")
@Test
public void testPartialScopeMatchingInMetricsQuery() {
  metrics.updatePhysical(
      bundle1,
      MetricUpdates.create(
          ImmutableList.of(
              MetricUpdate.create(MetricKey.create("Top1/Outer1/Inner1", NAME1), 5L),
              MetricUpdate.create(MetricKey.create("Top1/Outer1/Inner2", NAME1), 8L)),
          ImmutableList.of(),
          ImmutableList.of()));
  metrics.updatePhysical(
      bundle1,
      MetricUpdates.create(
          ImmutableList.of(
              MetricUpdate.create(MetricKey.create("Top2/Outer1/Inner1", NAME1), 12L),
              MetricUpdate.create(MetricKey.create("Top1/Outer2/Inner2", NAME1), 18L)),
          ImmutableList.of(),
          ImmutableList.of()));

  MetricQueryResults results =
      metrics.queryMetrics(MetricsFilter.builder().addStep("Top1/Outer1").build());

  assertThat(
      results.getCounters(),
      containsInAnyOrder(
          attemptedMetricsResult("ns1", "name1", "Top1/Outer1/Inner1", 5L),
          attemptedMetricsResult("ns1", "name1", "Top1/Outer1/Inner2", 8L)));

  results = metrics.queryMetrics(MetricsFilter.builder().addStep("Inner2").build());

  assertThat(
      results.getCounters(),
      containsInAnyOrder(
          attemptedMetricsResult("ns1", "name1", "Top1/Outer1/Inner2", 8L),
          attemptedMetricsResult("ns1", "name1", "Top1/Outer2/Inner2", 18L)));
}
 

/**
 * Update Flink's internal metrics ({@link #flinkCounterCache}) with the latest metrics for
 * a given step.
 */
private void updateMetrics(String stepName) {
	MetricResults metricResults = asAttemptedOnlyMetricResults(metricsContainers);
	MetricQueryResults metricQueryResults =
		metricResults.queryMetrics(MetricsFilter.builder().addStep(stepName).build());
	updateCounterOrMeter(metricQueryResults.getCounters());
	updateDistributions(metricQueryResults.getDistributions());
	updateGauge(metricQueryResults.getGauges());
}
 

private void assertGauge(
    String name,
    MetricQueryResults metricQueryResults,
    String step,
    GaugeResult expected,
    boolean isCommitted) {
  assertThat(
      metricQueryResults.getGauges(),
      hasItem(metricsResult(NAMESPACE, name, step, expected, isCommitted)));
}
 

@Ignore
public void createRunPipeline( PipelineMeta pipelineMeta ) throws Exception {

  /*
  FileOutputStream fos = new FileOutputStream( "/tmp/"+pipelineMeta.getName()+".ktr" );
  fos.write( pipelineMeta.getXML().getBytes() );
  fos.close();
  */

  PipelineOptions pipelineOptions = PipelineOptionsFactory.create();

  pipelineOptions.setJobName( pipelineMeta.getName() );
  pipelineOptions.setUserAgent( BeamConst.STRING_HOP_BEAM );

  BeamDirectPipelineRunConfiguration beamRunConfig = new BeamDirectPipelineRunConfiguration();
  beamRunConfig.setTempLocation( System.getProperty( "java.io.tmpdir" ) );

  // No extra plugins to load : null option
  HopPipelineMetaToBeamPipelineConverter converter = new HopPipelineMetaToBeamPipelineConverter( pipelineMeta, metadataProvider, beamRunConfig );
  Pipeline pipeline = converter.createPipeline();

  PipelineResult pipelineResult = pipeline.run();
  pipelineResult.waitUntilFinish();

  MetricResults metricResults = pipelineResult.metrics();

  MetricQueryResults allResults = metricResults.queryMetrics( MetricsFilter.builder().build() );
  for ( MetricResult<Long> result : allResults.getCounters() ) {
    System.out.println( "Name: " + result.getName() + " Attempted: " + result.getAttempted() );
  }
}
 

private void logMetrics( PipelineResult pipelineResult ) {
  MetricResults metricResults = pipelineResult.metrics();

  logChannel.logBasic( "  ----------------- Metrics refresh @ " + new SimpleDateFormat( "yyyy/MM/dd HH:mm:ss" ).format( new Date() ) + " -----------------------" );

  MetricQueryResults allResults = metricResults.queryMetrics( MetricsFilter.builder().build() );
  for ( MetricResult<Long> result : allResults.getCounters() ) {
    logChannel.logBasic( "Name: " + result.getName() + " Attempted: " + result.getAttempted() );
  }
}
 

@Ignore
public void createRunPipeline( TransMeta transMeta ) throws Exception {

  /*
  FileOutputStream fos = new FileOutputStream( "/tmp/"+transMeta.getName()+".ktr" );
  fos.write( transMeta.getXML().getBytes() );
  fos.close();
  */

  PipelineOptions pipelineOptions = PipelineOptionsFactory.create();

  pipelineOptions.setJobName( transMeta.getName() );
  pipelineOptions.setUserAgent( BeamConst.STRING_KETTLE_BEAM );

  BeamJobConfig jobConfig = new BeamJobConfig();
  jobConfig.setName("Direct runner test");
  jobConfig.setRunnerTypeName( RunnerType.Direct.name() );

  // No extra plugins to load : null option
  TransMetaPipelineConverter converter = new TransMetaPipelineConverter( transMeta, metaStore, (String) null, jobConfig );
  Pipeline pipeline = converter.createPipeline( pipelineOptions );

  PipelineResult pipelineResult = pipeline.run();
  pipelineResult.waitUntilFinish();

  MetricResults metricResults = pipelineResult.metrics();

  MetricQueryResults allResults = metricResults.queryMetrics( MetricsFilter.builder().build() );
  for ( MetricResult<Long> result : allResults.getCounters() ) {
    System.out.println( "Name: " + result.getName() + " Attempted: " + result.getAttempted() );
  }
}
 

public static Map<String, Long> getMetrics(final PipelineResult result) {
  final MetricQueryResults metricQueryResults =
      result.metrics().queryMetrics(MetricsFilter.builder().build());

  final Map<String, Long> gauges =
      StreamSupport.stream(metricQueryResults.getGauges().spliterator(), false)
          .collect(
              Collectors.groupingBy(
                  MetricResult::getName,
                  Collectors.reducing(
                      GaugeResult.empty(),
                      GET_COMMITTED_GAUGE,
                      BinaryOperator.maxBy(Comparator.comparing(GaugeResult::getTimestamp)))))
          .entrySet()
          .stream()
          .collect(Collectors.toMap(e -> e.getKey().getName(), e -> e.getValue().getValue()));

  final Map<String, Long> counters =
      StreamSupport.stream(metricQueryResults.getCounters().spliterator(), false)
          .collect(
              Collectors.groupingBy(
                  m -> m.getName().getName(), Collectors.summingLong(GET_COMMITTED_COUNTER)));
  Map<String, Long> ret = new HashMap<>();
  ret.putAll(gauges);
  ret.putAll(counters);
  addCalculatedMetrics(counters, ret);
  return Collections.unmodifiableMap(ret);
}