org.apache.beam.sdk.metrics.MetricResult Java Examples

private static <T> String createNormalizedMetricName(
    MetricResult<T> metric,
    String metricType,
    String valueType,
    CommittedOrAttemped committedOrAttemped) {
  String metricName =
      String.format(
          "beam.%s.%s.%s.%s.%s",
          metricType,
          metric.getName().getNamespace(),
          metric.getName().getName(),
          committedOrAttemped,
          valueType);

  return WHITESPACE.matcher(metricName).replaceAll(SPACE_REPLACEMENT);
}
 

private static <T> void mergeCommittedResults(
    Map<MetricKey, MetricResult<T>> metricResultMap,
    Iterable<MetricUpdate<T>> updates,
    BiFunction<T, T, T> combine) {
  for (MetricUpdate<T> metricUpdate : updates) {
    MetricKey key = metricUpdate.getKey();
    MetricResult<T> current = metricResultMap.get(key);
    if (current == null) {
      throw new IllegalStateException(
          String.format(
              "%s: existing 'attempted' result not found for 'committed' value %s",
              key, metricUpdate.getUpdate()));
    }
    metricResultMap.put(key, current.addCommitted(metricUpdate.getUpdate(), combine));
  }
}
 

private void updateDistributions(Iterable<MetricResult<DistributionResult>> distributions) {
  for (MetricResult<DistributionResult> metricResult : distributions) {
    String flinkMetricName = getFlinkMetricNameString(metricResult.getKey());

    DistributionResult update = metricResult.getAttempted();

    // update flink metric
    FlinkDistributionGauge gauge = flinkDistributionGaugeCache.get(flinkMetricName);
    if (gauge == null) {
      gauge =
          runtimeContext
              .getMetricGroup()
              .gauge(flinkMetricName, new FlinkDistributionGauge(update));
      flinkDistributionGaugeCache.put(flinkMetricName, gauge);
    } else {
      gauge.update(update);
    }
  }
}
 

private void updateGauge(Iterable<MetricResult<GaugeResult>> gauges) {
  for (MetricResult<GaugeResult> metricResult : gauges) {
    String flinkMetricName = getFlinkMetricNameString(metricResult.getKey());

    GaugeResult update = metricResult.getAttempted();

    // update flink metric
    FlinkGauge gauge = flinkGaugeCache.get(flinkMetricName);
    if (gauge == null) {
      gauge = runtimeContext.getMetricGroup().gauge(flinkMetricName, new FlinkGauge(update));
      flinkGaugeCache.put(flinkMetricName, gauge);
    } else {
      gauge.update(update);
    }
  }
}
 

@Override
public MetricQueryResults queryMetrics(@Nullable MetricsFilter filter) {
  List<MetricUpdate> metricUpdates;
  ImmutableList<MetricResult<Long>> counters = ImmutableList.of();
  ImmutableList<MetricResult<DistributionResult>> distributions = ImmutableList.of();
  ImmutableList<MetricResult<GaugeResult>> gauges = ImmutableList.of();
  JobMetrics jobMetrics;
  try {
    jobMetrics = getJobMetrics();
  } catch (IOException e) {
    LOG.warn("Unable to query job metrics.\n");
    return MetricQueryResults.create(counters, distributions, gauges);
  }
  metricUpdates = firstNonNull(jobMetrics.getMetrics(), Collections.emptyList());
  return populateMetricQueryResults(metricUpdates, filter);
}
 

@VisibleForTesting
String renderName(MetricResult<?> metricResult) {
  MetricKey key = metricResult.getKey();
  MetricName name = key.metricName();
  String step = key.stepName();

  ArrayList<String> pieces = new ArrayList<>();

  if (step != null) {
    step = step.replaceAll(ILLEGAL_CHARACTERS, "_");
    if (step.endsWith("_")) {
      step = step.substring(0, step.length() - 1);
    }
    pieces.add(step);
  }

  pieces.addAll(
      ImmutableList.of(name.getNamespace(), name.getName()).stream()
          .map(str -> str.replaceAll(ILLEGAL_CHARACTERS, "_"))
          .collect(toList()));

  return String.join(".", pieces);
}
 

@VisibleForTesting
String renderName(MetricResult<?> metricResult) {
  MetricKey key = metricResult.getKey();
  MetricName name = key.metricName();
  String step = key.stepName();

  ArrayList<String> pieces = new ArrayList<>();

  if (step != null) {
    step = step.replaceAll(ILLEGAL_CHARACTERS, "_");
    if (step.endsWith("_")) {
      step = step.substring(0, step.length() - 1);
    }
    pieces.add(step);
  }

  pieces.addAll(
      ImmutableList.of(name.getNamespace(), name.getName()).stream()
          .map(str -> str.replaceAll(ILLEGAL_CHARACTERS, "_"))
          .collect(toList()));

  return String.join(".", pieces);
}
 

/**
 * Return the current value for a long counter, or -1 if can't be retrieved. Note this uses only
 * attempted metrics because some runners don't support committed metrics.
 */
public long getCounterMetric(String name) {
  MetricQueryResults metrics =
      result
          .metrics()
          .queryMetrics(
              MetricsFilter.builder()
                  .addNameFilter(MetricNameFilter.named(namespace, name))
                  .build());
  Iterable<MetricResult<Long>> counters = metrics.getCounters();

  checkIfMetricResultIsUnique(name, counters);

  try {
    MetricResult<Long> metricResult = counters.iterator().next();
    return metricResult.getAttempted();
  } catch (NoSuchElementException e) {
    LOG.error("Failed to get metric {}, from namespace {}", name, namespace);
  }
  return ERRONEOUS_METRIC_VALUE;
}
 

@Override
public MetricResults metrics() {
  return new MetricResults() {
    @Override
    public MetricQueryResults queryMetrics(@Nullable MetricsFilter filter) {
      return new MetricQueryResults() {
        @Override
        public Iterable<MetricResult<Long>> getCounters() {
          return Collections.emptyList();
        }

        @Override
        public Iterable<MetricResult<DistributionResult>> getDistributions() {
          return Collections.emptyList();
        }

        @Override
        public Iterable<MetricResult<GaugeResult>> getGauges() {
          return Collections.emptyList();
        }
      };
    }
  };
}
 

@Test
@Category(NeedsRunner.class)
public void testWordCountSimple() {
  PCollection<KV<String, Long>> pc =
      pipeline.apply(Create.of(INPUT_STRS)).apply(new CountWords());
  PAssert.that(pc).containsInAnyOrder(KV.of("hello", 2L), KV.of(("world"), 1L));
  PipelineResult result = pipeline.run();
  result.waitUntilFinish();

  Map<String, Long> expectedCounters = new HashMap<>();
  expectedCounters.put("emptyLines", 2L);
  for (MetricResult c :
      result.metrics().queryMetrics(MetricsFilter.builder().build()).getCounters()) {
    String name = c.getName().getName();
    if (expectedCounters.containsKey(name)) {
      assertEquals(expectedCounters.get(name), c.getCommitted());
      expectedCounters.remove(name);
    }
  }
  assertTrue(expectedCounters.isEmpty());
}
 

@Override
public MetricQueryResults queryMetrics(@Nullable MetricsFilter filter) {
  ImmutableList.Builder<MetricResult<Long>> counterResults = ImmutableList.builder();
  for (Entry<MetricKey, DirectMetric<Long, Long>> counter : counters.entries()) {
    maybeExtractResult(filter, counterResults, counter);
  }
  ImmutableList.Builder<MetricResult<DistributionResult>> distributionResults =
      ImmutableList.builder();
  for (Entry<MetricKey, DirectMetric<DistributionData, DistributionResult>> distribution :
      distributions.entries()) {
    maybeExtractResult(filter, distributionResults, distribution);
  }
  ImmutableList.Builder<MetricResult<GaugeResult>> gaugeResults = ImmutableList.builder();
  for (Entry<MetricKey, DirectMetric<GaugeData, GaugeResult>> gauge : gauges.entries()) {
    maybeExtractResult(filter, gaugeResults, gauge);
  }

  return MetricQueryResults.create(
      counterResults.build(), distributionResults.build(), gaugeResults.build());
}
 

private void updateDistributions(Iterable<MetricResult<DistributionResult>> distributions) {
	for (MetricResult<DistributionResult> metricResult : distributions) {
		if (!isUserMetric(metricResult)) {
			continue;
		}
		// get identifier
		String flinkMetricIdentifier = getFlinkMetricIdentifierString(metricResult.getKey());
		DistributionResult update = metricResult.getAttempted();

		// update flink metric
		FlinkDistributionGauge gauge = flinkDistributionGaugeCache.get(flinkMetricIdentifier);
		if (gauge == null) {
			MetricGroup metricGroup =
				registerMetricGroup(metricResult.getKey(), baseMetricGroup);
			gauge = metricGroup.gauge(
				metricResult.getKey().metricName().getName(),
				new FlinkDistributionGauge(update));
			flinkDistributionGaugeCache.put(flinkMetricIdentifier, gauge);
		} else {
			gauge.update(update);
		}
	}
}
 

private void updateGauge(Iterable<MetricResult<GaugeResult>> gauges) {
	for (MetricResult<GaugeResult> metricResult : gauges) {
		if (!isUserMetric(metricResult)) {
			continue;
		}
		// get identifier
		String flinkMetricIdentifier = getFlinkMetricIdentifierString(metricResult.getKey());

		GaugeResult update = metricResult.getAttempted();

		// update flink metric
		FlinkGauge gauge = flinkGaugeCache.get(flinkMetricIdentifier);
		if (gauge == null) {
			MetricGroup metricGroup = registerMetricGroup(metricResult.getKey(), baseMetricGroup);
			gauge = metricGroup.gauge(
				metricResult.getKey().metricName().getName(),
				new FlinkGauge(update));
			flinkGaugeCache.put(flinkMetricIdentifier, gauge);
		} else {
			gauge.update(update);
		}
	}
}
 

@SuppressWarnings("ConstantConditions")
private static <T> void mergeAttemptedResults(
    Map<MetricKey, MetricResult<T>> metricResultMap,
    Iterable<MetricUpdate<T>> updates,
    BiFunction<T, T, T> combine) {
  for (MetricUpdate<T> metricUpdate : updates) {
    MetricKey key = metricUpdate.getKey();
    MetricResult<T> current = metricResultMap.get(key);
    if (current == null) {
      metricResultMap.put(key, MetricResult.attempted(key, metricUpdate.getUpdate()));
    } else {
      metricResultMap.put(key, current.addAttempted(metricUpdate.getUpdate(), combine));
    }
  }
}
 

private QueryResults(
    Iterable<MetricResult<Long>> counters,
    Iterable<MetricResult<DistributionResult>> distributions,
    Iterable<MetricResult<GaugeResult>> gauges) {
  this.counters = counters;
  this.distributions = distributions;
  this.gauges = gauges;
}
 

private static MetricResult<Long> convertCounterMonitoringInfoToCounter(
    MetricsApi.MonitoringInfo counterMonInfo) {
  Map<String, String> labelsMap = counterMonInfo.getLabelsMap();
  MetricKey key =
      MetricKey.create(
          labelsMap.get(STEP_NAME_LABEL),
          MetricName.named(labelsMap.get(NAMESPACE_LABEL), labelsMap.get(METRIC_NAME_LABEL)));
  return MetricResult.create(key, false, decodeInt64Counter(counterMonInfo.getPayload()));
}
 

private static Iterable<MetricResult<Long>> extractCountersFromJobMetrics(
    List<MetricsApi.MonitoringInfo> monitoringInfoList) {
  return monitoringInfoList.stream()
      .filter(item -> SUM_INT64_TYPE.equals(item.getType()))
      .filter(
          item ->
              item.getLabelsMap().get(NAMESPACE_LABEL) != null) // filter out pcollection metrics
      .map(PortableMetrics::convertCounterMonitoringInfoToCounter)
      .collect(Collectors.toList());
}
 

@Test
public void testRenderName() {
  MetricResult<Object> metricResult =
      MetricResult.create(
          MetricKey.create(
              "myStep.one.two(three)", MetricName.named("myNameSpace//", "myName()")),
          123,
          456);
  String renderedName = new SparkBeamMetric().renderName(metricResult);
  assertThat(
      "Metric name was not rendered correctly",
      renderedName,
      equalTo("myStep_one_two_three.myNameSpace__.myName__"));
}
 

@Test
public void testRenderName() {
  MetricResult<Object> metricResult =
      MetricResult.create(
          MetricKey.create(
              "myStep.one.two(three)", MetricName.named("myNameSpace//", "myName()")),
          123,
          456);
  String renderedName = new SparkBeamMetric().renderName(metricResult);
  assertThat(
      "Metric name was not rendered correctly",
      renderedName,
      equalTo("myStep_one_two_three.myNameSpace__.myName__"));
}
 

@Ignore
public void createRunPipeline( PipelineMeta pipelineMeta ) throws Exception {

  /*
  FileOutputStream fos = new FileOutputStream( "/tmp/"+pipelineMeta.getName()+".ktr" );
  fos.write( pipelineMeta.getXML().getBytes() );
  fos.close();
  */

  PipelineOptions pipelineOptions = PipelineOptionsFactory.create();

  pipelineOptions.setJobName( pipelineMeta.getName() );
  pipelineOptions.setUserAgent( BeamConst.STRING_HOP_BEAM );

  BeamDirectPipelineRunConfiguration beamRunConfig = new BeamDirectPipelineRunConfiguration();
  beamRunConfig.setTempLocation( System.getProperty( "java.io.tmpdir" ) );

  // No extra plugins to load : null option
  HopPipelineMetaToBeamPipelineConverter converter = new HopPipelineMetaToBeamPipelineConverter( pipelineMeta, metadataProvider, beamRunConfig );
  Pipeline pipeline = converter.createPipeline();

  PipelineResult pipelineResult = pipeline.run();
  pipelineResult.waitUntilFinish();

  MetricResults metricResults = pipelineResult.metrics();

  MetricQueryResults allResults = metricResults.queryMetrics( MetricsFilter.builder().build() );
  for ( MetricResult<Long> result : allResults.getCounters() ) {
    System.out.println( "Name: " + result.getName() + " Attempted: " + result.getAttempted() );
  }
}
 

private void updateCounters(Iterable<MetricResult<Long>> counters) {
  for (MetricResult<Long> metricResult : counters) {
    String flinkMetricName = getFlinkMetricNameString(metricResult.getKey());

    Long update = metricResult.getAttempted();

    // update flink metric
    Counter counter =
        flinkCounterCache.computeIfAbsent(
            flinkMetricName, n -> runtimeContext.getMetricGroup().counter(n));
    // Beam counters are already pre-aggregated, just update with the current value here
    counter.inc(update - counter.getCount());
  }
}
 

@Override
public void accept(MetricResult<Long> metricResult) {
  final String metricName = getMetricName(metricResult);
  Counter counter = (Counter) getSamzaMetricFor(metricName);
  if (counter == null) {
    counter = metricsRegistry.newCounter(BEAM_METRICS_GROUP, metricName);
  }
  counter.dec(counter.getCount());
  counter.inc(metricResult.getAttempted());
}
 

@Override
public void accept(MetricResult<GaugeResult> metricResult) {
  final String metricName = getMetricName(metricResult);
  @SuppressWarnings("unchecked")
  Gauge<Long> gauge = (Gauge<Long>) getSamzaMetricFor(metricName);
  if (gauge == null) {
    gauge = metricsRegistry.newGauge(BEAM_METRICS_GROUP, metricName, 0L);
  }
  gauge.set(metricResult.getAttempted().getValue());
}
 

@Override
public void serialize(MetricResult value, JsonGenerator gen, SerializerProvider provider)
    throws IOException {
  gen.writeStartObject();
  gen.writeObjectField("attempted", value.getAttempted());
  if (value.hasCommitted()) {
    gen.writeObjectField("committed", value.getCommitted());
  }
  keySerializer.inline(value.getKey(), gen, provider);
  gen.writeEndObject();
}
 

private String serializeMetrics(MetricQueryResults metricQueryResults) throws Exception {
  SimpleModule module = new JodaModule();
  module.addSerializer(new MetricNameSerializer(MetricName.class));
  module.addSerializer(new MetricKeySerializer(MetricKey.class));
  module.addSerializer(new MetricResultSerializer(MetricResult.class));
  objectMapper.registerModule(module);
  objectMapper.disable(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS);
  objectMapper.configure(MapperFeature.SORT_PROPERTIES_ALPHABETICALLY, true);
  // need to register a filter as soon as @JsonFilter annotation is specified.
  // So specify an pass through filter
  SimpleBeanPropertyFilter filter = SimpleBeanPropertyFilter.serializeAll();
  SimpleFilterProvider filterProvider = new SimpleFilterProvider();
  filterProvider.addFilter("committedMetrics", filter);
  objectMapper.setFilterProvider(filterProvider);
  String result;
  try {
    result = objectMapper.writeValueAsString(metricQueryResults);
  } catch (JsonMappingException exception) {
    if ((exception.getCause() instanceof UnsupportedOperationException)
        && exception.getCause().getMessage().contains("committed metrics")) {
      filterProvider.removeFilter("committedMetrics");
      filter = SimpleBeanPropertyFilter.serializeAllExcept("committed");
      filterProvider.addFilter("committedMetrics", filter);
      result = objectMapper.writeValueAsString(metricQueryResults);
    } else {
      throw exception;
    }
  }
  return result;
}
 

@Override
public void writeMetrics(MetricQueryResults metricQueryResults) throws Exception {
  final long metricTimestamp = System.currentTimeMillis() / 1000L;
  Socket socket = new Socket(InetAddress.getByName(address), port);
  BufferedWriter writer =
      new BufferedWriter(new OutputStreamWriter(socket.getOutputStream(), charset));
  StringBuilder messagePayload = new StringBuilder();
  Iterable<MetricResult<Long>> counters = metricQueryResults.getCounters();
  Iterable<MetricResult<GaugeResult>> gauges = metricQueryResults.getGauges();
  Iterable<MetricResult<DistributionResult>> distributions =
      metricQueryResults.getDistributions();

  for (MetricResult<Long> counter : counters) {
    messagePayload.append(new CounterMetricMessage(counter, "value", metricTimestamp).toString());
  }

  for (MetricResult<GaugeResult> gauge : gauges) {
    messagePayload.append(new GaugeMetricMessage(gauge, "value").toString());
  }

  for (MetricResult<DistributionResult> distribution : distributions) {
    messagePayload.append(
        new DistributionMetricMessage(distribution, "min", metricTimestamp).toString());
    messagePayload.append(
        new DistributionMetricMessage(distribution, "max", metricTimestamp).toString());
    messagePayload.append(
        new DistributionMetricMessage(distribution, "count", metricTimestamp).toString());
    messagePayload.append(
        new DistributionMetricMessage(distribution, "sum", metricTimestamp).toString());
    messagePayload.append(
        new DistributionMetricMessage(distribution, "mean", metricTimestamp).toString());
  }
  writer.write(messagePayload.toString());
  writer.flush();
  writer.close();
  socket.close();
}
 

private <T> List<MetricResult<T>> makeResults(
    String step, String name, T committed, T attempted) {
  MetricName metricName = MetricName.named(NAMESPACE, name);
  MetricKey key = MetricKey.create(step, metricName);
  return Collections.singletonList(
      isCommittedSupported
          ? MetricResult.create(key, committed, attempted)
          : MetricResult.attempted(key, attempted));
}
 

@Override
public List<MetricResult<DistributionResult>> getDistributions() {
  return makeResults(
      "s2",
      "n2",
      DistributionResult.create(10L, 2L, 5L, 8L),
      DistributionResult.create(25L, 4L, 3L, 9L));
}
 

@Override
public List<MetricResult<GaugeResult>> getGauges() {
  return makeResults(
      "s3",
      "n3",
      GaugeResult.create(100L, new Instant(345862800L)),
      GaugeResult.create(120L, new Instant(345862800L)));
}
 

private <ResultT> void maybeExtractResult(
    MetricsFilter filter,
    ImmutableList.Builder<MetricResult<ResultT>> resultsBuilder,
    Map.Entry<MetricKey, ? extends DirectMetric<?, ResultT>> entry) {
  if (MetricFiltering.matches(filter, entry.getKey())) {
    resultsBuilder.add(
        MetricResult.create(
            entry.getKey(),
            entry.getValue().extractCommitted(),
            entry.getValue().extractLatestAttempted()));
  }
}