Java Code Examples for org.apache.beam.sdk.metrics.MetricResult#getAttempted()

The following examples show how to use org.apache.beam.sdk.metrics.MetricResult#getAttempted() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example 1

Source File: MetricsReader.java From beam with Apache License 2.0

6 votes

/**
 * Return the current value for a long counter, or -1 if can't be retrieved. Note this uses only
 * attempted metrics because some runners don't support committed metrics.
 */
public long getCounterMetric(String name) {
  MetricQueryResults metrics =
      result
          .metrics()
          .queryMetrics(
              MetricsFilter.builder()
                  .addNameFilter(MetricNameFilter.named(namespace, name))
                  .build());
  Iterable<MetricResult<Long>> counters = metrics.getCounters();

  checkIfMetricResultIsUnique(name, counters);

  try {
    MetricResult<Long> metricResult = counters.iterator().next();
    return metricResult.getAttempted();
  } catch (NoSuchElementException e) {
    LOG.error("Failed to get metric {}, from namespace {}", name, namespace);
  }
  return ERRONEOUS_METRIC_VALUE;
}

Example 2

Source File: FlinkMetricContainer.java From beam with Apache License 2.0

6 votes

private void updateDistributions(Iterable<MetricResult<DistributionResult>> distributions) {
  for (MetricResult<DistributionResult> metricResult : distributions) {
    String flinkMetricName = getFlinkMetricNameString(metricResult.getKey());

    DistributionResult update = metricResult.getAttempted();

    // update flink metric
    FlinkDistributionGauge gauge = flinkDistributionGaugeCache.get(flinkMetricName);
    if (gauge == null) {
      gauge =
          runtimeContext
              .getMetricGroup()
              .gauge(flinkMetricName, new FlinkDistributionGauge(update));
      flinkDistributionGaugeCache.put(flinkMetricName, gauge);
    } else {
      gauge.update(update);
    }
  }
}

Example 3

Source File: FlinkMetricContainer.java From beam with Apache License 2.0

6 votes

private void updateGauge(Iterable<MetricResult<GaugeResult>> gauges) {
  for (MetricResult<GaugeResult> metricResult : gauges) {
    String flinkMetricName = getFlinkMetricNameString(metricResult.getKey());

    GaugeResult update = metricResult.getAttempted();

    // update flink metric
    FlinkGauge gauge = flinkGaugeCache.get(flinkMetricName);
    if (gauge == null) {
      gauge = runtimeContext.getMetricGroup().gauge(flinkMetricName, new FlinkGauge(update));
      flinkGaugeCache.put(flinkMetricName, gauge);
    } else {
      gauge.update(update);
    }
  }
}

Example 4

Source File: FlinkMetricContainer.java From flink with Apache License 2.0

6 votes

private void updateDistributions(Iterable<MetricResult<DistributionResult>> distributions) {
	for (MetricResult<DistributionResult> metricResult : distributions) {
		if (!isUserMetric(metricResult)) {
			continue;
		}
		// get identifier
		String flinkMetricIdentifier = getFlinkMetricIdentifierString(metricResult.getKey());
		DistributionResult update = metricResult.getAttempted();

		// update flink metric
		FlinkDistributionGauge gauge = flinkDistributionGaugeCache.get(flinkMetricIdentifier);
		if (gauge == null) {
			MetricGroup metricGroup =
				registerMetricGroup(metricResult.getKey(), baseMetricGroup);
			gauge = metricGroup.gauge(
				metricResult.getKey().metricName().getName(),
				new FlinkDistributionGauge(update));
			flinkDistributionGaugeCache.put(flinkMetricIdentifier, gauge);
		} else {
			gauge.update(update);
		}
	}
}

Example 5

Source File: FlinkMetricContainer.java From flink with Apache License 2.0

6 votes

private void updateGauge(Iterable<MetricResult<GaugeResult>> gauges) {
	for (MetricResult<GaugeResult> metricResult : gauges) {
		if (!isUserMetric(metricResult)) {
			continue;
		}
		// get identifier
		String flinkMetricIdentifier = getFlinkMetricIdentifierString(metricResult.getKey());

		GaugeResult update = metricResult.getAttempted();

		// update flink metric
		FlinkGauge gauge = flinkGaugeCache.get(flinkMetricIdentifier);
		if (gauge == null) {
			MetricGroup metricGroup = registerMetricGroup(metricResult.getKey(), baseMetricGroup);
			gauge = metricGroup.gauge(
				metricResult.getKey().metricName().getName(),
				new FlinkGauge(update));
			flinkGaugeCache.put(flinkMetricIdentifier, gauge);
		} else {
			gauge.update(update);
		}
	}
}

Example 6

Source File: TestPipeline.java From beam with Apache License 2.0

5 votes

/**
 * Verifies all {{@link PAssert PAsserts}} in the pipeline have been executed and were successful.
 *
 * <p>Note this only runs for runners which support Metrics. Runners which do not should verify
 * this in some other way. See: https://issues.apache.org/jira/browse/BEAM-2001
 */
public static void verifyPAssertsSucceeded(Pipeline pipeline, PipelineResult pipelineResult) {
  if (MetricsEnvironment.isMetricsSupported()) {
    long expectedNumberOfAssertions = (long) PAssert.countAsserts(pipeline);

    long successfulAssertions = 0;
    Iterable<MetricResult<Long>> successCounterResults =
        pipelineResult
            .metrics()
            .queryMetrics(
                MetricsFilter.builder()
                    .addNameFilter(MetricNameFilter.named(PAssert.class, PAssert.SUCCESS_COUNTER))
                    .build())
            .getCounters();
    for (MetricResult<Long> counter : successCounterResults) {
      if (counter.getAttempted() > 0) {
        successfulAssertions++;
      }
    }

    assertThat(
        String.format(
            "Expected %d successful assertions, but found %d.",
            expectedNumberOfAssertions, successfulAssertions),
        successfulAssertions,
        is(expectedNumberOfAssertions));
  }
}

Example 7

Source File: TestMetricsSink.java From beam with Apache License 2.0

5 votes

public static long getCounterValue(String counterName) {
  for (MetricResult<Long> metricResult : metricQueryResults.getCounters()) {
    if (metricResult.getName().getName().equals(counterName)) {
      return metricResult.getAttempted();
    }
  }
  return 0L;
}

Example 8

Source File: FlinkMetricContainer.java From beam with Apache License 2.0

5 votes

private void updateCounters(Iterable<MetricResult<Long>> counters) {
  for (MetricResult<Long> metricResult : counters) {
    String flinkMetricName = getFlinkMetricNameString(metricResult.getKey());

    Long update = metricResult.getAttempted();

    // update flink metric
    Counter counter =
        flinkCounterCache.computeIfAbsent(
            flinkMetricName, n -> runtimeContext.getMetricGroup().counter(n));
    // Beam counters are already pre-aggregated, just update with the current value here
    counter.inc(update - counter.getCount());
  }
}

Example 9

Source File: BeamPipelineEngine.java From hop with Apache License 2.0

4 votes

/**
 * Grab the Beam pipeline results and convert it into engine metrics
 */
protected synchronized void populateEngineMetrics() throws HopException {
  ClassLoader oldContextClassLoader = Thread.currentThread().getContextClassLoader();
  try {
    Thread.currentThread().setContextClassLoader( this.getClass().getClassLoader() );

    EngineMetrics em = new EngineMetrics();
    evaluatePipelineStatus();

    em.setStartDate( getExecutionStartDate() );
    em.setEndDate( getExecutionEndDate() );

    if ( beamPipelineResults != null ) {
      Set<String> transformNames = new HashSet<>( Arrays.asList( pipelineMeta.getTransformNames() ) );
      Map<String, EngineComponent> componentsMap = new HashMap<>();
      MetricResults metrics = beamPipelineResults.metrics();
      MetricQueryResults allResults = metrics.queryMetrics( MetricsFilter.builder().build() );

      for ( MetricResult<Long> result : allResults.getCounters() ) {
        String metricsType = result.getName().getNamespace();
        String metricsName = result.getName().getName();
        long processed = result.getAttempted();

        // This is a transform executing in Beam
        //
        if ( transformNames.contains( metricsName ) ) {
          EngineComponent engineComponent = componentsMap.get( metricsName );
          if ( engineComponent == null ) {
            engineComponent = new EngineComponent( metricsName, 0 );
            componentsMap.put( metricsName, engineComponent );
          }
          if ( Pipeline.METRIC_NAME_READ.equalsIgnoreCase( metricsType ) ) {
            engineComponent.setLinesRead( processed );
            em.setComponentMetric( engineComponent, Pipeline.METRIC_READ, processed );
          } else if ( Pipeline.METRIC_NAME_WRITTEN.equalsIgnoreCase( metricsType ) ) {
            engineComponent.setLinesWritten( processed );
            em.setComponentMetric( engineComponent, Pipeline.METRIC_WRITTEN, processed );
          } else if ( Pipeline.METRIC_NAME_INPUT.equalsIgnoreCase( metricsType ) ) {
            engineComponent.setLinesInput( processed );
            em.setComponentMetric( engineComponent, Pipeline.METRIC_INPUT, processed );
          } else if ( Pipeline.METRIC_NAME_OUTPUT.equalsIgnoreCase( metricsType ) ) {
            engineComponent.setLinesOutput( processed );
            em.setComponentMetric( engineComponent, Pipeline.METRIC_OUTPUT, processed );
          } else if ( Pipeline.METRIC_NAME_INIT.equalsIgnoreCase( metricsType ) ) {
            em.setComponentMetric( engineComponent, Pipeline.METRIC_INIT, processed );
          } else if ( Pipeline.METRIC_NAME_FLUSH_BUFFER.equalsIgnoreCase( metricsType ) ) {
            em.setComponentMetric( engineComponent, Pipeline.METRIC_FLUSH_BUFFER, processed );
          }

          // Copy the execution start and end date from the pipeline
          //
          engineComponent.setExecutionStartDate( getExecutionStartDate() );
          engineComponent.setExecutionEndDate( getExecutionEndDate() );
          engineComponent.setExecutionDuration( calculateDuration( getExecutionStartDate(), getExecutionEndDate() ) );

          // Set the transform status to reflect the pipeline status.
          //
          switch ( beamPipelineResults.getState() ) {
            case DONE:
              engineComponent.setRunning( false );
              engineComponent.setStatus( ComponentExecutionStatus.STATUS_FINISHED );
              break;
            case CANCELLED:
            case FAILED:
            case STOPPED:
              engineComponent.setStopped( true );
              engineComponent.setRunning( false );
              engineComponent.setStatus( ComponentExecutionStatus.STATUS_STOPPED );
              break;
            case RUNNING:
              engineComponent.setRunning( true );
              engineComponent.setStopped( false );
              engineComponent.setStatus( ComponentExecutionStatus.STATUS_RUNNING );
              break;
            case UNKNOWN:
              break;
            case UPDATED:
              break;
            default:
              break;
          }
        }
      }

      em.getComponents().clear();
      em.getComponents().addAll( componentsMap.values() );
    }

    // Swap the engine metrics with the new value
    //
    synchronized ( engineMetrics ) {
      engineMetrics = em;
    }
  } finally {
    Thread.currentThread().setContextClassLoader( oldContextClassLoader );
  }
}