Java Code Examples for org.apache.beam.sdk.metrics.MetricResults#queryMetrics()

The following examples show how to use org.apache.beam.sdk.metrics.MetricResults#queryMetrics() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: PipelineTestBase.java    From hop with Apache License 2.0 5 votes vote down vote up 
@Ignore
public void createRunPipeline( PipelineMeta pipelineMeta ) throws Exception {

  /*
  FileOutputStream fos = new FileOutputStream( "/tmp/"+pipelineMeta.getName()+".ktr" );
  fos.write( pipelineMeta.getXML().getBytes() );
  fos.close();
  */

  PipelineOptions pipelineOptions = PipelineOptionsFactory.create();

  pipelineOptions.setJobName( pipelineMeta.getName() );
  pipelineOptions.setUserAgent( BeamConst.STRING_HOP_BEAM );

  BeamDirectPipelineRunConfiguration beamRunConfig = new BeamDirectPipelineRunConfiguration();
  beamRunConfig.setTempLocation( System.getProperty( "java.io.tmpdir" ) );

  // No extra plugins to load : null option
  HopPipelineMetaToBeamPipelineConverter converter = new HopPipelineMetaToBeamPipelineConverter( pipelineMeta, metadataProvider, beamRunConfig );
  Pipeline pipeline = converter.createPipeline();

  PipelineResult pipelineResult = pipeline.run();
  pipelineResult.waitUntilFinish();

  MetricResults metricResults = pipelineResult.metrics();

  MetricQueryResults allResults = metricResults.queryMetrics( MetricsFilter.builder().build() );
  for ( MetricResult<Long> result : allResults.getCounters() ) {
    System.out.println( "Name: " + result.getName() + " Attempted: " + result.getAttempted() );
  }
}
Example 2
Source File: KettleBeamPipelineExecutor.java    From kettle-beam with Apache License 2.0 5 votes vote down vote up 
private void logMetrics( PipelineResult pipelineResult ) {
  MetricResults metricResults = pipelineResult.metrics();

  logChannel.logBasic( "  ----------------- Metrics refresh @ " + new SimpleDateFormat( "yyyy/MM/dd HH:mm:ss" ).format( new Date() ) + " -----------------------" );

  MetricQueryResults allResults = metricResults.queryMetrics( MetricsFilter.builder().build() );
  for ( MetricResult<Long> result : allResults.getCounters() ) {
    logChannel.logBasic( "Name: " + result.getName() + " Attempted: " + result.getAttempted() );
  }
}
Example 3
Source File: PipelineTestBase.java    From kettle-beam with Apache License 2.0 5 votes vote down vote up 
@Ignore
public void createRunPipeline( TransMeta transMeta ) throws Exception {

  /*
  FileOutputStream fos = new FileOutputStream( "/tmp/"+transMeta.getName()+".ktr" );
  fos.write( transMeta.getXML().getBytes() );
  fos.close();
  */

  PipelineOptions pipelineOptions = PipelineOptionsFactory.create();

  pipelineOptions.setJobName( transMeta.getName() );
  pipelineOptions.setUserAgent( BeamConst.STRING_KETTLE_BEAM );

  BeamJobConfig jobConfig = new BeamJobConfig();
  jobConfig.setName("Direct runner test");
  jobConfig.setRunnerTypeName( RunnerType.Direct.name() );

  // No extra plugins to load : null option
  TransMetaPipelineConverter converter = new TransMetaPipelineConverter( transMeta, metaStore, (String) null, jobConfig );
  Pipeline pipeline = converter.createPipeline( pipelineOptions );

  PipelineResult pipelineResult = pipeline.run();
  pipelineResult.waitUntilFinish();

  MetricResults metricResults = pipelineResult.metrics();

  MetricQueryResults allResults = metricResults.queryMetrics( MetricsFilter.builder().build() );
  for ( MetricResult<Long> result : allResults.getCounters() ) {
    System.out.println( "Name: " + result.getName() + " Attempted: " + result.getAttempted() );
  }
}
Example 4
Source File: MetricsContainerStepMapTest.java    From beam with Apache License 2.0 5 votes vote down vote up 
@Test
public void testCounterCommittedUnsupportedInAttemptedAccumulatedMetricResults() {
  MetricsContainerStepMap attemptedMetrics = new MetricsContainerStepMap();
  attemptedMetrics.update(STEP1, metricsContainer);
  MetricResults metricResults = asAttemptedOnlyMetricResults(attemptedMetrics);

  MetricQueryResults step1res =
      metricResults.queryMetrics(MetricsFilter.builder().addStep(STEP1).build());

  thrown.expect(UnsupportedOperationException.class);
  thrown.expectMessage("This runner does not currently support committed metrics results.");

  assertCounter(COUNTER_NAME, step1res, STEP1, VALUE, true);
}
Example 5
Source File: MetricsContainerStepMapTest.java    From beam with Apache License 2.0 5 votes vote down vote up 
@Test
public void testDistributionCommittedUnsupportedInAttemptedAccumulatedMetricResults() {
  MetricsContainerStepMap attemptedMetrics = new MetricsContainerStepMap();
  attemptedMetrics.update(STEP1, metricsContainer);
  MetricResults metricResults = asAttemptedOnlyMetricResults(attemptedMetrics);

  MetricQueryResults step1res =
      metricResults.queryMetrics(MetricsFilter.builder().addStep(STEP1).build());

  thrown.expect(UnsupportedOperationException.class);
  thrown.expectMessage("This runner does not currently support committed metrics results.");

  assertDistribution(
      DISTRIBUTION_NAME, step1res, STEP1, DistributionResult.IDENTITY_ELEMENT, true);
}
Example 6
Source File: MetricsContainerStepMapTest.java    From beam with Apache License 2.0 5 votes vote down vote up 
@Test
public void testGaugeCommittedUnsupportedInAttemptedAccumulatedMetricResults() {
  MetricsContainerStepMap attemptedMetrics = new MetricsContainerStepMap();
  attemptedMetrics.update(STEP1, metricsContainer);
  MetricResults metricResults = asAttemptedOnlyMetricResults(attemptedMetrics);

  MetricQueryResults step1res =
      metricResults.queryMetrics(MetricsFilter.builder().addStep(STEP1).build());

  thrown.expect(UnsupportedOperationException.class);
  thrown.expectMessage("This runner does not currently support committed metrics results.");

  assertGauge(GAUGE_NAME, step1res, STEP1, GaugeResult.empty(), true);
}
Example 7
Source File: FlinkMetricContainer.java    From beam with Apache License 2.0 5 votes vote down vote up 
/**
 * Update Flink's internal metrics ({@link this#flinkCounterCache}) with the latest metrics for a
 * given step.
 */
void updateMetrics(String stepName) {
  MetricResults metricResults = asAttemptedOnlyMetricResults(metricsContainers);
  MetricQueryResults metricQueryResults =
      metricResults.queryMetrics(MetricsFilter.builder().addStep(stepName).build());
  updateCounters(metricQueryResults.getCounters());
  updateDistributions(metricQueryResults.getDistributions());
  updateGauge(metricQueryResults.getGauges());
}
Example 8
Source File: FlinkMetricContainer.java    From flink with Apache License 2.0 5 votes vote down vote up 
/**
 * Update Flink's internal metrics ({@link #flinkCounterCache}) with the latest metrics for
 * a given step.
 */
private void updateMetrics(String stepName) {
	MetricResults metricResults = asAttemptedOnlyMetricResults(metricsContainers);
	MetricQueryResults metricQueryResults =
		metricResults.queryMetrics(MetricsFilter.builder().addStep(stepName).build());
	updateCounterOrMeter(metricQueryResults.getCounters());
	updateDistributions(metricQueryResults.getDistributions());
	updateGauge(metricQueryResults.getGauges());
}
Example 9
Source File: BeamPipelineEngine.java    From hop with Apache License 2.0 4 votes vote down vote up 
/**
 * Grab the Beam pipeline results and convert it into engine metrics
 */
protected synchronized void populateEngineMetrics() throws HopException {
  ClassLoader oldContextClassLoader = Thread.currentThread().getContextClassLoader();
  try {
    Thread.currentThread().setContextClassLoader( this.getClass().getClassLoader() );

    EngineMetrics em = new EngineMetrics();
    evaluatePipelineStatus();

    em.setStartDate( getExecutionStartDate() );
    em.setEndDate( getExecutionEndDate() );

    if ( beamPipelineResults != null ) {
      Set<String> transformNames = new HashSet<>( Arrays.asList( pipelineMeta.getTransformNames() ) );
      Map<String, EngineComponent> componentsMap = new HashMap<>();
      MetricResults metrics = beamPipelineResults.metrics();
      MetricQueryResults allResults = metrics.queryMetrics( MetricsFilter.builder().build() );

      for ( MetricResult<Long> result : allResults.getCounters() ) {
        String metricsType = result.getName().getNamespace();
        String metricsName = result.getName().getName();
        long processed = result.getAttempted();

        // This is a transform executing in Beam
        //
        if ( transformNames.contains( metricsName ) ) {
          EngineComponent engineComponent = componentsMap.get( metricsName );
          if ( engineComponent == null ) {
            engineComponent = new EngineComponent( metricsName, 0 );
            componentsMap.put( metricsName, engineComponent );
          }
          if ( Pipeline.METRIC_NAME_READ.equalsIgnoreCase( metricsType ) ) {
            engineComponent.setLinesRead( processed );
            em.setComponentMetric( engineComponent, Pipeline.METRIC_READ, processed );
          } else if ( Pipeline.METRIC_NAME_WRITTEN.equalsIgnoreCase( metricsType ) ) {
            engineComponent.setLinesWritten( processed );
            em.setComponentMetric( engineComponent, Pipeline.METRIC_WRITTEN, processed );
          } else if ( Pipeline.METRIC_NAME_INPUT.equalsIgnoreCase( metricsType ) ) {
            engineComponent.setLinesInput( processed );
            em.setComponentMetric( engineComponent, Pipeline.METRIC_INPUT, processed );
          } else if ( Pipeline.METRIC_NAME_OUTPUT.equalsIgnoreCase( metricsType ) ) {
            engineComponent.setLinesOutput( processed );
            em.setComponentMetric( engineComponent, Pipeline.METRIC_OUTPUT, processed );
          } else if ( Pipeline.METRIC_NAME_INIT.equalsIgnoreCase( metricsType ) ) {
            em.setComponentMetric( engineComponent, Pipeline.METRIC_INIT, processed );
          } else if ( Pipeline.METRIC_NAME_FLUSH_BUFFER.equalsIgnoreCase( metricsType ) ) {
            em.setComponentMetric( engineComponent, Pipeline.METRIC_FLUSH_BUFFER, processed );
          }

          // Copy the execution start and end date from the pipeline
          //
          engineComponent.setExecutionStartDate( getExecutionStartDate() );
          engineComponent.setExecutionEndDate( getExecutionEndDate() );
          engineComponent.setExecutionDuration( calculateDuration( getExecutionStartDate(), getExecutionEndDate() ) );

          // Set the transform status to reflect the pipeline status.
          //
          switch ( beamPipelineResults.getState() ) {
            case DONE:
              engineComponent.setRunning( false );
              engineComponent.setStatus( ComponentExecutionStatus.STATUS_FINISHED );
              break;
            case CANCELLED:
            case FAILED:
            case STOPPED:
              engineComponent.setStopped( true );
              engineComponent.setRunning( false );
              engineComponent.setStatus( ComponentExecutionStatus.STATUS_STOPPED );
              break;
            case RUNNING:
              engineComponent.setRunning( true );
              engineComponent.setStopped( false );
              engineComponent.setStatus( ComponentExecutionStatus.STATUS_RUNNING );
              break;
            case UNKNOWN:
              break;
            case UPDATED:
              break;
            default:
              break;
          }
        }
      }

      em.getComponents().clear();
      em.getComponents().addAll( componentsMap.values() );
    }

    // Swap the engine metrics with the new value
    //
    synchronized ( engineMetrics ) {
      engineMetrics = em;
    }
  } finally {
    Thread.currentThread().setContextClassLoader( oldContextClassLoader );
  }
}
Example 10
Source File: MetricsContainerStepMapTest.java    From beam with Apache License 2.0 4 votes vote down vote up 
@Test
public void testAttemptedAccumulatedMetricResults() {
  MetricsContainerStepMap attemptedMetrics = new MetricsContainerStepMap();
  attemptedMetrics.update(STEP1, metricsContainer);
  attemptedMetrics.update(STEP2, metricsContainer);
  attemptedMetrics.update(STEP2, metricsContainer);

  MetricResults metricResults = asAttemptedOnlyMetricResults(attemptedMetrics);

  MetricQueryResults step1res =
      metricResults.queryMetrics(MetricsFilter.builder().addStep(STEP1).build());

  assertIterableSize(step1res.getCounters(), 1);
  assertIterableSize(step1res.getDistributions(), 1);
  assertIterableSize(step1res.getGauges(), 1);

  assertCounter(COUNTER_NAME, step1res, STEP1, VALUE, false);
  assertDistribution(
      DISTRIBUTION_NAME,
      step1res,
      STEP1,
      DistributionResult.create(VALUE * 3, 2, VALUE, VALUE * 2),
      false);
  assertGauge(GAUGE_NAME, step1res, STEP1, GaugeResult.create(VALUE, Instant.now()), false);

  MetricQueryResults step2res =
      metricResults.queryMetrics(MetricsFilter.builder().addStep(STEP2).build());

  assertIterableSize(step2res.getCounters(), 1);
  assertIterableSize(step2res.getDistributions(), 1);
  assertIterableSize(step2res.getGauges(), 1);

  assertCounter(COUNTER_NAME, step2res, STEP2, VALUE * 2, false);
  assertDistribution(
      DISTRIBUTION_NAME,
      step2res,
      STEP2,
      DistributionResult.create(VALUE * 6, 4, VALUE, VALUE * 2),
      false);
  assertGauge(GAUGE_NAME, step2res, STEP2, GaugeResult.create(VALUE, Instant.now()), false);

  MetricQueryResults allres = metricResults.allMetrics();

  assertIterableSize(allres.getCounters(), 2);
  assertIterableSize(allres.getDistributions(), 2);
  assertIterableSize(allres.getGauges(), 2);
}
Example 11
Source File: MetricsContainerStepMapTest.java    From beam with Apache License 2.0 4 votes vote down vote up 
@Test
public void testAttemptedAndCommittedAccumulatedMetricResults() {
  MetricsContainerStepMap attemptedMetrics = new MetricsContainerStepMap();
  attemptedMetrics.update(STEP1, metricsContainer);
  attemptedMetrics.update(STEP1, metricsContainer);
  attemptedMetrics.update(STEP2, metricsContainer);
  attemptedMetrics.update(STEP2, metricsContainer);
  attemptedMetrics.update(STEP2, metricsContainer);

  MetricsContainerStepMap committedMetrics = new MetricsContainerStepMap();
  committedMetrics.update(STEP1, metricsContainer);
  committedMetrics.update(STEP2, metricsContainer);
  committedMetrics.update(STEP2, metricsContainer);

  MetricResults metricResults = asMetricResults(attemptedMetrics, committedMetrics);

  MetricQueryResults step1res =
      metricResults.queryMetrics(MetricsFilter.builder().addStep(STEP1).build());

  assertIterableSize(step1res.getCounters(), 1);
  assertIterableSize(step1res.getDistributions(), 1);
  assertIterableSize(step1res.getGauges(), 1);

  assertCounter(COUNTER_NAME, step1res, STEP1, VALUE * 2, false);
  assertDistribution(
      DISTRIBUTION_NAME,
      step1res,
      STEP1,
      DistributionResult.create(VALUE * 6, 4, VALUE, VALUE * 2),
      false);
  assertGauge(GAUGE_NAME, step1res, STEP1, GaugeResult.create(VALUE, Instant.now()), false);

  assertCounter(COUNTER_NAME, step1res, STEP1, VALUE, true);
  assertDistribution(
      DISTRIBUTION_NAME,
      step1res,
      STEP1,
      DistributionResult.create(VALUE * 3, 2, VALUE, VALUE * 2),
      true);
  assertGauge(GAUGE_NAME, step1res, STEP1, GaugeResult.create(VALUE, Instant.now()), true);

  MetricQueryResults step2res =
      metricResults.queryMetrics(MetricsFilter.builder().addStep(STEP2).build());

  assertIterableSize(step2res.getCounters(), 1);
  assertIterableSize(step2res.getDistributions(), 1);
  assertIterableSize(step2res.getGauges(), 1);

  assertCounter(COUNTER_NAME, step2res, STEP2, VALUE * 3, false);
  assertDistribution(
      DISTRIBUTION_NAME,
      step2res,
      STEP2,
      DistributionResult.create(VALUE * 9, 6, VALUE, VALUE * 2),
      false);
  assertGauge(GAUGE_NAME, step2res, STEP2, GaugeResult.create(VALUE, Instant.now()), false);

  assertCounter(COUNTER_NAME, step2res, STEP2, VALUE * 2, true);
  assertDistribution(
      DISTRIBUTION_NAME,
      step2res,
      STEP2,
      DistributionResult.create(VALUE * 6, 4, VALUE, VALUE * 2),
      true);
  assertGauge(GAUGE_NAME, step2res, STEP2, GaugeResult.create(VALUE, Instant.now()), true);

  MetricQueryResults allres = metricResults.queryMetrics(MetricsFilter.builder().build());

  assertIterableSize(allres.getCounters(), 2);
  assertIterableSize(allres.getDistributions(), 2);
  assertIterableSize(allres.getGauges(), 2);
}