Java Code Examples for org.apache.beam.sdk.metrics.MetricQueryResults#getCounters()

/**
 * Return the current value for a long counter, or -1 if can't be retrieved. Note this uses only
 * attempted metrics because some runners don't support committed metrics.
 */
public long getCounterMetric(String name) {
  MetricQueryResults metrics =
      result
          .metrics()
          .queryMetrics(
              MetricsFilter.builder()
                  .addNameFilter(MetricNameFilter.named(namespace, name))
                  .build());
  Iterable<MetricResult<Long>> counters = metrics.getCounters();

  checkIfMetricResultIsUnique(name, counters);

  try {
    MetricResult<Long> metricResult = counters.iterator().next();
    return metricResult.getAttempted();
  } catch (NoSuchElementException e) {
    LOG.error("Failed to get metric {}, from namespace {}", name, namespace);
  }
  return ERRONEOUS_METRIC_VALUE;
}
 

@Ignore
public void createRunPipeline( PipelineMeta pipelineMeta ) throws Exception {

  /*
  FileOutputStream fos = new FileOutputStream( "/tmp/"+pipelineMeta.getName()+".ktr" );
  fos.write( pipelineMeta.getXML().getBytes() );
  fos.close();
  */

  PipelineOptions pipelineOptions = PipelineOptionsFactory.create();

  pipelineOptions.setJobName( pipelineMeta.getName() );
  pipelineOptions.setUserAgent( BeamConst.STRING_HOP_BEAM );

  BeamDirectPipelineRunConfiguration beamRunConfig = new BeamDirectPipelineRunConfiguration();
  beamRunConfig.setTempLocation( System.getProperty( "java.io.tmpdir" ) );

  // No extra plugins to load : null option
  HopPipelineMetaToBeamPipelineConverter converter = new HopPipelineMetaToBeamPipelineConverter( pipelineMeta, metadataProvider, beamRunConfig );
  Pipeline pipeline = converter.createPipeline();

  PipelineResult pipelineResult = pipeline.run();
  pipelineResult.waitUntilFinish();

  MetricResults metricResults = pipelineResult.metrics();

  MetricQueryResults allResults = metricResults.queryMetrics( MetricsFilter.builder().build() );
  for ( MetricResult<Long> result : allResults.getCounters() ) {
    System.out.println( "Name: " + result.getName() + " Attempted: " + result.getAttempted() );
  }
}
 

private void logMetrics( PipelineResult pipelineResult ) {
  MetricResults metricResults = pipelineResult.metrics();

  logChannel.logBasic( "  ----------------- Metrics refresh @ " + new SimpleDateFormat( "yyyy/MM/dd HH:mm:ss" ).format( new Date() ) + " -----------------------" );

  MetricQueryResults allResults = metricResults.queryMetrics( MetricsFilter.builder().build() );
  for ( MetricResult<Long> result : allResults.getCounters() ) {
    logChannel.logBasic( "Name: " + result.getName() + " Attempted: " + result.getAttempted() );
  }
}
 

@Ignore
public void createRunPipeline( TransMeta transMeta ) throws Exception {

  /*
  FileOutputStream fos = new FileOutputStream( "/tmp/"+transMeta.getName()+".ktr" );
  fos.write( transMeta.getXML().getBytes() );
  fos.close();
  */

  PipelineOptions pipelineOptions = PipelineOptionsFactory.create();

  pipelineOptions.setJobName( transMeta.getName() );
  pipelineOptions.setUserAgent( BeamConst.STRING_KETTLE_BEAM );

  BeamJobConfig jobConfig = new BeamJobConfig();
  jobConfig.setName("Direct runner test");
  jobConfig.setRunnerTypeName( RunnerType.Direct.name() );

  // No extra plugins to load : null option
  TransMetaPipelineConverter converter = new TransMetaPipelineConverter( transMeta, metaStore, (String) null, jobConfig );
  Pipeline pipeline = converter.createPipeline( pipelineOptions );

  PipelineResult pipelineResult = pipeline.run();
  pipelineResult.waitUntilFinish();

  MetricResults metricResults = pipelineResult.metrics();

  MetricQueryResults allResults = metricResults.queryMetrics( MetricsFilter.builder().build() );
  for ( MetricResult<Long> result : allResults.getCounters() ) {
    System.out.println( "Name: " + result.getName() + " Attempted: " + result.getAttempted() );
  }
}
 

@Override
public void writeMetrics(MetricQueryResults metricQueryResults) throws Exception {
  final long metricTimestamp = System.currentTimeMillis() / 1000L;
  Socket socket = new Socket(InetAddress.getByName(address), port);
  BufferedWriter writer =
      new BufferedWriter(new OutputStreamWriter(socket.getOutputStream(), charset));
  StringBuilder messagePayload = new StringBuilder();
  Iterable<MetricResult<Long>> counters = metricQueryResults.getCounters();
  Iterable<MetricResult<GaugeResult>> gauges = metricQueryResults.getGauges();
  Iterable<MetricResult<DistributionResult>> distributions =
      metricQueryResults.getDistributions();

  for (MetricResult<Long> counter : counters) {
    messagePayload.append(new CounterMetricMessage(counter, "value", metricTimestamp).toString());
  }

  for (MetricResult<GaugeResult> gauge : gauges) {
    messagePayload.append(new GaugeMetricMessage(gauge, "value").toString());
  }

  for (MetricResult<DistributionResult> distribution : distributions) {
    messagePayload.append(
        new DistributionMetricMessage(distribution, "min", metricTimestamp).toString());
    messagePayload.append(
        new DistributionMetricMessage(distribution, "max", metricTimestamp).toString());
    messagePayload.append(
        new DistributionMetricMessage(distribution, "count", metricTimestamp).toString());
    messagePayload.append(
        new DistributionMetricMessage(distribution, "sum", metricTimestamp).toString());
    messagePayload.append(
        new DistributionMetricMessage(distribution, "mean", metricTimestamp).toString());
  }
  writer.write(messagePayload.toString());
  writer.flush();
  writer.close();
  socket.close();
}
 

/**
 * Grab the Beam pipeline results and convert it into engine metrics
 */
protected synchronized void populateEngineMetrics() throws HopException {
  ClassLoader oldContextClassLoader = Thread.currentThread().getContextClassLoader();
  try {
    Thread.currentThread().setContextClassLoader( this.getClass().getClassLoader() );

    EngineMetrics em = new EngineMetrics();
    evaluatePipelineStatus();

    em.setStartDate( getExecutionStartDate() );
    em.setEndDate( getExecutionEndDate() );

    if ( beamPipelineResults != null ) {
      Set<String> transformNames = new HashSet<>( Arrays.asList( pipelineMeta.getTransformNames() ) );
      Map<String, EngineComponent> componentsMap = new HashMap<>();
      MetricResults metrics = beamPipelineResults.metrics();
      MetricQueryResults allResults = metrics.queryMetrics( MetricsFilter.builder().build() );

      for ( MetricResult<Long> result : allResults.getCounters() ) {
        String metricsType = result.getName().getNamespace();
        String metricsName = result.getName().getName();
        long processed = result.getAttempted();

        // This is a transform executing in Beam
        //
        if ( transformNames.contains( metricsName ) ) {
          EngineComponent engineComponent = componentsMap.get( metricsName );
          if ( engineComponent == null ) {
            engineComponent = new EngineComponent( metricsName, 0 );
            componentsMap.put( metricsName, engineComponent );
          }
          if ( Pipeline.METRIC_NAME_READ.equalsIgnoreCase( metricsType ) ) {
            engineComponent.setLinesRead( processed );
            em.setComponentMetric( engineComponent, Pipeline.METRIC_READ, processed );
          } else if ( Pipeline.METRIC_NAME_WRITTEN.equalsIgnoreCase( metricsType ) ) {
            engineComponent.setLinesWritten( processed );
            em.setComponentMetric( engineComponent, Pipeline.METRIC_WRITTEN, processed );
          } else if ( Pipeline.METRIC_NAME_INPUT.equalsIgnoreCase( metricsType ) ) {
            engineComponent.setLinesInput( processed );
            em.setComponentMetric( engineComponent, Pipeline.METRIC_INPUT, processed );
          } else if ( Pipeline.METRIC_NAME_OUTPUT.equalsIgnoreCase( metricsType ) ) {
            engineComponent.setLinesOutput( processed );
            em.setComponentMetric( engineComponent, Pipeline.METRIC_OUTPUT, processed );
          } else if ( Pipeline.METRIC_NAME_INIT.equalsIgnoreCase( metricsType ) ) {
            em.setComponentMetric( engineComponent, Pipeline.METRIC_INIT, processed );
          } else if ( Pipeline.METRIC_NAME_FLUSH_BUFFER.equalsIgnoreCase( metricsType ) ) {
            em.setComponentMetric( engineComponent, Pipeline.METRIC_FLUSH_BUFFER, processed );
          }

          // Copy the execution start and end date from the pipeline
          //
          engineComponent.setExecutionStartDate( getExecutionStartDate() );
          engineComponent.setExecutionEndDate( getExecutionEndDate() );
          engineComponent.setExecutionDuration( calculateDuration( getExecutionStartDate(), getExecutionEndDate() ) );

          // Set the transform status to reflect the pipeline status.
          //
          switch ( beamPipelineResults.getState() ) {
            case DONE:
              engineComponent.setRunning( false );
              engineComponent.setStatus( ComponentExecutionStatus.STATUS_FINISHED );
              break;
            case CANCELLED:
            case FAILED:
            case STOPPED:
              engineComponent.setStopped( true );
              engineComponent.setRunning( false );
              engineComponent.setStatus( ComponentExecutionStatus.STATUS_STOPPED );
              break;
            case RUNNING:
              engineComponent.setRunning( true );
              engineComponent.setStopped( false );
              engineComponent.setStatus( ComponentExecutionStatus.STATUS_RUNNING );
              break;
            case UNKNOWN:
              break;
            case UPDATED:
              break;
            default:
              break;
          }
        }
      }

      em.getComponents().clear();
      em.getComponents().addAll( componentsMap.values() );
    }

    // Swap the engine metrics with the new value
    //
    synchronized ( engineMetrics ) {
      engineMetrics = em;
    }
  } finally {
    Thread.currentThread().setContextClassLoader( oldContextClassLoader );
  }
}
 

@Test
public void testUnboundedSourceMetrics() {
  int numElements = 1000;

  String readStep = "readFromKafka";

  p.apply(
      readStep,
      mkKafkaReadTransform(numElements, new ValueAsTimestampFn())
          .withConsumerConfigUpdates(
              ImmutableMap.of(ConsumerConfig.GROUP_ID_CONFIG, "test.group"))
          .commitOffsetsInFinalize()
          .withoutMetadata());

  PipelineResult result = p.run();

  String splitId = "0";

  MetricName elementsRead = SourceMetrics.elementsRead().getName();
  MetricName elementsReadBySplit = SourceMetrics.elementsReadBySplit(splitId).getName();
  MetricName bytesRead = SourceMetrics.bytesRead().getName();
  MetricName bytesReadBySplit = SourceMetrics.bytesReadBySplit(splitId).getName();
  MetricName backlogElementsOfSplit = SourceMetrics.backlogElementsOfSplit(splitId).getName();
  MetricName backlogBytesOfSplit = SourceMetrics.backlogBytesOfSplit(splitId).getName();

  MetricQueryResults metrics = result.metrics().allMetrics();

  Iterable<MetricResult<Long>> counters = metrics.getCounters();

  assertThat(
      counters,
      hasItem(
          attemptedMetricsResult(
              elementsRead.getNamespace(), elementsRead.getName(), readStep, 1000L)));

  assertThat(
      counters,
      hasItem(
          attemptedMetricsResult(
              elementsReadBySplit.getNamespace(),
              elementsReadBySplit.getName(),
              readStep,
              1000L)));

  assertThat(
      counters,
      hasItem(
          attemptedMetricsResult(
              bytesRead.getNamespace(), bytesRead.getName(), readStep, 12000L)));

  assertThat(
      counters,
      hasItem(
          attemptedMetricsResult(
              bytesReadBySplit.getNamespace(), bytesReadBySplit.getName(), readStep, 12000L)));

  MetricQueryResults backlogElementsMetrics =
      result
          .metrics()
          .queryMetrics(
              MetricsFilter.builder()
                  .addNameFilter(
                      MetricNameFilter.named(
                          backlogElementsOfSplit.getNamespace(),
                          backlogElementsOfSplit.getName()))
                  .build());

  // since gauge values may be inconsistent in some environments assert only on their existence.
  assertThat(backlogElementsMetrics.getGauges(), IsIterableWithSize.iterableWithSize(1));

  MetricQueryResults backlogBytesMetrics =
      result
          .metrics()
          .queryMetrics(
              MetricsFilter.builder()
                  .addNameFilter(
                      MetricNameFilter.named(
                          backlogBytesOfSplit.getNamespace(), backlogBytesOfSplit.getName()))
                  .build());

  // since gauge values may be inconsistent in some environments assert only on their existence.
  assertThat(backlogBytesMetrics.getGauges(), IsIterableWithSize.iterableWithSize(1));

  // Check checkpointMarkCommitsEnqueued metric.
  MetricQueryResults commitsEnqueuedMetrics =
      result
          .metrics()
          .queryMetrics(
              MetricsFilter.builder()
                  .addNameFilter(
                      MetricNameFilter.named(
                          KafkaUnboundedReader.METRIC_NAMESPACE,
                          KafkaUnboundedReader.CHECKPOINT_MARK_COMMITS_ENQUEUED_METRIC))
                  .build());

  assertThat(commitsEnqueuedMetrics.getCounters(), IsIterableWithSize.iterableWithSize(1));
  assertThat(
      commitsEnqueuedMetrics.getCounters().iterator().next().getAttempted(), greaterThan(0L));
}