org.apache.beam.sdk.metrics.MetricResults Java Examples

@Override
public MetricResults metrics() {
  return new MetricResults() {
    @Override
    public MetricQueryResults queryMetrics(@Nullable MetricsFilter filter) {
      return new MetricQueryResults() {
        @Override
        public Iterable<MetricResult<Long>> getCounters() {
          return Collections.emptyList();
        }

        @Override
        public Iterable<MetricResult<DistributionResult>> getDistributions() {
          return Collections.emptyList();
        }

        @Override
        public Iterable<MetricResult<GaugeResult>> getGauges() {
          return Collections.emptyList();
        }
      };
    }
  };
}
 

private void pushMetrics() {
  if (!(metricsSink instanceof NoOpMetricsSink)) {
    try {
      // merge metrics
      MetricResults metricResults = asAttemptedOnlyMetricResults(metricsContainerStepMap);
      MetricQueryResults metricQueryResults = metricResults.allMetrics();
      if ((Iterables.size(metricQueryResults.getDistributions()) != 0)
          || (Iterables.size(metricQueryResults.getGauges()) != 0)
          || (Iterables.size(metricQueryResults.getCounters()) != 0)) {
        metricsSink.writeMetrics(metricQueryResults);
      }

    } catch (Exception e) {
      MetricsPushException metricsPushException = new MetricsPushException(e);
      metricsPushException.printStackTrace();
    }
  }
}
 

/**
 * Update Flink's internal metrics ({@link #flinkCounterCache}) with the latest metrics for
 * a given step.
 */
private void updateMetrics(String stepName) {
	MetricResults metricResults = asAttemptedOnlyMetricResults(metricsContainers);
	MetricQueryResults metricQueryResults =
		metricResults.queryMetrics(MetricsFilter.builder().addStep(stepName).build());
	updateCounterOrMeter(metricQueryResults.getCounters());
	updateDistributions(metricQueryResults.getDistributions());
	updateGauge(metricQueryResults.getGauges());
}
 

@Test
public void testDirectRunnerApiSurface() throws Exception {
  // The DirectRunner can expose the Core SDK, anything exposed by the Core SDK, and itself
  @SuppressWarnings("unchecked")
  final Set<String> allowed =
      ImmutableSet.of(
          "org.apache.beam.sdk",
          "org.apache.beam.runners.direct",
          "org.joda.time",
          "javax.annotation",
          "java.math");

  final Package thisPackage = getClass().getPackage();
  final ClassLoader thisClassLoader = getClass().getClassLoader();
  ApiSurface apiSurface =
      ApiSurface.ofPackage(thisPackage, thisClassLoader)
          // Do not include dependencies that are required based on the known exposures. This
          // could alternatively prune everything exposed by the public parts of the Core SDK
          .pruningClass(Pipeline.class)
          .pruningClass(PipelineRunner.class)
          .pruningClass(PipelineOptions.class)
          .pruningClass(PipelineOptionsRegistrar.class)
          .pruningClass(PipelineOptions.DirectRunner.class)
          .pruningClass(DisplayData.Builder.class)
          .pruningClass(MetricResults.class)
          .pruningClass(DirectGraphs.class)
          .pruningClass(
              WatermarkManager.class /* TODO: BEAM-4237 Consider moving to local-java */)
          .pruningPattern(
              "org[.]apache[.]beam[.]runners[.]direct[.]portable.*"
              /* TODO: BEAM-4237 reconsider package layout with the ReferenceRunner */ )
          .pruningPattern("org[.]apache[.]beam[.].*Test.*")
          .pruningPattern("org[.]apache[.]beam[.].*IT")
          .pruningPattern("java[.]io.*")
          .pruningPattern("java[.]lang.*")
          .pruningPattern("java[.]util.*");

  assertThat(apiSurface, containsOnlyPackages(allowed));
}
 

/**
 * Update Flink's internal metrics ({@link this#flinkCounterCache}) with the latest metrics for a
 * given step.
 */
void updateMetrics(String stepName) {
  MetricResults metricResults = asAttemptedOnlyMetricResults(metricsContainers);
  MetricQueryResults metricQueryResults =
      metricResults.queryMetrics(MetricsFilter.builder().addStep(stepName).build());
  updateCounters(metricQueryResults.getCounters());
  updateDistributions(metricQueryResults.getDistributions());
  updateGauge(metricQueryResults.getGauges());
}
 

private static PortableMetrics convertMonitoringInfosToMetricResults(
    JobApi.MetricResults jobMetrics) {
  List<MetricsApi.MonitoringInfo> monitoringInfoList = new ArrayList<>();
  monitoringInfoList.addAll(jobMetrics.getAttemptedList());
  monitoringInfoList.addAll(jobMetrics.getCommittedList());
  Iterable<MetricResult<Long>> countersFromJobMetrics =
      extractCountersFromJobMetrics(monitoringInfoList);
  Iterable<MetricResult<DistributionResult>> distributionsFromMetrics =
      extractDistributionMetricsFromJobMetrics(monitoringInfoList);
  Iterable<MetricResult<GaugeResult>> gaugesFromMetrics =
      extractGaugeMetricsFromJobMetrics(monitoringInfoList);
  return new PortableMetrics(countersFromJobMetrics, distributionsFromMetrics, gaugesFromMetrics);
}
 

@Test
public void testGaugeCommittedUnsupportedInAttemptedAccumulatedMetricResults() {
  MetricsContainerStepMap attemptedMetrics = new MetricsContainerStepMap();
  attemptedMetrics.update(STEP1, metricsContainer);
  MetricResults metricResults = asAttemptedOnlyMetricResults(attemptedMetrics);

  MetricQueryResults step1res =
      metricResults.queryMetrics(MetricsFilter.builder().addStep(STEP1).build());

  thrown.expect(UnsupportedOperationException.class);
  thrown.expectMessage("This runner does not currently support committed metrics results.");

  assertGauge(GAUGE_NAME, step1res, STEP1, GaugeResult.empty(), true);
}
 

@Test
public void testDistributionCommittedUnsupportedInAttemptedAccumulatedMetricResults() {
  MetricsContainerStepMap attemptedMetrics = new MetricsContainerStepMap();
  attemptedMetrics.update(STEP1, metricsContainer);
  MetricResults metricResults = asAttemptedOnlyMetricResults(attemptedMetrics);

  MetricQueryResults step1res =
      metricResults.queryMetrics(MetricsFilter.builder().addStep(STEP1).build());

  thrown.expect(UnsupportedOperationException.class);
  thrown.expectMessage("This runner does not currently support committed metrics results.");

  assertDistribution(
      DISTRIBUTION_NAME, step1res, STEP1, DistributionResult.IDENTITY_ELEMENT, true);
}
 

@Test
public void testCounterCommittedUnsupportedInAttemptedAccumulatedMetricResults() {
  MetricsContainerStepMap attemptedMetrics = new MetricsContainerStepMap();
  attemptedMetrics.update(STEP1, metricsContainer);
  MetricResults metricResults = asAttemptedOnlyMetricResults(attemptedMetrics);

  MetricQueryResults step1res =
      metricResults.queryMetrics(MetricsFilter.builder().addStep(STEP1).build());

  thrown.expect(UnsupportedOperationException.class);
  thrown.expectMessage("This runner does not currently support committed metrics results.");

  assertCounter(COUNTER_NAME, step1res, STEP1, VALUE, true);
}
 

@Override
public String toString() {
  JobApi.MetricResults results =
      JobApi.MetricResults.newBuilder().addAllAttempted(getMonitoringInfos()).build();
  GetJobMetricsResponse response = GetJobMetricsResponse.newBuilder().setMetrics(results).build();
  try {
    JsonFormat.Printer printer = JsonFormat.printer();
    return printer.print(response);
  } catch (InvalidProtocolBufferException e) {
    throw new RuntimeException(e);
  }
}
 

@Test(timeout = 10_000)
public void testReturnsMetricsFromJobInvocationAfterSuccess() throws Exception {
  JobApi.MetricResults expectedMonitoringInfos = JobApi.MetricResults.newBuilder().build();
  TestPipelineResult result =
      new TestPipelineResult(PipelineResult.State.DONE, expectedMonitoringInfos);

  jobInvocation.start();
  runner.setResult(result);

  awaitJobState(jobInvocation, JobApi.JobState.Enum.DONE);

  assertThat(
      jobInvocation.getMetrics(),
      allOf(is(notNullValue()), is(sameInstance(result.portableMetrics()))));
}
 

@Ignore
public void createRunPipeline( PipelineMeta pipelineMeta ) throws Exception {

  /*
  FileOutputStream fos = new FileOutputStream( "/tmp/"+pipelineMeta.getName()+".ktr" );
  fos.write( pipelineMeta.getXML().getBytes() );
  fos.close();
  */

  PipelineOptions pipelineOptions = PipelineOptionsFactory.create();

  pipelineOptions.setJobName( pipelineMeta.getName() );
  pipelineOptions.setUserAgent( BeamConst.STRING_HOP_BEAM );

  BeamDirectPipelineRunConfiguration beamRunConfig = new BeamDirectPipelineRunConfiguration();
  beamRunConfig.setTempLocation( System.getProperty( "java.io.tmpdir" ) );

  // No extra plugins to load : null option
  HopPipelineMetaToBeamPipelineConverter converter = new HopPipelineMetaToBeamPipelineConverter( pipelineMeta, metadataProvider, beamRunConfig );
  Pipeline pipeline = converter.createPipeline();

  PipelineResult pipelineResult = pipeline.run();
  pipelineResult.waitUntilFinish();

  MetricResults metricResults = pipelineResult.metrics();

  MetricQueryResults allResults = metricResults.queryMetrics( MetricsFilter.builder().build() );
  for ( MetricResult<Long> result : allResults.getCounters() ) {
    System.out.println( "Name: " + result.getName() + " Attempted: " + result.getAttempted() );
  }
}
 

@Ignore
public void createRunPipeline( TransMeta transMeta ) throws Exception {

  /*
  FileOutputStream fos = new FileOutputStream( "/tmp/"+transMeta.getName()+".ktr" );
  fos.write( transMeta.getXML().getBytes() );
  fos.close();
  */

  PipelineOptions pipelineOptions = PipelineOptionsFactory.create();

  pipelineOptions.setJobName( transMeta.getName() );
  pipelineOptions.setUserAgent( BeamConst.STRING_KETTLE_BEAM );

  BeamJobConfig jobConfig = new BeamJobConfig();
  jobConfig.setName("Direct runner test");
  jobConfig.setRunnerTypeName( RunnerType.Direct.name() );

  // No extra plugins to load : null option
  TransMetaPipelineConverter converter = new TransMetaPipelineConverter( transMeta, metaStore, (String) null, jobConfig );
  Pipeline pipeline = converter.createPipeline( pipelineOptions );

  PipelineResult pipelineResult = pipeline.run();
  pipelineResult.waitUntilFinish();

  MetricResults metricResults = pipelineResult.metrics();

  MetricQueryResults allResults = metricResults.queryMetrics( MetricsFilter.builder().build() );
  for ( MetricResult<Long> result : allResults.getCounters() ) {
    System.out.println( "Name: " + result.getName() + " Attempted: " + result.getAttempted() );
  }
}
 

@Test
public void shouldThrownExceptionInCaseFailedPipelineResult() {
  PipelineResult mockResult =
      new PipelineResult() {
        @Override
        public State getState() {
          return null;
        }

        @Override
        public State cancel() throws IOException {
          return null;
        }

        @Override
        public State waitUntilFinish(org.joda.time.Duration duration) {
          return PipelineResult.State.FAILED;
        }

        @Override
        public State waitUntilFinish() {
          return null;
        }

        @Override
        public MetricResults metrics() {
          return null;
        }
      };
  try {
    BeamHelper.waitUntilDone(mockResult, Duration.ofMinutes(1));
    Assert.fail("A PipelineExecutionException should be thrown");
  } catch (Pipeline.PipelineExecutionException exception) {
    Assert.assertEquals(
        "java.lang.Exception: Job finished with terminalState FAILED", exception.getMessage());
  }
}
 

@Test
public void shouldCancelInCaseOfTimeout() {
  PipelineResult mockResult =
      new PipelineResult() {
        @Override
        public State getState() {
          return null;
        }

        @Override
        public State cancel() throws IOException {
          return null;
        }

        @Override
        public State waitUntilFinish(org.joda.time.Duration duration) {
          return State.RUNNING;
        }

        @Override
        public State waitUntilFinish() {
          return null;
        }

        @Override
        public MetricResults metrics() {
          return null;
        }
      };
  try {
    BeamHelper.waitUntilDone(mockResult, Duration.ofMinutes(1));
    Assert.fail("A PipelineExecutionException should be thrown");
  } catch (Pipeline.PipelineExecutionException exception) {
    Assert.assertEquals(
        "java.lang.Exception: Job cancelled after exceeding timeout PT1M",
        exception.getMessage());
  }
}
 

@Test
public void shouldCancelInCaseOfNullState() {
  PipelineResult mockResult =
      new PipelineResult() {
        @Override
        public State getState() {
          return null;
        }

        @Override
        public State cancel() throws IOException {
          return null;
        }

        @Override
        public State waitUntilFinish(org.joda.time.Duration duration) {
          return null;
        }

        @Override
        public State waitUntilFinish() {
          return null;
        }

        @Override
        public MetricResults metrics() {
          return null;
        }
      };
  try {
    BeamHelper.waitUntilDone(mockResult, Duration.ofMinutes(1));
    Assert.fail("A PipelineExecutionException should be thrown");
  } catch (Pipeline.PipelineExecutionException exception) {
    Assert.assertEquals(
        "java.lang.Exception: Job cancelled after exceeding timeout PT1M",
        exception.getMessage());
  }
}
 

private void logMetrics( PipelineResult pipelineResult ) {
  MetricResults metricResults = pipelineResult.metrics();

  logChannel.logBasic( "  ----------------- Metrics refresh @ " + new SimpleDateFormat( "yyyy/MM/dd HH:mm:ss" ).format( new Date() ) + " -----------------------" );

  MetricQueryResults allResults = metricResults.queryMetrics( MetricsFilter.builder().build() );
  for ( MetricResult<Long> result : allResults.getCounters() ) {
    logChannel.logBasic( "Name: " + result.getName() + " Attempted: " + result.getAttempted() );
  }
}
 

@Test
public void shouldFailAfterFailureToCancelAfterTimeout() {
  PipelineResult mockResult =
      new PipelineResult() {
        @Override
        public State getState() {
          return null;
        }

        @Override
        public State cancel() throws IOException {
          throw new IOException("something wrong");
        }

        @Override
        public State waitUntilFinish(org.joda.time.Duration duration) {
          return State.RUNNING;
        }

        @Override
        public State waitUntilFinish() {
          return null;
        }

        @Override
        public MetricResults metrics() {
          return null;
        }
      };
  try {
    BeamHelper.waitUntilDone(mockResult, Duration.ofMinutes(1));
    Assert.fail("A PipelineExecutionException should be thrown");
  } catch (Pipeline.PipelineExecutionException exception) {
    Assert.assertEquals(
        "java.lang.Exception: Job exceeded timeout of PT1M, "
            + "but was not possible to cancel, finished with terminalState RUNNING",
        exception.getMessage());
  }
}
 

@Override
public JobApi.MetricResults portableMetrics() {
  return JobApi.MetricResults.newBuilder()
      .addAllAttempted(MetricsAccumulator.getInstance().value().getMonitoringInfos())
      .build();
}
 

@Override
public MetricResults metrics() {
  return PortableMetrics.of(jobMetrics);
}
 

@Override
public MetricResults metrics() {
  return metricResults;
}
 

@Override
public MetricResults metrics() {
  return asAttemptedOnlyMetricResults(MetricsAccumulator.getInstance().value());
}
 

@Override
public MetricResults metrics() {
  return asAttemptedOnlyMetricResults(MetricsAccumulator.getInstance().value());
}
 

@Test
public void testReset() {
  MetricsContainerStepMap attemptedMetrics = new MetricsContainerStepMap();
  attemptedMetrics.update(STEP1, metricsContainer);
  attemptedMetrics.update(STEP2, metricsContainer);
  attemptedMetrics.update(STEP2, metricsContainer);

  MetricResults metricResults = asAttemptedOnlyMetricResults(attemptedMetrics);
  MetricQueryResults allres = metricResults.allMetrics();
  assertCounter(COUNTER_NAME, allres, STEP1, VALUE, false);
  assertDistribution(
      DISTRIBUTION_NAME,
      allres,
      STEP1,
      DistributionResult.create(VALUE * 3, 2, VALUE, VALUE * 2),
      false);
  assertGauge(GAUGE_NAME, allres, STEP1, GaugeResult.create(VALUE, Instant.now()), false);

  assertCounter(COUNTER_NAME, allres, STEP2, VALUE * 2, false);
  assertDistribution(
      DISTRIBUTION_NAME,
      allres,
      STEP2,
      DistributionResult.create(VALUE * 6, 4, VALUE, VALUE * 2),
      false);
  assertGauge(GAUGE_NAME, allres, STEP2, GaugeResult.create(VALUE, Instant.now()), false);

  attemptedMetrics.reset();
  metricResults = asAttemptedOnlyMetricResults(attemptedMetrics);
  allres = metricResults.allMetrics();

  // Check that the metrics container for STEP1 is reset
  assertCounter(COUNTER_NAME, allres, STEP1, 0L, false);
  assertDistribution(
      DISTRIBUTION_NAME, allres, STEP1, DistributionResult.IDENTITY_ELEMENT, false);
  assertGauge(GAUGE_NAME, allres, STEP1, GaugeResult.empty(), false);

  // Check that the metrics container for STEP2 is reset
  assertCounter(COUNTER_NAME, allres, STEP2, 0L, false);
  assertDistribution(
      DISTRIBUTION_NAME, allres, STEP2, DistributionResult.IDENTITY_ELEMENT, false);
  assertGauge(GAUGE_NAME, allres, STEP2, GaugeResult.empty(), false);
}
 

@Override
public MetricResults metrics() {
  return asAttemptedOnlyMetricResults(getMetricsContainerStepMap());
}
 

@Override
public MetricResults metrics() {
  throw new UnsupportedOperationException("The FlinkRunner does not currently support metrics.");
}
 

@Override
public MetricResults metrics() {
  return asAttemptedOnlyMetricResults(executionContext.getMetricsContainer().getContainers());
}
 

@Override
public MetricResults metrics() {
  return dataflowMetrics;
}
 

@Override
public MetricResults metrics() {
  return evaluationContext.getMetrics();
}
 

/**
 * Grab the Beam pipeline results and convert it into engine metrics
 */
protected synchronized void populateEngineMetrics() throws HopException {
  ClassLoader oldContextClassLoader = Thread.currentThread().getContextClassLoader();
  try {
    Thread.currentThread().setContextClassLoader( this.getClass().getClassLoader() );

    EngineMetrics em = new EngineMetrics();
    evaluatePipelineStatus();

    em.setStartDate( getExecutionStartDate() );
    em.setEndDate( getExecutionEndDate() );

    if ( beamPipelineResults != null ) {
      Set<String> transformNames = new HashSet<>( Arrays.asList( pipelineMeta.getTransformNames() ) );
      Map<String, EngineComponent> componentsMap = new HashMap<>();
      MetricResults metrics = beamPipelineResults.metrics();
      MetricQueryResults allResults = metrics.queryMetrics( MetricsFilter.builder().build() );

      for ( MetricResult<Long> result : allResults.getCounters() ) {
        String metricsType = result.getName().getNamespace();
        String metricsName = result.getName().getName();
        long processed = result.getAttempted();

        // This is a transform executing in Beam
        //
        if ( transformNames.contains( metricsName ) ) {
          EngineComponent engineComponent = componentsMap.get( metricsName );
          if ( engineComponent == null ) {
            engineComponent = new EngineComponent( metricsName, 0 );
            componentsMap.put( metricsName, engineComponent );
          }
          if ( Pipeline.METRIC_NAME_READ.equalsIgnoreCase( metricsType ) ) {
            engineComponent.setLinesRead( processed );
            em.setComponentMetric( engineComponent, Pipeline.METRIC_READ, processed );
          } else if ( Pipeline.METRIC_NAME_WRITTEN.equalsIgnoreCase( metricsType ) ) {
            engineComponent.setLinesWritten( processed );
            em.setComponentMetric( engineComponent, Pipeline.METRIC_WRITTEN, processed );
          } else if ( Pipeline.METRIC_NAME_INPUT.equalsIgnoreCase( metricsType ) ) {
            engineComponent.setLinesInput( processed );
            em.setComponentMetric( engineComponent, Pipeline.METRIC_INPUT, processed );
          } else if ( Pipeline.METRIC_NAME_OUTPUT.equalsIgnoreCase( metricsType ) ) {
            engineComponent.setLinesOutput( processed );
            em.setComponentMetric( engineComponent, Pipeline.METRIC_OUTPUT, processed );
          } else if ( Pipeline.METRIC_NAME_INIT.equalsIgnoreCase( metricsType ) ) {
            em.setComponentMetric( engineComponent, Pipeline.METRIC_INIT, processed );
          } else if ( Pipeline.METRIC_NAME_FLUSH_BUFFER.equalsIgnoreCase( metricsType ) ) {
            em.setComponentMetric( engineComponent, Pipeline.METRIC_FLUSH_BUFFER, processed );
          }

          // Copy the execution start and end date from the pipeline
          //
          engineComponent.setExecutionStartDate( getExecutionStartDate() );
          engineComponent.setExecutionEndDate( getExecutionEndDate() );
          engineComponent.setExecutionDuration( calculateDuration( getExecutionStartDate(), getExecutionEndDate() ) );

          // Set the transform status to reflect the pipeline status.
          //
          switch ( beamPipelineResults.getState() ) {
            case DONE:
              engineComponent.setRunning( false );
              engineComponent.setStatus( ComponentExecutionStatus.STATUS_FINISHED );
              break;
            case CANCELLED:
            case FAILED:
            case STOPPED:
              engineComponent.setStopped( true );
              engineComponent.setRunning( false );
              engineComponent.setStatus( ComponentExecutionStatus.STATUS_STOPPED );
              break;
            case RUNNING:
              engineComponent.setRunning( true );
              engineComponent.setStopped( false );
              engineComponent.setStatus( ComponentExecutionStatus.STATUS_RUNNING );
              break;
            case UNKNOWN:
              break;
            case UPDATED:
              break;
            default:
              break;
          }
        }
      }

      em.getComponents().clear();
      em.getComponents().addAll( componentsMap.values() );
    }

    // Swap the engine metrics with the new value
    //
    synchronized ( engineMetrics ) {
      engineMetrics = em;
    }
  } finally {
    Thread.currentThread().setContextClassLoader( oldContextClassLoader );
  }
}