Java Code Examples for org.apache.beam.sdk.metrics.Metrics#counter()

@Setup
public void setUp() {
  try {
    inputCounter = Metrics.counter( Pipeline.METRIC_NAME_INPUT, transformName );
    writtenCounter = Metrics.counter( Pipeline.METRIC_NAME_WRITTEN, transformName );

    // Initialize Hop Beam
    //
    BeamHop.init( transformPluginClasses, xpPluginClasses );
    rowMeta = JsonRowMeta.fromJson( rowMetaJson );

    Metrics.counter( Pipeline.METRIC_NAME_INIT, transformName ).inc();
  } catch ( Exception e ) {
    Metrics.counter( Pipeline.METRIC_NAME_ERROR, transformName ).inc();
    LOG.error( "Error in setup of converting input data into Hop rows : " + e.getMessage() );
    throw new RuntimeException( "Error in setup of converting input data into Hop rows", e );
  }
}
 

@Setup
public void setUp() {
  try {
    writtenCounter = Metrics.counter( Pipeline.METRIC_NAME_WRITTEN, counterName );
    errorCounter = Metrics.counter( Pipeline.METRIC_NAME_ERROR, counterName );

    // Initialize Hop Beam
    //
    BeamHop.init( transformPluginClasses, xpPluginClasses );
    outputRowMeta = JsonRowMeta.fromJson( outputRowMetaJson );
    leftKRowMeta = JsonRowMeta.fromJson( leftKRowMetaJson );
    leftVRowMeta = JsonRowMeta.fromJson( leftVRowMetaJson );
    rightVRowMeta = JsonRowMeta.fromJson( rightVRowMetaJson );

    Metrics.counter( Pipeline.METRIC_NAME_INIT, counterName ).inc();
  } catch(Exception e) {
    errorCounter.inc();
    LOG.error( "Error initializing assembling rows", e);
    throw new RuntimeException( "Error initializing assembling output KV<row, KV<row, row>>", e );
  }
}
 

@Setup
public void setUp() {
  try {
    readCounter = Metrics.counter( Pipeline.METRIC_NAME_READ, transformName );
    writtenCounter = Metrics.counter( Pipeline.METRIC_NAME_WRITTEN, transformName );
    errorCounter = Metrics.counter( Pipeline.METRIC_NAME_ERROR, transformName );

    // Initialize Hop Beam
    //
    BeamHop.init( transformPluginClasses, xpPluginClasses );
    inputRowMeta = JsonRowMeta.fromJson( rowMetaJson );

    Metrics.counter( Pipeline.METRIC_NAME_INIT, transformName ).inc();
  } catch(Exception e) {
    errorCounter.inc();
    LOG.error( "Error in setup of adding window information to rows : " + e.getMessage() );
    throw new RuntimeException( "Error in setup of adding window information to rows", e );
  }
}
 

@Setup
public void setUp() {
  try {
    inputCounter = Metrics.counter( "input", stepname );
    writtenCounter = Metrics.counter( "written", stepname );

    // Initialize Kettle Beam
    //
    BeamKettle.init( stepPluginClasses, xpPluginClasses );
    rowMeta = JsonRowMeta.fromJson( rowMetaJson );

    Metrics.counter( "init", stepname ).inc();
  } catch ( Exception e ) {
    numErrors.inc();
    LOG.error( "Error in setup of pub/sub publish messages function", e );
    throw new RuntimeException( "Error in setup of pub/sub publish messages function", e );
  }
}
 

/** Factory method to create mapper instance. */
public static MapWithFailures<PubsubMessage, PubsubMessage, PubsubMessage> of(int maxBytes) {
  final Counter countPayloadTooLarge = Metrics.counter(LimitPayloadSize.class,
      "payload_too_large");
  return MapElements.into(TypeDescriptor.of(PubsubMessage.class)).via((PubsubMessage msg) -> {
    msg = PubsubConstraints.ensureNonNull(msg);
    int numBytes = msg.getPayload().length;
    if (numBytes > maxBytes) {
      countPayloadTooLarge.inc();
      throw new PayloadTooLargeException("Message payload is " + numBytes
          + "bytes, larger than the" + " configured limit of " + maxBytes);
    }
    return msg;
  }).exceptionsInto(TypeDescriptor.of(PubsubMessage.class))
      .exceptionsVia((ExceptionElement<PubsubMessage> ee) -> {
        try {
          throw ee.exception();
        } catch (PayloadTooLargeException e) {
          return FailureMessage.of(LimitPayloadSize.class.getSimpleName(), ee.element(),
              ee.exception());
        }
      });
}
 

@Setup
public void setUp() {
  try {
    inputCounter = Metrics.counter( "input", stepname );
    writtenCounter = Metrics.counter( "written", stepname );

    // Initialize Kettle Beam
    //
    BeamKettle.init( stepPluginClasses, xpPluginClasses );
    rowMeta = JsonRowMeta.fromJson( rowMetaJson );

    Metrics.counter( "init", stepname ).inc();
  } catch ( Exception e ) {
    Metrics.counter( "error", stepname ).inc();
    LOG.error( "Error in setup of converting input data into Kettle rows : " + e.getMessage() );
    throw new RuntimeException( "Error in setup of converting input data into Kettle rows", e );
  }
}
 

/** Generate Avro schema by reading one row. Expose Beam metrics via a Beam PTransform. */
public static Schema createSchema(
    final Pipeline pipeline, final JdbcExportArgs args, final Connection connection)
    throws Exception {
  final long startTime = System.nanoTime();
  final Schema generatedSchema = generateAvroSchema(args, connection);
  final long elapsedTimeSchema = (System.nanoTime() - startTime) / 1000000;
  LOGGER.info("Elapsed time to schema {} seconds", elapsedTimeSchema / 1000.0);

  final Counter cnt =
      Metrics.counter(BeamJdbcAvroSchema.class.getCanonicalName(), "schemaElapsedTimeMs");
  pipeline
      .apply(
          "ExposeSchemaCountersSeed",
          Create.of(Collections.singletonList(0)).withType(TypeDescriptors.integers()))
      .apply(
          "ExposeSchemaCounters",
          MapElements.into(TypeDescriptors.integers())
              .via(
                  v -> {
                    cnt.inc(elapsedTimeSchema);
                    return v;
                  }));
  return generatedSchema;
}
 

@Setup
public void setUp() {
  try {
    inputCounter = Metrics.counter( Pipeline.METRIC_NAME_INPUT, transformName );
    writtenCounter = Metrics.counter( Pipeline.METRIC_NAME_WRITTEN, transformName );

    // Initialize Hop Beam
    //
    BeamHop.init( transformPluginClasses, xpPluginClasses );
    rowMeta = JsonRowMeta.fromJson( rowMetaJson );

    Metrics.counter( Pipeline.METRIC_NAME_INIT, transformName ).inc();
  } catch ( Exception e ) {
    numErrors.inc();
    LOG.error( "Error in setup of HopRow to KV<String,String> function", e );
    throw new RuntimeException( "Error in setup of HopRow to KV<String,String> function", e );
  }
}
 

@Setup
public void setUp() {
  try {
    readCounter = Metrics.counter( Pipeline.METRIC_NAME_READ, transformName );
    outputCounter = Metrics.counter( Pipeline.METRIC_NAME_OUTPUT, transformName );

    // Initialize Hop Beam
    //
    BeamHop.init( transformPluginClasses, xpPluginClasses );
    rowMeta = JsonRowMeta.fromJson( rowMetaJson );

    Metrics.counter( Pipeline.METRIC_NAME_INIT, transformName ).inc();
  } catch ( Exception e ) {
    numErrors.inc();
    LOG.error( "Error in setup of pub/sub publish messages function", e );
    throw new RuntimeException( "Error in setup of pub/sub publish messages function", e );
  }
}
 

@Setup
public void setUp() {
  try {
    readCounter = Metrics.counter( Pipeline.METRIC_NAME_READ, transformName );
    outputCounter = Metrics.counter( Pipeline.METRIC_NAME_OUTPUT, transformName );

    // Initialize Hop Beam
    //
    BeamHop.init( transformPluginClasses, xpPluginClasses );
    rowMeta = JsonRowMeta.fromJson( rowMetaJson );

    Metrics.counter( Pipeline.METRIC_NAME_INIT, transformName ).inc();
  } catch ( Exception e ) {
    numErrors.inc();
    LOG.error( "Error in setup of pub/sub publish messages function", e );
    throw new RuntimeException( "Error in setup of pub/sub publish messages function", e );
  }
}
 

@Setup
public void setUp() {
  try {
    writtenCounter = Metrics.counter( "written", counterName );
    errorCounter = Metrics.counter( "error", counterName );

    // Initialize Kettle Beam
    //
    BeamKettle.init( stepPluginClasses, xpPluginClasses );
    outputRowMeta = JsonRowMeta.fromJson( outputRowMetaJson );
    leftKRowMeta = JsonRowMeta.fromJson( leftKRowMetaJson );
    leftVRowMeta = JsonRowMeta.fromJson( leftVRowMetaJson );
    rightVRowMeta = JsonRowMeta.fromJson( rightVRowMetaJson );

    Metrics.counter( "init", counterName ).inc();
  } catch(Exception e) {
    errorCounter.inc();
    LOG.error( "Error initializing assembling rows", e);
    throw new RuntimeException( "Error initializing assembling output KV<row, KV<row, row>>", e );
  }
}
 

public LateDataFilter(
    WindowingStrategy<?, ?> windowingStrategy, TimerInternals timerInternals) {
  this.windowingStrategy = windowingStrategy;
  this.timerInternals = timerInternals;
  this.droppedDueToLateness =
      Metrics.counter(LateDataDroppingDoFnRunner.class, DROPPED_DUE_TO_LATENESS);
}
 

GcsFileSystem(GcsOptions options) {
  this.options = checkNotNull(options, "options");
  if (options.getGcsPerformanceMetrics()) {
    numCopies = Metrics.counter(GcsFileSystem.class, "num_copies");
    copyTimeMsec = Metrics.counter(GcsFileSystem.class, "copy_time_msec");
  }
}
 

@Test
@Category({NeedsRunner.class, UsesCommittedMetrics.class, UsesCounterMetrics.class})
public void testRunPTransform() {
  final String namespace = PipelineRunnerTest.class.getName();
  final Counter counter = Metrics.counter(namespace, "count");
  final PipelineResult result =
      PipelineRunner.fromOptions(p.getOptions())
          .run(
              new PTransform<PBegin, POutput>() {
                @Override
                public POutput expand(PBegin input) {
                  PCollection<Double> output =
                      input
                          .apply(Create.of(1, 2, 3, 4))
                          .apply("ScaleByTwo", MapElements.via(new ScaleFn<>(2.0, counter)));
                  PAssert.that(output).containsInAnyOrder(2.0, 4.0, 6.0, 8.0);
                  return output;
                }
              });

  // Checking counters to verify the pipeline actually ran.
  assertThat(
      result
          .metrics()
          .queryMetrics(
              MetricsFilter.builder()
                  .addNameFilter(MetricNameFilter.inNamespace(namespace))
                  .build())
          .getCounters(),
      hasItem(metricsResult(namespace, "count", "ScaleByTwo", 4L, true)));
}
 

private JoinDoFn(String name, int maxAuctionsWaitingTime) {
  this.name = name;
  this.maxAuctionsWaitingTime = maxAuctionsWaitingTime;
  newAuctionCounter = Metrics.counter(name, "newAuction");
  newPersonCounter = Metrics.counter(name, "newPerson");
  newNewOutputCounter = Metrics.counter(name, "newNewOutput");
  newOldOutputCounter = Metrics.counter(name, "newOldOutput");
  oldNewOutputCounter = Metrics.counter(name, "oldNewOutput");
  fatalCounter = Metrics.counter(name, "fatal");
}
 

@Setup
public void setUp() {
  try {
    // Initialize Kettle Beam
    //
    BeamKettle.init( stepPluginClasses, xpPluginClasses );

    inputRowMeta = JsonRowMeta.fromJson( rowMetaJson );

    readCounter = Metrics.counter( "read", stepname );
    writtenCounter = Metrics.counter( "written", stepname );
    errorCounter = Metrics.counter( "error", stepname );

    fieldIndex = -1;
    if ( !getTimestamp && StringUtils.isNotEmpty( fieldName ) ) {
      fieldIndex = inputRowMeta.indexOfValue( fieldName );
      if ( fieldIndex < 0 ) {
        throw new RuntimeException( "Field '" + fieldName + "' couldn't be found in put : " + inputRowMeta.toString() );
      }
      fieldValueMeta = inputRowMeta.getValueMeta( fieldIndex );
    }

    Metrics.counter( "init", stepname ).inc();
  } catch(Exception e) {
    errorCounter.inc();
    LOG.error( "Error in setup of adding timestamp to rows : " + e.getMessage() );
    throw new RuntimeException( "Error setup of adding timestamp to rows", e );
  }
}
 

@Override public TableRow apply( KettleRow inputRow ) {

    try {
      if ( rowMeta == null ) {
        readCounter = Metrics.counter( "read", counterName );
        outputCounter = Metrics.counter( "output", counterName );
        errorCounter = Metrics.counter( "error", counterName );

        // Initialize Kettle Beam
        //
        BeamKettle.init( stepPluginClasses, xpPluginClasses );
        rowMeta = JsonRowMeta.fromJson( rowMetaJson );

        simpleDateFormat = new SimpleDateFormat( "yyyy-MM-dd HH:mm:ss.SSS" );
        Metrics.counter( "init", counterName ).inc();
      }

      readCounter.inc();

      TableRow tableRow = new TableRow();
      for (int i=0;i<rowMeta.size();i++) {
        ValueMetaInterface valueMeta = rowMeta.getValueMeta( i );
        Object valueData = inputRow.getRow()[i];
        if (!valueMeta.isNull( valueData )) {
          switch ( valueMeta.getType() ) {
            case ValueMetaInterface.TYPE_STRING: tableRow.put( valueMeta.getName(), valueMeta.getString( valueData ) ); break;
            case ValueMetaInterface.TYPE_INTEGER: tableRow.put( valueMeta.getName(), valueMeta.getInteger( valueData ) ); break;
            case ValueMetaInterface.TYPE_DATE:
              Date date = valueMeta.getDate( valueData );
              String formattedDate = simpleDateFormat.format( date );
              tableRow.put( valueMeta.getName(), formattedDate);
              break;
            case ValueMetaInterface.TYPE_BOOLEAN: tableRow.put( valueMeta.getName(), valueMeta.getBoolean( valueData ) ); break;
            case ValueMetaInterface.TYPE_NUMBER: tableRow.put( valueMeta.getName(), valueMeta.getNumber( valueData ) ); break;
            default:
              throw new RuntimeException( "Data type conversion from Kettle to BigQuery TableRow not supported yet: " +valueMeta.toString());
          }
        }
      }

      // Pass the row to the process context
      //
      outputCounter.inc();

      return tableRow;

    } catch ( Exception e ) {
      errorCounter.inc();
      LOG.info( "Conversion error KettleRow to BigQuery TableRow : " + e.getMessage() );
      throw new RuntimeException( "Error converting KettleRow to BigQuery TableRow", e );
    }
  }
 

@Setup
public void setUp() {
  try {
    readCounter = Metrics.counter( Pipeline.METRIC_NAME_READ, counterName );
    errorCounter = Metrics.counter( Pipeline.METRIC_NAME_ERROR, counterName );

    // Initialize Hop Beam
    //
    BeamHop.init(transformPluginClasses, xpPluginClasses);
    inputRowMeta = JsonRowMeta.fromJson( inputRowMetaJson );

    // Calculate key indexes
    //
    if ( keyFields.length==0) {
      throw new HopException( "There are no group fields" );
    }
    keyIndexes = new int[ keyFields.length];
    for ( int i = 0; i< keyFields.length; i++) {
      keyIndexes[i]=inputRowMeta.indexOfValue( keyFields[i] );
      if ( keyIndexes[i]<0) {
        throw new HopException( "Unable to find group by field '"+ keyFields[i]+"' in input "+inputRowMeta.toString() );
      }
    }

    // Calculate the value indexes
    //
    valueIndexes =new int[ valueFields.length];
    for ( int i = 0; i< valueFields.length; i++) {
      valueIndexes[i] = inputRowMeta.indexOfValue( valueFields[i] );
      if ( valueIndexes[i]<0) {
        throw new HopException( "Unable to find subject by field '"+ valueFields[i]+"' in input "+inputRowMeta.toString() );
      }
    }

    // Now that we know everything, we can split the row...
    //
    Metrics.counter( Pipeline.METRIC_NAME_INIT, counterName ).inc();
  } catch(Exception e) {
    errorCounter.inc();
    LOG.error("Error setup of splitting row into key and value", e);
    throw new RuntimeException( "Unable to setup of split row into key and value", e );
  }
}
 

@Setup
public void setUp() {
  try {
    readCounter = Metrics.counter( "read", counterName );
    errorCounter = Metrics.counter( "error", counterName );

    // Initialize Kettle Beam
    //
    BeamKettle.init(stepPluginClasses, xpPluginClasses);
    inputRowMeta = JsonRowMeta.fromJson( inputRowMetaJson );

    // Calculate key indexes
    //
    if ( keyFields.length==0) {
      throw new KettleException( "There are no group fields" );
    }
    keyIndexes = new int[ keyFields.length];
    for ( int i = 0; i< keyFields.length; i++) {
      keyIndexes[i]=inputRowMeta.indexOfValue( keyFields[i] );
      if ( keyIndexes[i]<0) {
        throw new KettleException( "Unable to find group by field '"+ keyFields[i]+"' in input "+inputRowMeta.toString() );
      }
    }

    // Calculate the value indexes
    //
    valueIndexes =new int[ valueFields.length];
    for ( int i = 0; i< valueFields.length; i++) {
      valueIndexes[i] = inputRowMeta.indexOfValue( valueFields[i] );
      if ( valueIndexes[i]<0) {
        throw new KettleException( "Unable to find subject by field '"+ valueFields[i]+"' in input "+inputRowMeta.toString() );
      }
    }

    // Now that we know everything, we can split the row...
    //
    Metrics.counter( "init", counterName ).inc();
  } catch(Exception e) {
    errorCounter.inc();
    LOG.error("Error setup of splitting row into key and value", e);
    throw new RuntimeException( "Unable to setup of split row into key and value", e );
  }
}
 

public ReduceFnRunner(
    K key,
    WindowingStrategy<?, W> windowingStrategy,
    ExecutableTriggerStateMachine triggerStateMachine,
    StateInternals stateInternals,
    TimerInternals timerInternals,
    OutputWindowedValue<KV<K, OutputT>> outputter,
    @Nullable SideInputReader sideInputReader,
    ReduceFn<K, InputT, OutputT, W> reduceFn,
    @Nullable PipelineOptions options) {
  this.key = key;
  this.timerInternals = timerInternals;
  this.paneInfoTracker = new PaneInfoTracker(timerInternals);
  this.stateInternals = stateInternals;
  this.outputter = outputter;
  this.reduceFn = reduceFn;
  this.droppedDueToClosedWindow =
      Metrics.counter(ReduceFnRunner.class, DROPPED_DUE_TO_CLOSED_WINDOW);

  @SuppressWarnings("unchecked")
  WindowingStrategy<Object, W> objectWindowingStrategy =
      (WindowingStrategy<Object, W>) windowingStrategy;
  this.windowingStrategy = objectWindowingStrategy;

  this.nonEmptyPanes = NonEmptyPanes.create(this.windowingStrategy, this.reduceFn);

  // Note this may incur I/O to load persisted window set data.
  this.activeWindows = createActiveWindowSet();

  this.contextFactory =
      new ReduceFnContextFactory<>(
          key,
          reduceFn,
          this.windowingStrategy,
          stateInternals,
          this.activeWindows,
          timerInternals,
          sideInputReader,
          options);

  this.watermarkHold = new WatermarkHold<>(timerInternals, windowingStrategy);
  this.triggerRunner =
      new TriggerStateMachineRunner<>(
          triggerStateMachine,
          new TriggerStateMachineContextFactory<>(
              windowingStrategy.getWindowFn(), stateInternals, activeWindows));
}