org.apache.beam.sdk.options.Description Java Examples

@Description(
    "The output file to write failures during the decompression process "
        + "(e.g. gs://bucket-name/decompressed/failed.txt). The contents will be one line for "
        + "each file which failed decompression. Note that this parameter will "
        + "allow the pipeline to continue processing in the event of a failure.")
@Required
ValueProvider<String> getOutputFailureFile();
 

@Description(
    "DLP API has a limit for payload size of 524KB /api call. "
        + "That's why dataflow process will need to chunk it. User will have to decide "
        + "on how they would like to batch the request depending on number of rows "
        + "and how big each row is.")
@Required
ValueProvider<Integer> getBatchSize();
 

@Description("If set, specifies the time when the snapshot must be taken."
  + " String is in the RFC 3339 format in UTC time. "
  + " Example - 1990-12-31T23:59:60Z"
  + " Timestamp must be in the past and Maximum timestamp staleness applies."
  + " https://cloud.google.com/spanner/docs/timestamp-bounds#maximum_timestamp_staleness")
@Default.String(value = "")
ValueProvider<String> getSnapshotTime();
 

@Description("When --bqReadMethod=storageapi, all rows of the input table are read by default,"
    + " but this option can take a SQL text filtering statement, similar to a WHERE clause;"
    + " currently, only a single predicate that is a comparison between a column and a constant"
    + " value is supported; a likely choice to limit partitions would be something like"
    + " \"CAST(submission_timestamp AS DATE) BETWEEN '2020-01-10' AND '2020-01-14'\"; see"
    + " https://cloud.google.com/bigquery/docs/reference/storage/rpc/google.cloud.bigquery.storage.v1beta1#tablereadoptions")
String getBqRowRestriction();
 

@Description(
    "The window duration in which data will be written. Defaults to 5m. "
        + "Allowed formats are: "
        + "Ns (for seconds, example: 5s), "
        + "Nm (for minutes, example: 12m), "
        + "Nh (for hours, example: 2h).")
@Default.String("1m")
String getWindowDuration();
 

@Description(
    "Fixed window size in seconds (default 60) to apply before aggregating the numerical value of "
        + "features and exporting the aggregated values as metrics. Refer to "
        + "feast/ingestion/transform/metrics/WriteFeatureValueMetricsDoFn.java"
        + "for the metric nameas and types used.")
@Default.Integer(60)
int getWindowSizeInSecForFeatureValueMetric();
 

@Description("Path to the data file(s) containing game data.")
/* The default maps to two large Google Cloud Storage files (each ~12GB) holding two subsequent
day's worth (roughly) of data.

Note: You may want to use a small sample dataset to test it locally/quickly : gs://apache-beam-samples/game/small/gaming_data.csv
You can also download it via the command line gsutil cp gs://apache-beam-samples/game/small/gaming_data.csv ./destination_folder/gaming_data.csv */
@Default.String("gs://apache-beam-samples/game/gaming_data*.csv")
String getInput();
 

@Description("A JSON-formatted map of channel name to sampling ratio; for example,"
    + " {\"nightly\":1.0,\"release\":0.01} would republish 100% of nightly pings to the"
    + " sampled nightly topic and 1% of release pings to the sampled release topic")
Map<String, Double> getPerChannelSampleRatios();
 

@Description("The prefix of the files to write to. Default is: output.")
@Default.String("output")
String getOutputFilePrefix();
 

@Description("Compression format for --errorOutputType=file")
@Default.Enum("GZIP")
Compression getErrorOutputFileCompression();
 

@Description("The maximum number of output shards produced when writing.")
@Default.Integer(1)
Integer getNumShards();
 

@Description(
    "The filename prefix of the files to write to. Default file prefix is set to \"output\". ")
String getOutputFilenamePrefix();
 

@Description("RowKeySeparator")
@Default.String("#")
ValueProvider<String> getRowKeySeparator();
 

@Description("The table to set as staging table.")
@Default.String("")
String getStagingTable();
 

@Description("Cassandra keyspace to read from")
ValueProvider<String> getCassandraKeyspace();
 

@Description("A pattern for output topic names (assuming --outputType=pubsub) for per-channel"
    + " sampling; the pattern must contain a placeholder ${channel} that will be filled in"
    + " to give a distinct publisher per channel configured in --perChannelSampleRatios")
String getPerChannelDestination();
 

@Description("Path (local or gs://) to a file containing the bytes that will be used as"
    + "the hash key for the client IP")
ValueProvider<String> getClientIpHashKey();
 

@Description("The Cloud Pub/Sub topic to read from.")
@Required
ValueProvider<String> getInputTopic();
 

@Description("Name of the BQ column storing the unique identifier of the row")
ValueProvider<String> getReadIdColumn();
 

@Description("UDF Javascript Function Name")
String getJavascriptTextTransformFunctionName();
 

@Description("Indicates rate of messages per second to be published to Pub/Sub.")
@Required
Long getQps();
 

@Description(
    "KMS Encryption Key should be in the format projects/{gcp_project}/locations/{key_region}/keyRings/{key_ring}/cryptoKeys/{kms_key_name}")
ValueProvider<String> getKMSEncryptionKey();
 

@Description("JSON file with BigQuery Schema description")
ValueProvider<String> getJSONPath();
 

@Description("BigQuery table name")
@Default.InstanceFactory(BigQueryTableFactory.class)
String getBigQueryTable();
 

@Description(
    "The shard template of the output file. Specified as repeating sequences "
        + "of the letters 'S' or 'N' (example: SSS-NNN). These are replaced with the "
        + "shard number, or number of shards respectively")
@Default.String("W-P-SS-of-NN")
ValueProvider<String> getOutputShardTemplate();
 

@Description(
    "Regex filter pattern to use in DebuggingWordCount. "
        + "Only words matching this pattern will be counted.")
@Default.String("Flourish|stomach")
String getFilterPattern();
 

@Description("A comma-separated list of docTypes for which we will not accumulate an"
    + " additional_properties field before publishing to BigQuery;"
    + " this is especially useful for telemetry/main where we expect to send the"
    + " same payload to multiple tables, each with only a subset of the overall schema;"
    + " each docType must be qualified with a namespace like 'telemetry/main'")
ValueProvider<List<String>> getBqStrictSchemaDocTypes();
 

@Description("The directory to output files to. Must end with a slash. ")
String getOutputDirectory();
 

@Description(
    "The JDBC connection URL string. " + "for example: jdbc:mysql://some-host:3306/sampledb")
ValueProvider<String> getConnectionURL();
 

@Description("JDBC connection user name. ")
ValueProvider<String> getUsername();