Java Code Examples for org.apache.beam.sdk.options.PipelineOptionsFactory#register()

@BeforeClass
public static void setUp() {
  PipelineOptionsFactory.register(MongoDBPipelineOptions.class);
  options = TestPipeline.testingPipelineOptions().as(MongoDBPipelineOptions.class);
  collection = String.format("test_%s", new Date().getTime());
  bigQueryDataset = options.getBigQueryDataset();
  bigQueryTable = options.getBigQueryTable();
  mongoUrl =
      String.format("mongodb://%s:%s", options.getMongoDBHostName(), options.getMongoDBPort());
  mongoClient = MongoClients.create(mongoUrl);
  settings =
      InfluxDBSettings.builder()
          .withHost(options.getInfluxHost())
          .withDatabase(options.getInfluxDatabase())
          .withMeasurement(options.getInfluxMeasurement())
          .get();
}
 

@Before
public void setup() throws Exception {
  PipelineOptionsFactory.register(BigtableTestOptions.class);
  options = TestPipeline.testingPipelineOptions().as(BigtableTestOptions.class);
  project = options.as(GcpOptions.class).getProject();

  bigtableOptions =
      new Builder()
          .setProjectId(project)
          .setInstanceId(options.getInstanceId())
          .setUserAgent("apache-beam-test")
          .build();

  session =
      new BigtableSession(
          bigtableOptions
              .toBuilder()
              .setCredentialOptions(
                  CredentialOptions.credential(options.as(GcpOptions.class).getGcpCredential()))
              .build());
  tableAdminClient = session.getTableAdminClient();
}
 

@BeforeClass
public static void beforeClass() throws Exception {
  PipelineOptionsFactory.register(ElasticsearchPipelineOptions.class);
  options = TestPipeline.testingPipelineOptions().as(ElasticsearchPipelineOptions.class);
  readConnectionConfiguration =
      ElasticsearchIOITCommon.getConnectionConfiguration(
          options, ElasticsearchIOITCommon.IndexMode.READ);
  writeConnectionConfiguration =
      ElasticsearchIOITCommon.getConnectionConfiguration(
          options, ElasticsearchIOITCommon.IndexMode.WRITE);
  updateConnectionConfiguration =
      ElasticsearchIOITCommon.getConnectionConfiguration(
          options, ElasticsearchIOITCommon.IndexMode.WRITE_PARTIAL);
  restClient = readConnectionConfiguration.createClient();
  elasticsearchIOTestCommon =
      new ElasticsearchIOTestCommon(readConnectionConfiguration, restClient, true);
}
 

@Before
public void setup() {
  PipelineOptionsFactory.register(V1TestOptions.class);
  options = TestPipeline.testingPipelineOptions().as(V1TestOptions.class);
  project = TestPipeline.testingPipelineOptions().as(GcpOptions.class).getProject();
  ancestor = UUID.randomUUID().toString();
}
 

@Before
public void setUp() throws Exception {
  PipelineOptionsFactory.register(SpannerTestPipelineOptions.class);
  options = TestPipeline.testingPipelineOptions().as(SpannerTestPipelineOptions.class);

  project = options.getInstanceProjectId();
  if (project == null) {
    project = options.as(GcpOptions.class).getProject();
  }

  spanner = SpannerOptions.newBuilder().setProjectId(project).build().getService();

  databaseName = generateDatabaseName();

  databaseAdminClient = spanner.getDatabaseAdminClient();

  // Delete database if exists.
  databaseAdminClient.dropDatabase(options.getInstanceId(), databaseName);

  OperationFuture<Database, CreateDatabaseMetadata> op =
      databaseAdminClient.createDatabase(
          options.getInstanceId(),
          databaseName,
          Collections.singleton(
              "CREATE TABLE "
                  + options.getTable()
                  + " ("
                  + "  Key           INT64,"
                  + "  Value         STRING(MAX),"
                  + ") PRIMARY KEY (Key)"));
  op.get();
  makeTestData();
}
 

private void setUpTestEnvironment(String tableSize) {
  PipelineOptionsFactory.register(BigQueryIOStorageQueryOptions.class);
  options = TestPipeline.testingPipelineOptions().as(BigQueryIOStorageQueryOptions.class);
  options.setNumRecords(EXPECTED_NUM_RECORDS.get(tableSize));
  String project = TestPipeline.testingPipelineOptions().as(GcpOptions.class).getProject();
  options.setInputTable(project + '.' + DATASET_ID + '.' + TABLE_PREFIX + tableSize);
}
 

@Before
public void setUp() {
  PipelineOptionsFactory.register(BigQueryClusteringITOptions.class);
  options = TestPipeline.testingPipelineOptions().as(BigQueryClusteringITOptions.class);
  options.setTempLocation(options.getTempRoot() + "/temp-it/");
  bqClient = BigqueryClient.getNewBigquerryClient(options.getAppName());
}
 

@Before
public void setUp() throws Exception {
  PipelineOptionsFactory.register(SpannerTestPipelineOptions.class);
  options = TestPipeline.testingPipelineOptions().as(SpannerTestPipelineOptions.class);

  project = options.getInstanceProjectId();
  if (project == null) {
    project = options.as(GcpOptions.class).getProject();
  }

  spanner = SpannerOptions.newBuilder().setProjectId(project).build().getService();

  databaseName = generateDatabaseName();

  databaseAdminClient = spanner.getDatabaseAdminClient();

  // Delete database if exists.
  databaseAdminClient.dropDatabase(options.getInstanceId(), databaseName);

  OperationFuture<Database, CreateDatabaseMetadata> op =
      databaseAdminClient.createDatabase(
          options.getInstanceId(),
          databaseName,
          Collections.singleton(
              "CREATE TABLE "
                  + options.getTable()
                  + " ("
                  + "  Key           INT64,"
                  + "  Value         STRING(MAX) NOT NULL,"
                  + ") PRIMARY KEY (Key)"));
  op.get();
}
 

private QueryBuilderArgs pareOptions(String cmdLineArgs) throws IOException {
  PipelineOptionsFactory.register(JdbcExportPipelineOptions.class);
  JdbcExportPipelineOptions opts =
      PipelineOptionsFactory.fromArgs(cmdLineArgs.split(" "))
          .withValidation()
          .create()
          .as(JdbcExportPipelineOptions.class);
  return JdbcExportArgsFactory.createQueryArgs(opts);
}
 

private void setupTestEnvironment(String recordSize) {
  PipelineOptionsFactory.register(BigQueryIOReadOptions.class);
  options = TestPipeline.testingPipelineOptions().as(BigQueryIOReadOptions.class);
  options.setNumRecords(numOfRecords.get(recordSize));
  options.setTempLocation(options.getTempRoot() + "/temp-it/");
  project = TestPipeline.testingPipelineOptions().as(GcpOptions.class).getProject();
  options.setInputTable(project + ":" + datasetId + "." + tablePrefix + recordSize);
}
 

@BeforeClass
public static void setup() {
  PipelineOptionsFactory.register(KinesisTestOptions.class);
  options = TestPipeline.testingPipelineOptions().as(KinesisTestOptions.class);
  numberOfShards = options.getNumberOfShards();
  numberOfRows = options.getNumberOfRecords();
}
 

JdbcExportArgs optionsFromArgs(String[] cmdLineArgs) throws IOException, ClassNotFoundException {
  PipelineOptionsFactory.register(JdbcExportPipelineOptions.class);
  PipelineOptions opts = PipelineOptionsFactory.fromArgs(cmdLineArgs).withValidation().create();
  return JdbcExportArgsFactory.fromPipelineOptions(opts);
}
 

@BeforeClass
public static void setUp() {
  PipelineOptionsFactory.register(TestPipelineOptions.class);
}
 

@BeforeClass
public static void setUp() {
  PipelineOptionsFactory.register(TfIdfITOptions.class);
}
 

@BeforeClass
public static void setUp() {
  PipelineOptionsFactory.register(HadoopFormatIOTestOptions.class);
  options = TestPipeline.testingPipelineOptions().as(HadoopFormatIOTestOptions.class);
}
 

@BeforeClass
public static void setUp() {
  PipelineOptionsFactory.register(BigQueryTornadoesITOptions.class);
}
 

@BeforeClass
public static void setUp() {
  PipelineOptionsFactory.register(HadoopFormatIOTestOptions.class);
  options = TestPipeline.testingPipelineOptions().as(HadoopFormatIOTestOptions.class);
}
 

@BeforeClass
public static void setup() {
  PipelineOptionsFactory.register(TestPipelineOptions.class);
}
 

/**
 * Use this to create the index for reading before IT read tests.
 *
 * <p>To invoke this class, you can use this command line from elasticsearch-tests-common module
 * directory after setting the correct server IP:
 *
 * <pre>
 * mvn test-compile exec:java \
 * -Dexec.mainClass=org.apache.beam.sdk.io.elasticsearch.ElasticsearchIOITCommon \
 * -Dexec.args="--elasticsearchServer=127.0.0.1 \
 * --elasticsearchHttpPort=9200" \
 * -Dexec.classpathScope=test
 *   </pre>
 *
 * @param args Please pass options from ElasticsearchTestOptions used for connection to
 *     Elasticsearch as shown above.
 */
public static void main(String[] args) throws Exception {
  PipelineOptionsFactory.register(ElasticsearchPipelineOptions.class);
  ElasticsearchPipelineOptions options =
      PipelineOptionsFactory.fromArgs(args).as(ElasticsearchPipelineOptions.class);
  createAndPopulateReadIndex(options);
}
 

public static <T extends PipelineOptions> T readIOTestPipelineOptions(Class<T> optionsType) {

    PipelineOptionsFactory.register(optionsType);
    PipelineOptions options = TestPipeline.testingPipelineOptions().as(optionsType);

    return PipelineOptionsValidator.validate(optionsType, options);
  }