org.apache.beam.sdk.options.PipelineOptionsValidator Java Examples

/**
 * Constructs a runner from the provided {@link PipelineOptions}.
 *
 * @return The newly created runner.
 */
public static PipelineRunner<? extends PipelineResult> fromOptions(PipelineOptions options) {
  checkNotNull(options);
  PipelineOptionsValidator.validate(PipelineOptions.class, options);

  // (Re-)register standard FileSystems. Clobbers any prior credentials.
  FileSystems.setDefaultPipelineOptions(options);

  @SuppressWarnings("unchecked")
  PipelineRunner<? extends PipelineResult> result =
      InstanceBuilder.ofType(PipelineRunner.class)
          .fromClass(options.getRunner())
          .fromFactoryMethod("fromOptions")
          .withArg(PipelineOptions.class, options)
          .build();
  return result;
}
 

/**
 * Creates and returns a new SparkStructuredStreamingRunner with specified options.
 *
 * @param options The PipelineOptions to use when executing the job.
 * @return A pipeline runner that will execute with specified options.
 */
public static SparkStructuredStreamingRunner fromOptions(PipelineOptions options) {
  SparkStructuredStreamingPipelineOptions sparkOptions =
      PipelineOptionsValidator.validate(SparkStructuredStreamingPipelineOptions.class, options);

  if (sparkOptions.getFilesToStage() == null
      && !PipelineTranslator.isLocalSparkMaster(sparkOptions)) {
    sparkOptions.setFilesToStage(
        detectClassPathResourcesToStage(
            SparkStructuredStreamingRunner.class.getClassLoader(), options));
    LOG.info(
        "PipelineOptions.filesToStage was not specified. "
            + "Defaulting to files from the classpath: will stage {} files. "
            + "Enable logging at DEBUG level to see which files will be staged.",
        sparkOptions.getFilesToStage().size());
    LOG.debug("Classpath elements: {}", sparkOptions.getFilesToStage());
  }

  return new SparkStructuredStreamingRunner(sparkOptions);
}
 

/**
 * Creates and returns a new SparkRunner with specified options.
 *
 * @param options The PipelineOptions to use when executing the job.
 * @return A pipeline runner that will execute with specified options.
 */
public static SparkRunner fromOptions(PipelineOptions options) {
  SparkPipelineOptions sparkOptions =
      PipelineOptionsValidator.validate(SparkPipelineOptions.class, options);

  if (sparkOptions.getFilesToStage() == null
      && !SparkPipelineOptions.isLocalSparkMaster(sparkOptions)) {
    sparkOptions.setFilesToStage(
        detectClassPathResourcesToStage(SparkRunner.class.getClassLoader(), options));
    LOG.info(
        "PipelineOptions.filesToStage was not specified. "
            + "Defaulting to files from the classpath: will stage {} files. "
            + "Enable logging at DEBUG level to see which files will be staged.",
        sparkOptions.getFilesToStage().size());
    LOG.debug("Classpath elements: {}", sparkOptions.getFilesToStage());
  }

  return new SparkRunner(sparkOptions);
}
 

public static SamzaPipelineOptions createSamzaPipelineOptions(PipelineOptions options) {
  try {
    final SamzaPipelineOptions samzaOptions =
        PipelineOptionsValidator.validate(SamzaPipelineOptions.class, options);
    final Map<String, String> config = new HashMap<>(ConfigBuilder.localRunConfig());
    final File storeDir =
        Paths.get(System.getProperty("java.io.tmpdir"), "beam-samza-test").toFile();
    //  Re-create the folder for test stores
    FileUtils.deleteDirectory(storeDir);
    if (!storeDir.mkdir()) {
      // ignore
    }

    config.put(JOB_LOGGED_STORE_BASE_DIR, storeDir.getAbsolutePath());
    config.put(JOB_NON_LOGGED_STORE_BASE_DIR, storeDir.getAbsolutePath());

    if (samzaOptions.getConfigOverride() != null) {
      config.putAll(samzaOptions.getConfigOverride());
    }
    samzaOptions.setConfigOverride(config);
    return samzaOptions;
  } catch (Exception e) {
    throw new RuntimeException(e);
  }
}
 

public static Twister2Runner fromOptions(PipelineOptions options) {
  Twister2PipelineOptions pipelineOptions =
      PipelineOptionsValidator.validate(Twister2PipelineOptions.class, options);
  if (pipelineOptions.getFilesToStage() == null) {
    pipelineOptions.setFilesToStage(
        detectClassPathResourcesToStage(Twister2Runner.class.getClassLoader(), pipelineOptions));
    LOG.info(
        "PipelineOptions.filesToStage was not specified. "
            + "Defaulting to files from the classpath: will stage {} files. "
            + "Enable logging at DEBUG level to see which files will be staged"
            + pipelineOptions.getFilesToStage().size());
  }
  return new Twister2Runner(pipelineOptions);
}
 

@VisibleForTesting
static PortableRunner create(PipelineOptions options, ManagedChannelFactory channelFactory) {
  PortablePipelineOptions portableOptions =
      PipelineOptionsValidator.validate(PortablePipelineOptions.class, options);

  String endpoint = portableOptions.getJobEndpoint();

  return new PortableRunner(options, endpoint, channelFactory);
}
 

public static SparkRunnerDebugger fromOptions(PipelineOptions options) {
  if (options instanceof TestSparkPipelineOptions) {
    TestSparkPipelineOptions testSparkPipelineOptions =
        PipelineOptionsValidator.validate(TestSparkPipelineOptions.class, options);
    return new SparkRunnerDebugger(testSparkPipelineOptions);
  } else {
    SparkPipelineOptions sparkPipelineOptions =
        PipelineOptionsValidator.validate(SparkPipelineOptions.class, options);
    return new SparkRunnerDebugger(sparkPipelineOptions);
  }
}
 

public static Twister2LegacyRunner fromOptions(PipelineOptions options) {
  Twister2PipelineOptions pipelineOptions =
      PipelineOptionsValidator.validate(Twister2PipelineOptions.class, options);
  return new Twister2LegacyRunner(pipelineOptions);
}
 

public static Twister2TestRunner fromOptions(PipelineOptions options) {
  Twister2PipelineOptions pipelineOptions =
      PipelineOptionsValidator.validate(Twister2PipelineOptions.class, options);
  return new Twister2TestRunner(pipelineOptions);
}
 

public static SamzaRunner fromOptions(PipelineOptions opts) {
  final SamzaPipelineOptions samzaOptions =
      PipelineOptionsValidator.validate(SamzaPipelineOptions.class, opts);
  return new SamzaRunner(samzaOptions);
}
 

public static <T extends PipelineOptions> T readIOTestPipelineOptions(Class<T> optionsType) {

    PipelineOptionsFactory.register(optionsType);
    PipelineOptions options = TestPipeline.testingPipelineOptions().as(optionsType);

    return PipelineOptionsValidator.validate(optionsType, options);
  }
 

/**
 * Static initializer for creating PipelineRunner with the given options.
 *
 * @param options given PipelineOptions.
 * @return The created PipelineRunner.
 */
public static NemoRunner fromOptions(final PipelineOptions options) {
  final NemoPipelineOptions nemoOptions = PipelineOptionsValidator.validate(NemoPipelineOptions.class, options);
  return new NemoRunner(nemoOptions);
}
 

/**
 * Static initializer for creating PipelineRunner with the given options.
 * @param options given PipelineOptions.
 * @return The created PipelineRunner.
 */
public static PipelineRunner<NemoPipelineResult> fromOptions(final PipelineOptions options) {
  final NemoPipelineOptions nemoOptions = PipelineOptionsValidator.validate(NemoPipelineOptions.class, options);
  return new NemoPipelineRunner(nemoOptions);
}
 

/**
 * Construct a runner from the provided options.
 *
 * @param options Properties which configure the runner.
 * @return The newly created runner.
 */
public static FlinkRunner fromOptions(PipelineOptions options) {
  FlinkPipelineOptions flinkOptions =
      PipelineOptionsValidator.validate(FlinkPipelineOptions.class, options);
  return new FlinkRunner(flinkOptions);
}