Java Code Examples for org.apache.beam.sdk.options.PipelineOptionsValidator#validate()

The following examples show how to use org.apache.beam.sdk.options.PipelineOptionsValidator#validate() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: PipelineRunner.java    From beam with Apache License 2.0 6 votes vote down vote up
/**
 * Constructs a runner from the provided {@link PipelineOptions}.
 *
 * @return The newly created runner.
 */
public static PipelineRunner<? extends PipelineResult> fromOptions(PipelineOptions options) {
  checkNotNull(options);
  PipelineOptionsValidator.validate(PipelineOptions.class, options);

  // (Re-)register standard FileSystems. Clobbers any prior credentials.
  FileSystems.setDefaultPipelineOptions(options);

  @SuppressWarnings("unchecked")
  PipelineRunner<? extends PipelineResult> result =
      InstanceBuilder.ofType(PipelineRunner.class)
          .fromClass(options.getRunner())
          .fromFactoryMethod("fromOptions")
          .withArg(PipelineOptions.class, options)
          .build();
  return result;
}
 
Example 2
Source File: SparkStructuredStreamingRunner.java    From beam with Apache License 2.0 6 votes vote down vote up
/**
 * Creates and returns a new SparkStructuredStreamingRunner with specified options.
 *
 * @param options The PipelineOptions to use when executing the job.
 * @return A pipeline runner that will execute with specified options.
 */
public static SparkStructuredStreamingRunner fromOptions(PipelineOptions options) {
  SparkStructuredStreamingPipelineOptions sparkOptions =
      PipelineOptionsValidator.validate(SparkStructuredStreamingPipelineOptions.class, options);

  if (sparkOptions.getFilesToStage() == null
      && !PipelineTranslator.isLocalSparkMaster(sparkOptions)) {
    sparkOptions.setFilesToStage(
        detectClassPathResourcesToStage(
            SparkStructuredStreamingRunner.class.getClassLoader(), options));
    LOG.info(
        "PipelineOptions.filesToStage was not specified. "
            + "Defaulting to files from the classpath: will stage {} files. "
            + "Enable logging at DEBUG level to see which files will be staged.",
        sparkOptions.getFilesToStage().size());
    LOG.debug("Classpath elements: {}", sparkOptions.getFilesToStage());
  }

  return new SparkStructuredStreamingRunner(sparkOptions);
}
 
Example 3
Source File: SparkRunner.java    From beam with Apache License 2.0 6 votes vote down vote up
/**
 * Creates and returns a new SparkRunner with specified options.
 *
 * @param options The PipelineOptions to use when executing the job.
 * @return A pipeline runner that will execute with specified options.
 */
public static SparkRunner fromOptions(PipelineOptions options) {
  SparkPipelineOptions sparkOptions =
      PipelineOptionsValidator.validate(SparkPipelineOptions.class, options);

  if (sparkOptions.getFilesToStage() == null
      && !SparkPipelineOptions.isLocalSparkMaster(sparkOptions)) {
    sparkOptions.setFilesToStage(
        detectClassPathResourcesToStage(SparkRunner.class.getClassLoader(), options));
    LOG.info(
        "PipelineOptions.filesToStage was not specified. "
            + "Defaulting to files from the classpath: will stage {} files. "
            + "Enable logging at DEBUG level to see which files will be staged.",
        sparkOptions.getFilesToStage().size());
    LOG.debug("Classpath elements: {}", sparkOptions.getFilesToStage());
  }

  return new SparkRunner(sparkOptions);
}
 
Example 4
Source File: TestSamzaRunner.java    From beam with Apache License 2.0 6 votes vote down vote up
public static SamzaPipelineOptions createSamzaPipelineOptions(PipelineOptions options) {
  try {
    final SamzaPipelineOptions samzaOptions =
        PipelineOptionsValidator.validate(SamzaPipelineOptions.class, options);
    final Map<String, String> config = new HashMap<>(ConfigBuilder.localRunConfig());
    final File storeDir =
        Paths.get(System.getProperty("java.io.tmpdir"), "beam-samza-test").toFile();
    //  Re-create the folder for test stores
    FileUtils.deleteDirectory(storeDir);
    if (!storeDir.mkdir()) {
      // ignore
    }

    config.put(JOB_LOGGED_STORE_BASE_DIR, storeDir.getAbsolutePath());
    config.put(JOB_NON_LOGGED_STORE_BASE_DIR, storeDir.getAbsolutePath());

    if (samzaOptions.getConfigOverride() != null) {
      config.putAll(samzaOptions.getConfigOverride());
    }
    samzaOptions.setConfigOverride(config);
    return samzaOptions;
  } catch (Exception e) {
    throw new RuntimeException(e);
  }
}
 
Example 5
Source File: Twister2Runner.java    From beam with Apache License 2.0 5 votes vote down vote up
public static Twister2Runner fromOptions(PipelineOptions options) {
  Twister2PipelineOptions pipelineOptions =
      PipelineOptionsValidator.validate(Twister2PipelineOptions.class, options);
  if (pipelineOptions.getFilesToStage() == null) {
    pipelineOptions.setFilesToStage(
        detectClassPathResourcesToStage(Twister2Runner.class.getClassLoader(), pipelineOptions));
    LOG.info(
        "PipelineOptions.filesToStage was not specified. "
            + "Defaulting to files from the classpath: will stage {} files. "
            + "Enable logging at DEBUG level to see which files will be staged"
            + pipelineOptions.getFilesToStage().size());
  }
  return new Twister2Runner(pipelineOptions);
}
 
Example 6
Source File: PortableRunner.java    From beam with Apache License 2.0 5 votes vote down vote up
@VisibleForTesting
static PortableRunner create(PipelineOptions options, ManagedChannelFactory channelFactory) {
  PortablePipelineOptions portableOptions =
      PipelineOptionsValidator.validate(PortablePipelineOptions.class, options);

  String endpoint = portableOptions.getJobEndpoint();

  return new PortableRunner(options, endpoint, channelFactory);
}
 
Example 7
Source File: SparkRunnerDebugger.java    From beam with Apache License 2.0 5 votes vote down vote up
public static SparkRunnerDebugger fromOptions(PipelineOptions options) {
  if (options instanceof TestSparkPipelineOptions) {
    TestSparkPipelineOptions testSparkPipelineOptions =
        PipelineOptionsValidator.validate(TestSparkPipelineOptions.class, options);
    return new SparkRunnerDebugger(testSparkPipelineOptions);
  } else {
    SparkPipelineOptions sparkPipelineOptions =
        PipelineOptionsValidator.validate(SparkPipelineOptions.class, options);
    return new SparkRunnerDebugger(sparkPipelineOptions);
  }
}
 
Example 8
Source File: Twister2LegacyRunner.java    From twister2 with Apache License 2.0 4 votes vote down vote up
public static Twister2LegacyRunner fromOptions(PipelineOptions options) {
  Twister2PipelineOptions pipelineOptions =
      PipelineOptionsValidator.validate(Twister2PipelineOptions.class, options);
  return new Twister2LegacyRunner(pipelineOptions);
}
 
Example 9
Source File: Twister2TestRunner.java    From beam with Apache License 2.0 4 votes vote down vote up
public static Twister2TestRunner fromOptions(PipelineOptions options) {
  Twister2PipelineOptions pipelineOptions =
      PipelineOptionsValidator.validate(Twister2PipelineOptions.class, options);
  return new Twister2TestRunner(pipelineOptions);
}
 
Example 10
Source File: SamzaRunner.java    From beam with Apache License 2.0 4 votes vote down vote up
public static SamzaRunner fromOptions(PipelineOptions opts) {
  final SamzaPipelineOptions samzaOptions =
      PipelineOptionsValidator.validate(SamzaPipelineOptions.class, opts);
  return new SamzaRunner(samzaOptions);
}
 
Example 11
Source File: IOITHelper.java    From beam with Apache License 2.0 3 votes vote down vote up
public static <T extends PipelineOptions> T readIOTestPipelineOptions(Class<T> optionsType) {

    PipelineOptionsFactory.register(optionsType);
    PipelineOptions options = TestPipeline.testingPipelineOptions().as(optionsType);

    return PipelineOptionsValidator.validate(optionsType, options);
  }
 
Example 12
Source File: NemoRunner.java    From incubator-nemo with Apache License 2.0 2 votes vote down vote up
/**
 * Static initializer for creating PipelineRunner with the given options.
 *
 * @param options given PipelineOptions.
 * @return The created PipelineRunner.
 */
public static NemoRunner fromOptions(final PipelineOptions options) {
  final NemoPipelineOptions nemoOptions = PipelineOptionsValidator.validate(NemoPipelineOptions.class, options);
  return new NemoRunner(nemoOptions);
}
 
Example 13
Source File: NemoPipelineRunner.java    From nemo with Apache License 2.0 2 votes vote down vote up
/**
 * Static initializer for creating PipelineRunner with the given options.
 * @param options given PipelineOptions.
 * @return The created PipelineRunner.
 */
public static PipelineRunner<NemoPipelineResult> fromOptions(final PipelineOptions options) {
  final NemoPipelineOptions nemoOptions = PipelineOptionsValidator.validate(NemoPipelineOptions.class, options);
  return new NemoPipelineRunner(nemoOptions);
}
 
Example 14
Source File: FlinkRunner.java    From beam with Apache License 2.0 2 votes vote down vote up
/**
 * Construct a runner from the provided options.
 *
 * @param options Properties which configure the runner.
 * @return The newly created runner.
 */
public static FlinkRunner fromOptions(PipelineOptions options) {
  FlinkPipelineOptions flinkOptions =
      PipelineOptionsValidator.validate(FlinkPipelineOptions.class, options);
  return new FlinkRunner(flinkOptions);
}