Java Code Examples for org.apache.beam.runners.dataflow.options.DataflowPipelineOptions#setGcpTempLocation()

The following examples show how to use org.apache.beam.runners.dataflow.options.DataflowPipelineOptions#setGcpTempLocation() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: DataflowGroupByKeyTest.java    From beam with Apache License 2.0 5 votes vote down vote up 
/**
 * Create a test pipeline that uses the {@link DataflowRunner} so that {@link GroupByKey} is not
 * expanded. This is used for verifying that even without expansion the proper errors show up.
 */
private Pipeline createTestServiceRunner() {
  DataflowPipelineOptions options = PipelineOptionsFactory.as(DataflowPipelineOptions.class);
  options.setRunner(DataflowRunner.class);
  options.setProject("someproject");
  options.setRegion("some-region1");
  options.setGcpTempLocation("gs://staging");
  options.setPathValidatorClass(NoopPathValidator.class);
  options.setDataflowClient(dataflow);
  return Pipeline.create(options);
}
Example 2
Source File: DataflowViewTest.java    From beam with Apache License 2.0 5 votes vote down vote up 
private Pipeline createTestBatchRunner() {
  DataflowPipelineOptions options = PipelineOptionsFactory.as(DataflowPipelineOptions.class);
  options.setRunner(DataflowRunner.class);
  options.setProject("someproject");
  options.setRegion("some-region1");
  options.setGcpTempLocation("gs://staging");
  options.setPathValidatorClass(NoopPathValidator.class);
  options.setDataflowClient(dataflow);
  return Pipeline.create(options);
}
Example 3
Source File: DataflowViewTest.java    From beam with Apache License 2.0 5 votes vote down vote up 
private Pipeline createTestStreamingRunner() {
  DataflowPipelineOptions options = PipelineOptionsFactory.as(DataflowPipelineOptions.class);
  options.setRunner(DataflowRunner.class);
  options.setStreaming(true);
  options.setProject("someproject");
  options.setRegion("some-region1");
  options.setGcpTempLocation("gs://staging");
  options.setPathValidatorClass(NoopPathValidator.class);
  options.setDataflowClient(dataflow);
  return Pipeline.create(options);
}
Example 4
Source File: DataflowRunnerTest.java    From beam with Apache License 2.0 5 votes vote down vote up 
@Test
public void testInvalidGcpTempLocation() throws IOException {
  DataflowPipelineOptions options = buildPipelineOptions();
  options.setGcpTempLocation("file://temp/location");

  thrown.expect(IllegalArgumentException.class);
  thrown.expectMessage(containsString("Expected a valid 'gs://' path but was given"));
  DataflowRunner.fromOptions(options);

  ArgumentCaptor<Job> jobCaptor = ArgumentCaptor.forClass(Job.class);
  Mockito.verify(mockJobs).create(eq(PROJECT_ID), eq(REGION_ID), jobCaptor.capture());
  assertValidJob(jobCaptor.getValue());
}
Example 5
Source File: DataflowRunnerTest.java    From beam with Apache License 2.0 5 votes vote down vote up 
@Test
public void testNonExistentTempLocation() throws IOException {
  DataflowPipelineOptions options = buildPipelineOptions();
  options.setGcpTempLocation(NON_EXISTENT_BUCKET);

  thrown.expect(IllegalArgumentException.class);
  thrown.expectMessage(
      containsString("Output path does not exist or is not writeable: " + NON_EXISTENT_BUCKET));
  DataflowRunner.fromOptions(options);

  ArgumentCaptor<Job> jobCaptor = ArgumentCaptor.forClass(Job.class);
  Mockito.verify(mockJobs).create(eq(PROJECT_ID), eq(REGION_ID), jobCaptor.capture());
  assertValidJob(jobCaptor.getValue());
}
Example 6
Source File: DataflowRunnerTest.java    From beam with Apache License 2.0 5 votes vote down vote up 
@Test
public void testGcpTempAndNoTempLocationSucceeds() throws Exception {
  DataflowPipelineOptions options = PipelineOptionsFactory.as(DataflowPipelineOptions.class);
  options.setRunner(DataflowRunner.class);
  options.setGcpCredential(new TestCredential());
  options.setProject("foo-project");
  options.setRegion(REGION_ID);
  options.setGcpTempLocation(VALID_TEMP_BUCKET);
  options.setGcsUtil(mockGcsUtil);

  DataflowRunner.fromOptions(options);
}