Java Code Examples for com.google.cloud.dataflow.sdk.options.PipelineOptionsFactory#create()

The following examples show how to use com.google.cloud.dataflow.sdk.options.PipelineOptionsFactory#create() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: GCSFilesSourceTest.java    From policyscanner with Apache License 2.0 6 votes vote down vote up 
@Test
public void testBundleSplitIsJustSource() throws Exception {
  PipelineOptions options = PipelineOptionsFactory.create();
  List<GCSFilesSource> bundles = source.splitIntoBundles(0, null);
  assertEquals(bundles.size(), 1);
  assertEquals(bundles.get(0), source);

  bundles = source.splitIntoBundles(0, options);
  assertEquals(bundles.size(), 1);
  assertEquals(bundles.get(0), source);

  bundles = source.splitIntoBundles(1, options);
  assertEquals(bundles.size(), 1);
  assertEquals(bundles.get(0), source);

  bundles = source.splitIntoBundles(100000, options);
  assertEquals(bundles.size(), 1);
  assertEquals(bundles.get(0), source);

  bundles = source.splitIntoBundles(10, null);
  assertEquals(bundles.size(), 1);
  assertEquals(bundles.get(0), source);
}
Example 2
Source File: GCSFilesSourceTest.java    From policyscanner with Apache License 2.0 6 votes vote down vote up 
@Test
public void testReaderGetCurrent() {
  String projectName = "sampleProject";
  String objectName = REPOSITORY + this.source.getDirDelimiter() + projectName;
  String fileContent = "sample file content";
  ByteArrayOutputStream[] out = new ByteArrayOutputStream[1];
  PipelineOptions options = PipelineOptionsFactory.create();

  setUpGetFilesPage(objectName);
  setUpGetFileContent(fileContent, out);

  try {
    BoundedReader<KV<List<String>, String>> reader = this.source.createReader(options);
    reader.start();
    KV<List<String>, String> value = reader.getCurrent();
    assertEquals(value.getKey().size(), 2);
    assertEquals(value.getKey().get(0), REPOSITORY);
    assertEquals(value.getKey().get(1), projectName);
    assertEquals(value.getValue(), fileContent);
  } catch (IOException e) {
    fail();
  }
}
Example 3
Source File: GCSFilesSourceTest.java    From policyscanner with Apache License 2.0 6 votes vote down vote up 
@Test
public void testReaderAdvance() {
  String objectName = REPOSITORY + this.source.getDirDelimiter() + "sampleProject";
  PipelineOptions options = PipelineOptionsFactory.create();
  BoundedReader<KV<List<String>, String>> reader;

  try {
    setUpGetFilesPage(objectName, 0);
    reader = this.source.createReader(options);
    assertFalse(reader.start());

    setUpGetFilesPage(objectName, 1);
    reader = this.source.createReader(options);
    assertTrue(reader.start());
    assertFalse(reader.advance());

    setUpGetFilesPage(objectName, 2);
    reader = this.source.createReader(options);
    assertTrue(reader.start());
    assertTrue(reader.advance());
    assertFalse(reader.advance());
  } catch (IOException e) {
    fail();
  }
}
Example 4
Source File: LiveProjectSourceTest.java    From policyscanner with Apache License 2.0 6 votes vote down vote up 
@Test
public void testBundleSplitIsJustSource() throws Exception {
  PipelineOptions options = PipelineOptionsFactory.create();
  List<LiveProjectSource> bundles = source.splitIntoBundles(0, null);
  assertEquals(bundles.size(), 1);
  assertEquals(bundles.get(0), source);

  bundles = source.splitIntoBundles(0, options);
  assertEquals(bundles.size(), 1);
  assertEquals(bundles.get(0), source);

  bundles = source.splitIntoBundles(1, options);
  assertEquals(bundles.size(), 1);
  assertEquals(bundles.get(0), source);

  bundles = source.splitIntoBundles(100000, options);
  assertEquals(bundles.size(), 1);
  assertEquals(bundles.get(0), source);

  bundles = source.splitIntoBundles(10, null);
  assertEquals(bundles.size(), 1);
  assertEquals(bundles.get(0), source);
}
Example 5
Source File: LiveProjectSourceTest.java    From policyscanner with Apache License 2.0 6 votes vote down vote up 
@Test
public void testAdvanceWithoutStart() {
  PipelineOptions options = PipelineOptionsFactory.create();
  LiveProjectReader reader;

  this.listProjectsResponse.setProjects(new ArrayList<Project>(0));
  this.listProjectsResponse.setNextPageToken(null);
  try {
    reader = (LiveProjectReader) this.source.createReader(options);
    assertFalse(reader.advance());
    assertNull(reader.getNextPageToken());
    assertTrue(reader.getProjects().isEmpty());
    reader.getCurrent();
  } catch (IOException e) {
    fail("IOException in reader.start");
  } catch (NoSuchElementException ignored) {
    // test passed.
  }
}
Example 6
Source File: GCSFilesSourceTest.java    From policyscanner with Apache License 2.0 5 votes vote down vote up 
@Test
public void testReaderStart() {
  String objectName = REPOSITORY + this.source.getDirDelimiter() + "sampleProject";
  PipelineOptions options = PipelineOptionsFactory.create();
  setUpGetFilesPage(objectName);
  try {
    assertTrue(this.source.createReader(options).start());
  } catch (IOException e) {
    fail();
  }
}
Example 7
Source File: LiveProjectSourceTest.java    From policyscanner with Apache License 2.0 5 votes vote down vote up 
@Test
public void testAdvanceWhenPageTokenNull() {
  String projectName = "sampleProjectName";
  String projectId = "sampleProjectId";
  String orgId = ORG;
  ResourceId resourceId = new ResourceId().setId(orgId);
  GCPProject gcpProject = new GCPProject(projectId, orgId, projectName);
  Project project =
      new Project()
          .setProjectId(projectId)
          .setParent(resourceId)
          .setName(projectName)
          .setLifecycleState("ACTIVE");
  List<Project> projects = Arrays.asList(project);
  PipelineOptions options = PipelineOptionsFactory.create();
  LiveProjectReader reader;

  this.listProjectsResponse.setProjects(projects);
  this.listProjectsResponse.setNextPageToken(null);
  try {
    reader = (LiveProjectReader) this.source.createReader(options);
    assertTrue(reader.start());
    assertEquals(reader.getNextPageToken(), null);
    assertEquals(reader.getCurrent(), gcpProject);
    assertFalse(reader.advance());
    reader.getCurrent();
    fail("No exception when reading from empty source");
  } catch (IOException e) {
    fail("IOException in reader.start");
  } catch (NoSuchElementException ignored) {
    // test passed.
  }
}
Example 8
Source File: LiveStateCheckerTest.java    From policyscanner with Apache License 2.0 4 votes vote down vote up 
@Test
public void testUnmatchedStatesOutputIsCorrect() throws IOException {
  // create the policy for the live project
  String editorRole = "roles/editor";
  String editorMember = "serviceAccount:[email protected]";
  String ownerRole = "roles/owner";
  String ownerMember = "user:[email protected]";
  String fileContent = "[\n"
      + "      {\n"
      + "        \"role\": \"" + ownerRole + "\",\n"
      + "        \"members\": [\n"
      + "          \"" + ownerMember + "\"\n"
      + "        ]\n"
      + "      },\n"
      + "      {\n"
      + "        \"role\": \"" + editorRole + "\",\n"
      + "        \"members\": [\n"
      + "          \"" + editorMember + "\"\n"
      + "        ]\n"
      + "      }\n"
      + "    ]";
  String liveProjectName = "someLiveProjectName";
  String liveProjectId = "someLiveProjectId";
  String orgId = ORG_ID;
  ResourceId resourceId = new ResourceId().setId(orgId);
  Project liveProject =
      new Project()
          .setProjectId(liveProjectId)
          .setParent(resourceId)
          .setName(liveProjectName)
          .setLifecycleState("ACTIVE");
  Binding editorBinding = new Binding()
      .setRole(editorRole)
      .setMembers(Arrays.asList(editorMember));
  Binding ownerBinding = new Binding()
      .setRole(ownerRole)
      .setMembers(Arrays.asList(ownerMember));
  List<Binding> bindings = Arrays.asList(ownerBinding, editorBinding);
  Policy iamPolicy = new Policy().setBindings(bindings);
  // when calling projects().list(), return the live project
  when(listProjects.execute())
  .thenReturn(this.listProjectsResponse
      .setNextPageToken("halting string")
      .setProjects(Arrays.asList(liveProject)));
  when(this.getIamPolicy.execute()).thenReturn(iamPolicy);

  // mock out the desired policy
  String desiredProjectId = "someKnownGoodProject";
  String desiredPolicyPath = ORG_ID + DELIM + desiredProjectId + DELIM + POLICY_FILE;

  setUpGetFileContent(fileContent);
  setUpGetFilesPage(desiredPolicyPath);

  PipelineOptions options = PipelineOptionsFactory.create();

  LiveStateChecker liveStateChecker =
      new LiveStateChecker(options, this.checkedSource, ORG_ID)
        .build();

  String[] expectedOutput = new String[] {
      "DESIRED:someKnownGoodProject",
      "LIVE:someLiveProjectId"
  };

  DataflowAssert
      .that(liveStateChecker.getUnmatchedStatesOutput())
      .containsInAnyOrder(expectedOutput);

  liveStateChecker.run();
}
Example 9
Source File: LiveStateCheckerApp.java    From policyscanner with Apache License 2.0 4 votes vote down vote up 
private PipelineOptions getLocalExecutionOptions() {
  return PipelineOptionsFactory.create();
}
Example 10
Source File: LiveStateCheckerTest.java    From policyscanner with Apache License 2.0 4 votes vote down vote up 
@Test
public void testPipeline() throws IOException {
  String editorRole = "roles/editor";
  String editorMember = "serviceAccount:[email protected]";
  String editorMemberLive = "serviceAccount:[email protected]";
  String ownerRole = "roles/owner";
  String ownerMember = "user:[email protected]";
  String fileContent = "[\n"
      + "      {\n"
      + "        \"role\": \"" + ownerRole + "\",\n"
      + "        \"members\": [\n"
      + "          \"" + ownerMember + "\"\n"
      + "        ]\n"
      + "      },\n"
      + "      {\n"
      + "        \"role\": \"" + editorRole + "\",\n"
      + "        \"members\": [\n"
      + "          \"" + editorMember + "\"\n"
      + "        ]\n"
      + "      }\n"
      + "    ]";
  String filePath = ORG_ID + DELIM + PROJECT_ID + DELIM + POLICY_FILE;
  String projectName = "sampleProjectName";
  String projectId = PROJECT_ID;
  String orgId = ORG_ID;
  ResourceId resourceId = new ResourceId().setId(orgId);
  Project project =
      new Project()
          .setProjectId(projectId)
          .setParent(resourceId)
          .setName(projectName)
          .setLifecycleState("ACTIVE");
  Binding editorBinding = new Binding()
      .setRole(editorRole)
      .setMembers(Arrays.asList(editorMemberLive));
  Binding ownerBinding = new Binding()
      .setRole(ownerRole)
      .setMembers(Arrays.asList(ownerMember));
  List<Binding> bindings = Arrays.asList(ownerBinding, editorBinding);
  Policy iamPolicy = new Policy().setBindings(bindings);
  PipelineOptions options = PipelineOptionsFactory.create();

  setUpGetFileContent(fileContent);
  setUpGetFilesPage(filePath);
  when(listProjects.execute())
      .thenReturn(this.listProjectsResponse
          .setNextPageToken("halting string")
          .setProjects(Arrays.asList(project)));
  when(this.getIamPolicy.execute()).thenReturn(iamPolicy);

  GCPProject.setProjectsApiStub(this.projectsObject);

  // setting up the output objects.
  GCPProject gcpProject = new GCPProject(projectId, orgId, projectName);
  PolicyBinding ownerPolicyBinding = new PolicyBinding(ownerRole, Arrays.asList(ownerMember));
  PolicyBinding editorPolicyBinding =
      new PolicyBinding(editorRole, Arrays.asList(editorMember));
  PolicyBinding editorPolicyBindingLive =
      new PolicyBinding(editorRole, Arrays.asList(editorMemberLive));
  GCPResourcePolicy desiredPolicy = new GCPResourcePolicy(
      gcpProject, Arrays.asList(ownerPolicyBinding, editorPolicyBinding));
  GCPResourcePolicy livePolicy = new GCPResourcePolicy(
      gcpProject, Arrays.asList(ownerPolicyBinding, editorPolicyBindingLive));
  GCPResourcePolicyDiff diff = GCPResourcePolicyDiff.diff(desiredPolicy, livePolicy);
  MessageConstructor messageConstructor =
      new MessageConstructor(gcpProject, desiredPolicy, livePolicy, diff);

  new LiveStateChecker(options, this.checkedSource, ORG_ID)
      .build()
      .appendAssertContains(new String[]{messageConstructor.constructMessage()})
      .run();
}
Example 11
Source File: DesiredStateEnforcerTest.java    From policyscanner with Apache License 2.0 4 votes vote down vote up 
@Test
public void testPipeline() throws IOException {
  String editorRole = "roles/editor";
  String editorMember = "serviceAccount:[email protected]";
  String editorMemberLive = "serviceAccount:[email protected]";
  String ownerRole = "roles/owner";
  String ownerMember = "user:[email protected]";
  String fileContent = "[\n"
      + "      {\n"
      + "        \"role\": \"" + ownerRole + "\",\n"
      + "        \"members\": [\n"
      + "          \"" + ownerMember + "\"\n"
      + "        ]\n"
      + "      },\n"
      + "      {\n"
      + "        \"role\": \"" + editorRole + "\",\n"
      + "        \"members\": [\n"
      + "          \"" + editorMember + "\"\n"
      + "        ]\n"
      + "      }\n"
      + "    ]";
  String filePath = ORG_ID + DELIM + PROJECT_ID + DELIM + POLICY_FILE;
  String projectName = "sampleProjectName";
  String projectId = PROJECT_ID;
  String orgId = ORG_ID;
  ResourceId resourceId = new ResourceId().setId(orgId);
  Project project =
      new Project()
          .setProjectId(projectId)
          .setParent(resourceId)
          .setName(projectName)
          .setLifecycleState("ACTIVE");
  Binding liveEditorBinding = new Binding()
      .setRole(editorRole)
      .setMembers(Arrays.asList(editorMemberLive));
  Binding editorBinding = new Binding()
      .setRole(editorRole)
      .setMembers(Arrays.asList(editorMember));
  Binding ownerBinding = new Binding()
      .setRole(ownerRole)
      .setMembers(Arrays.asList(ownerMember));
  List<Binding> liveBindings = Arrays.asList(ownerBinding, liveEditorBinding);
  List<Binding> checkedBindings = Arrays.asList(ownerBinding, editorBinding);
  Policy liveIamPolicy = new Policy().setBindings(liveBindings);
  Policy checkedIamPolicy = new Policy().setBindings(checkedBindings);
  PipelineOptions options = PipelineOptionsFactory.create();
  final Policy[] fixedPolicy = new Policy[1];

  setUpGetFileContent(fileContent);
  setUpGetFilesPage(filePath);
  setUpSetIamPolicy(fixedPolicy);
  when(listProjects.execute())
      .thenReturn(this.listProjectsResponse
          .setNextPageToken("halting string")
          .setProjects(Arrays.asList(project)));
  when(this.getIamPolicy.execute()).thenReturn(liveIamPolicy);

  GCPProject.setProjectsApiStub(this.projectsObject);

  // setting up the output objects.
  GCPProject gcpProject = new GCPProject(projectId, orgId, projectName);
  PolicyBinding ownerPolicyBinding = new PolicyBinding(ownerRole, Arrays.asList(ownerMember));
  PolicyBinding editorPolicyBinding =
      new PolicyBinding(editorRole, Arrays.asList(editorMember));
  PolicyBinding editorPolicyBindingLive =
      new PolicyBinding(editorRole, Arrays.asList(editorMemberLive));
  GCPResourcePolicy checkedPolicy = new GCPResourcePolicy(
      gcpProject, Arrays.asList(ownerPolicyBinding, editorPolicyBinding));
  GCPResourcePolicy livePolicy = new GCPResourcePolicy(
      gcpProject, Arrays.asList(ownerPolicyBinding, editorPolicyBindingLive));
  Map<StateSource, GCPResourceState> outputMap = new HashMap<>(2);
  outputMap.put(StateSource.DESIRED, checkedPolicy);
  outputMap.put(StateSource.LIVE, livePolicy);

  try {
    new DesiredStateEnforcer(options, this.checkedSource, ORG_ID)
        .appendAssertContains(new String[]{constructMessage(gcpProject, outputMap)})
        .run();
  } catch (AggregatorRetrievalException are) {
    are.printStackTrace();
  }
  assertEquals(fixedPolicy[0], checkedIamPolicy);
}
Example 12
Source File: OnDemandLiveStateCheckerTest.java    From policyscanner with Apache License 2.0 4 votes vote down vote up 
@Test
public void testPipeline() throws IOException {
  String editorRole = "roles/editor";
  String editorMember = "serviceAccount:[email protected]";
  String editorMemberLive = "serviceAccount:[email protected]";
  String ownerRole = "roles/owner";
  String ownerMember = "user:[email protected]";
  String fileContent = "[\n"
      + "      {\n"
      + "        \"role\": \"" + ownerRole + "\",\n"
      + "        \"members\": [\n"
      + "          \"" + ownerMember + "\"\n"
      + "        ]\n"
      + "      },\n"
      + "      {\n"
      + "        \"role\": \"" + editorRole + "\",\n"
      + "        \"members\": [\n"
      + "          \"" + editorMember + "\"\n"
      + "        ]\n"
      + "      }\n"
      + "    ]";
  String filePath = ORG_ID + DELIM + PROJECT_ID + DELIM + POLICY_FILE;
  String projectName = "sampleProjectName";
  String projectId = PROJECT_ID;
  String orgId = ORG_ID;
  ResourceId resourceId = new ResourceId().setId(orgId);
  Project project =
      new Project()
          .setProjectId(projectId)
          .setParent(resourceId)
          .setName(projectName)
          .setLifecycleState("ACTIVE");
  Binding editorBinding = new Binding()
      .setRole(editorRole)
      .setMembers(Arrays.asList(editorMemberLive));
  Binding ownerBinding = new Binding()
      .setRole(ownerRole)
      .setMembers(Arrays.asList(ownerMember));
  List<Binding> bindings = Arrays.asList(ownerBinding, editorBinding);
  Policy iamPolicy = new Policy().setBindings(bindings);
  PipelineOptions options = PipelineOptionsFactory.create();

  setUpGetFileContent(fileContent);
  setUpGetFilesPage(filePath);
  when(listProjects.execute())
      .thenReturn(this.listProjectsResponse
          .setNextPageToken("halting string")
          .setProjects(Arrays.asList(project)));
  when(this.getIamPolicy.execute()).thenReturn(iamPolicy);

  GCPProject.setProjectsApiStub(this.projectsObject);

  // setting up the output objects.
  GCPProject gcpProject = new GCPProject(projectId, orgId, projectName);
  PolicyBinding ownerPolicyBinding = new PolicyBinding(ownerRole, Arrays.asList(ownerMember));
  PolicyBinding editorPolicyBinding =
      new PolicyBinding(editorRole, Arrays.asList(editorMember));
  PolicyBinding editorPolicyBindingLive =
      new PolicyBinding(editorRole, Arrays.asList(editorMemberLive));
  GCPResourcePolicy checkedPolicy = new GCPResourcePolicy(
      gcpProject, Arrays.asList(ownerPolicyBinding, editorPolicyBinding));
  GCPResourcePolicy livePolicy = new GCPResourcePolicy(
      gcpProject, Arrays.asList(ownerPolicyBinding, editorPolicyBindingLive));
  Map<StateSource, GCPResourceState> outputMap = new HashMap<>(2);
  outputMap.put(StateSource.DESIRED, checkedPolicy);
  outputMap.put(StateSource.LIVE, livePolicy);

  new OnDemandLiveStateChecker(options, this.checkedSource)
  .appendAssertContains(new String[]{constructMessage(gcpProject, outputMap)})
  .run();
}
Example 13
Source File: LiveProjectSourceTest.java    From policyscanner with Apache License 2.0 4 votes vote down vote up 
@Test
public void testAdvance() {
  String projectName = "sampleProjectName";
  String projectId = "sampleProjectId";
  String orgId = "sampleOrgId";
  ResourceId resourceId = new ResourceId().setId(orgId);
  GCPProject gcpProject = new GCPProject(projectId, orgId, projectName);
  Project project =
      new Project()
          .setProjectId(projectId)
          .setParent(resourceId)
          .setName(projectName)
          .setLifecycleState("ACTIVE");
  List<Project> projects = new ArrayList<>();
  String nextPageToken = null;
  PipelineOptions options = PipelineOptionsFactory.create();
  LiveProjectReader reader;

  projects = Arrays.asList(project);
  nextPageToken = "samplePageToken";
  this.listProjectsResponse.setProjects(projects);
  this.listProjectsResponse.setNextPageToken(nextPageToken);

  try {
    reader = (LiveProjectReader) this.source.createReader(options);
    assertTrue(reader.start());
    assertEquals(reader.getNextPageToken(), nextPageToken);
    assertEquals(reader.getProjects().size(), 1);
    assertEquals(reader.getCurrent(), gcpProject);

    this.listProjectsResponse.setNextPageToken(null);
    assertTrue(reader.advance());
    assertEquals(reader.getProjects().size(), 1);
    assertEquals(reader.getCurrent(), gcpProject);
    assertFalse(reader.advance());
    assertEquals(reader.getProjects().size(), 0);

    projects = Arrays.asList(project, project);
    this.listProjectsResponse.setProjects(projects);
    reader = (LiveProjectReader) this.source.createReader(options);
    assertTrue(reader.start());
    assertEquals(reader.getProjects().size(), 2);
    assertEquals(reader.getCurrent(), gcpProject);
    assertTrue(reader.advance());
    assertEquals(reader.getProjects().size(), 1);
    assertEquals(reader.getCurrent(), gcpProject);

    projects = new ArrayList<>();
    this.listProjectsResponse.setProjects(projects);
    assertFalse(reader.advance());
    assertEquals(reader.getProjects().size(), 0);
    assertFalse(reader.advance());
    assertEquals(reader.getProjects().size(), 0);
  } catch (IOException e) {
    fail("IOException in reader.start");
  }
}
Example 14
Source File: LiveProjectSourceTest.java    From policyscanner with Apache License 2.0 4 votes vote down vote up 
@Test
public void testProducesSortedKeys() throws Exception {
  PipelineOptions options = PipelineOptionsFactory.create();
  assertFalse(source.producesSortedKeys(options));
  assertFalse(source.producesSortedKeys(null));
}
Example 15
Source File: GCSFilesSourceTest.java    From policyscanner with Apache License 2.0 4 votes vote down vote up 
@Test
public void testProducesSortedKeys() throws Exception {
  PipelineOptions options = PipelineOptionsFactory.create();
  assertFalse(source.producesSortedKeys(options));
  assertFalse(source.producesSortedKeys(null));
}
Example 16
Source File: DesiredStateEnforcerApp.java    From policyscanner with Apache License 2.0 4 votes vote down vote up 
private PipelineOptions getLocalExecutionOptions() {
  return PipelineOptionsFactory.create();
}
Example 17
Source File: LiveStateCheckerRunner.java    From policyscanner with Apache License 2.0 4 votes vote down vote up 
private static PipelineOptions getLocalExecutionOptions() {
  return PipelineOptionsFactory.create();
}
Example 18
Source File: UserManagedKeysApp.java    From policyscanner with Apache License 2.0 4 votes vote down vote up 
private PipelineOptions getLocalExecutionOptions() {
  return PipelineOptionsFactory.create();
}