Java Code Examples for org.apache.beam.sdk.transforms.Create#of()

@Test
public void groupByKey() {
  Create.Values<String> values = Create.of("apple", "ball", "car", "bear", "cheetah", "ant");
  PCollection<String> numbers = testPipeline.apply(values);

  PCollection<KV<String, Iterable<String>>> results = Task.applyTransform(numbers);

  PAssert.that(results)
      .satisfies(
          containsKvs(
              KV.of("a", ImmutableList.of("apple", "ant")),
              KV.of("b", ImmutableList.of("ball", "bear")),
              KV.of("c", ImmutableList.of("car", "cheetah"))
          )
      );

  testPipeline.run().waitUntilFinish();
}
 

@Test
public void branching() {
  Create.Values<Integer> values = Create.of(1, 2, 3, 4, 5);
  PCollection<Integer> numbers = testPipeline.apply(values);

  PCollection<Integer> mult5Results = Task.applyMultiply5Transform(numbers);
  PCollection<Integer> mult10Results = Task.applyMultiply10Transform(numbers);

  PAssert.that(mult5Results)
      .containsInAnyOrder(5, 10, 15, 20, 25);

  PAssert.that(mult10Results)
      .containsInAnyOrder(10, 20, 30, 40, 50);

  testPipeline.run().waitUntilFinish();
}
 

@Test
public void shouldCacheTest() {
  SparkPipelineOptions options = createOptions();
  options.setCacheDisabled(true);
  Pipeline pipeline = Pipeline.create(options);

  Values<String> valuesTransform = Create.of("foo", "bar");
  PCollection pCollection = mock(PCollection.class);

  JavaSparkContext jsc = SparkContextFactory.getSparkContext(options);
  EvaluationContext ctxt = new EvaluationContext(jsc, pipeline, options);
  ctxt.getCacheCandidates().put(pCollection, 2L);

  assertFalse(ctxt.shouldCache(valuesTransform, pCollection));

  options.setCacheDisabled(false);
  assertTrue(ctxt.shouldCache(valuesTransform, pCollection));

  GroupByKey<String, String> gbkTransform = GroupByKey.create();
  assertFalse(ctxt.shouldCache(gbkTransform, pCollection));
}
 

@Test
public void parDo() {
  Create.Values<String> lines =
      Create.of(
          "apple orange grape banana apple banana",
          "banana orange banana papaya");

  PCollection<String> linesPColl = testPipeline.apply(lines);

  PCollection<String> results = Task.applyTransform(linesPColl);

  PAssert.that(results)
      .containsInAnyOrder(
          "apple:2",
          "banana:4",
          "grape:1",
          "orange:2",
          "papaya:1"
      );

  testPipeline.run().waitUntilFinish();
}
 

@Test
public void registerTransformNoChildren() throws IOException {
  Create.Values<Integer> create = Create.of(1, 2, 3);
  PCollection<Integer> pt = pipeline.apply(create);
  String userName = "my_transform/my_nesting";
  AppliedPTransform<?, ?, ?> transform =
      AppliedPTransform.of(userName, pipeline.begin().expand(), pt.expand(), create, pipeline);
  String componentName = components.registerPTransform(transform, Collections.emptyList());
  assertThat(componentName, equalTo(userName));
  assertThat(components.getExistingPTransformId(transform), equalTo(componentName));
}
 

@Test
public void min() {
  Create.Values<Integer> values = Create.of(1, 2, 3, 4, 5, 6, 7, 8, 9, 10);
  PCollection<Integer> numbers = testPipeline.apply(values);

  PCollection<Integer> results = Task.applyTransform(numbers);

  PAssert.that(results)
      .containsInAnyOrder(1);

  testPipeline.run().waitUntilFinish();
}
 

@Test
public void registerTransformEmptyFullName() throws IOException {
  Create.Values<Integer> create = Create.of(1, 2, 3);
  PCollection<Integer> pt = pipeline.apply(create);
  AppliedPTransform<?, ?, ?> transform =
      AppliedPTransform.of("", pipeline.begin().expand(), pt.expand(), create, pipeline);

  thrown.expect(IllegalArgumentException.class);
  thrown.expectMessage(transform.toString());
  components.getExistingPTransformId(transform);
}
 

@Test
public void sum() {
  Create.Values<Integer> values = Create.of(1, 2, 3, 4, 5, 6, 7, 8, 9, 10);
  PCollection<Integer> numbers = testPipeline.apply(values);

  PCollection<Integer> results = Task.applyTransform(numbers);

  PAssert.that(results)
      .containsInAnyOrder(55);

  testPipeline.run().waitUntilFinish();
}
 

@Test
public void filter_parDo() {
  Create.Values<Integer> values = Create.of(1, 2, 3, 4, 5, 6, 7, 8, 9, 10);
  PCollection<Integer> numbers = testPipeline.apply(values);

  PCollection<Integer> results = Task.applyTransform(numbers);

  PAssert.that(results)
      .containsInAnyOrder(1, 3, 5, 7, 9);

  testPipeline.run().waitUntilFinish();
}
 

@Test
public void filter() {
  Create.Values<Integer> values = Create.of(1, 2, 3, 4, 5, 6, 7, 8, 9, 10);
  PCollection<Integer> numbers = testPipeline.apply(values);

  PCollection<Integer> results = Task.applyTransform(numbers);

  PAssert.that(results)
      .containsInAnyOrder(2, 4, 6, 8, 10);

  testPipeline.run().waitUntilFinish();
}
 

@Test
public void compositeTransform() {
  Create.Values<String> values = Create.of("1,2,3,4,5", "6,7,8,9,10");

  PCollection<Integer> results =
      testPipeline
          .apply(values)
          .apply(new ExtractAndMultiplyNumbers());

  PAssert.that(results)
      .containsInAnyOrder(10, 20, 30, 40, 50, 60, 70, 80, 90, 100);

  testPipeline.run().waitUntilFinish();
}
 

@Test
public void parDo() {
  Create.Values<Integer> values = Create.of(1, 2, 3, 4, 5);
  PCollection<Integer> numbers = testPipeline.apply(values);

  PCollection<Integer> results = Task.applyTransform(numbers);

  PAssert.that(results)
      .containsInAnyOrder(10, 20, 30, 40, 50);

  testPipeline.run().waitUntilFinish();
}
 

@Test
public void flatMapElements() {
  Create.Values<String> values = Create.of("Apache Beam", "Unified Batch and Streaming");
  PCollection<String> numbers = testPipeline.apply(values);

  PCollection<String> results = Task.applyTransform(numbers);

  PAssert.that(results)
      .containsInAnyOrder("Apache", "Beam", "Unified", "Batch", "and", "Streaming");

  testPipeline.run().waitUntilFinish();
}
 

@Test
public void parDo_oneToMany() {
  Create.Values<String> values = Create.of("Hello Beam", "It is awesome");
  PCollection<String> numbers = testPipeline.apply(values);

  PCollection<String> results = Task.applyTransform(numbers);

  PAssert.that(results)
      .containsInAnyOrder("Hello", "Beam", "It", "is", "awesome");

  testPipeline.run().waitUntilFinish();
}
 

private PTransform<PBegin, PCollection<String>> getValues(
    final SparkStructuredStreamingPipelineOptions options) {
  final boolean doNotSyncWithWatermark = false;
  return options.isStreaming()
      ? CreateStream.of(StringUtf8Coder.of(), Duration.millis(1), doNotSyncWithWatermark)
          .nextBatch("one", "two")
      : Create.of("one", "two");
}
 

@Test
public void combine_binaryCombineFn() {
  Create.Values<BigInteger> values = Create.of(
      BigInteger.valueOf(10), BigInteger.valueOf(20), BigInteger.valueOf(30),
      BigInteger.valueOf(40), BigInteger.valueOf(50)
  );
  PCollection<BigInteger> numbers = testPipeline.apply(values);

  PCollection<BigInteger> results = Task.applyTransform(numbers);

  PAssert.that(results)
      .containsInAnyOrder(BigInteger.valueOf(150));

  testPipeline.run().waitUntilFinish();
}
 

@Test
public void combine_binaryCombineFn_lambda() {
  Create.Values<BigInteger> values = Create.of(
      BigInteger.valueOf(10), BigInteger.valueOf(20), BigInteger.valueOf(30),
      BigInteger.valueOf(40), BigInteger.valueOf(50)
  );
  PCollection<BigInteger> numbers = testPipeline.apply(values);

  PCollection<BigInteger> results = Task.applyTransform(numbers);

  PAssert.that(results)
      .containsInAnyOrder(BigInteger.valueOf(150));

  testPipeline.run().waitUntilFinish();
}
 

@Test
public void combine_simpleFn() {
  Create.Values<Integer> values = Create.of(10, 30, 50, 70, 90);
  PCollection<Integer> numbers = testPipeline.apply(values);

  PCollection<Integer> results = Task.applyTransform(numbers);

  PAssert.that(results)
      .containsInAnyOrder(250);

  testPipeline.run().waitUntilFinish();
}
 

@Test
public void combine_combineFn() {
  Create.Values<Integer> values = Create.of(10, 20, 50, 70, 90);
  PCollection<Integer> numbers = testPipeline.apply(values);

  PCollection<Double> results = Task.applyTransform(numbers);

  PAssert.that(results)
      .containsInAnyOrder(48.0);

  testPipeline.run().waitUntilFinish();
}
 

private PTransform<PBegin, PCollection<String>> getValues(final SparkPipelineOptions options) {
  final boolean doNotSyncWithWatermark = false;
  return options.isStreaming()
      ? CreateStream.of(StringUtf8Coder.of(), Duration.millis(1), doNotSyncWithWatermark)
          .nextBatch("one", "two")
      : Create.of("one", "two");
}