Java Code Examples for org.apache.beam.sdk.transforms.display.DisplayData#none()

The following examples show how to use org.apache.beam.sdk.transforms.display.DisplayData#none() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example 1

Source File: DoFnTransformTest.java From incubator-nemo with Apache License 2.0

5 votes

@Test
@SuppressWarnings("unchecked")
public void testSingleOutput() {

  final TupleTag<String> outputTag = new TupleTag<>("main-output");

  final DoFnTransform<String, String> doFnTransform =
    new DoFnTransform<>(
      new IdentityDoFn<>(),
      NULL_INPUT_CODER,
      NULL_OUTPUT_CODERS,
      outputTag,
      Collections.emptyList(),
      WindowingStrategy.globalDefault(),
      PipelineOptionsFactory.as(NemoPipelineOptions.class),
      DisplayData.none(),
      DoFnSchemaInformation.create(),
      Collections.emptyMap());

  final Transform.Context context = mock(Transform.Context.class);
  final OutputCollector<WindowedValue<String>> oc = new TestOutputCollector<>();
  doFnTransform.prepare(context, oc);

  doFnTransform.onData(WindowedValue.valueInGlobalWindow("Hello"));

  assertEquals(((TestOutputCollector<String>) oc).outputs.get(0), WindowedValue.valueInGlobalWindow("Hello"));

  doFnTransform.close();
}

Example 2

Source File: DoFnTransformTest.java From incubator-nemo with Apache License 2.0

4 votes

@Test
@SuppressWarnings("unchecked")
public void testCountBundle() {

  final TupleTag<String> outputTag = new TupleTag<>("main-output");
  final NemoPipelineOptions pipelineOptions = PipelineOptionsFactory.as(NemoPipelineOptions.class);
  pipelineOptions.setMaxBundleSize(3L);
  pipelineOptions.setMaxBundleTimeMills(10000000L);

  final List<Integer> bundleOutput = new ArrayList<>();

  final DoFnTransform<String, String> doFnTransform =
    new DoFnTransform<>(
      new BundleTestDoFn(bundleOutput),
      NULL_INPUT_CODER,
      NULL_OUTPUT_CODERS,
      outputTag,
      Collections.emptyList(),
      WindowingStrategy.globalDefault(),
      pipelineOptions,
      DisplayData.none(),
      DoFnSchemaInformation.create(),
      Collections.emptyMap());

  final Transform.Context context = mock(Transform.Context.class);
  final OutputCollector<WindowedValue<String>> oc = new TestOutputCollector<>();
  doFnTransform.prepare(context, oc);

  doFnTransform.onData(WindowedValue.valueInGlobalWindow("a"));
  doFnTransform.onData(WindowedValue.valueInGlobalWindow("a"));
  doFnTransform.onData(WindowedValue.valueInGlobalWindow("a"));

  assertEquals(3, (int) bundleOutput.get(0));

  bundleOutput.clear();

  doFnTransform.onData(WindowedValue.valueInGlobalWindow("a"));
  doFnTransform.onData(WindowedValue.valueInGlobalWindow("a"));
  doFnTransform.onData(WindowedValue.valueInGlobalWindow("a"));

  assertEquals(3, (int) bundleOutput.get(0));

  doFnTransform.close();
}

Example 3

Source File: DoFnTransformTest.java From incubator-nemo with Apache License 2.0

4 votes

@Test
@SuppressWarnings("unchecked")
public void testTimeBundle() {

  final long maxBundleTimeMills = 1000L;
  final TupleTag<String> outputTag = new TupleTag<>("main-output");
  final NemoPipelineOptions pipelineOptions = PipelineOptionsFactory.as(NemoPipelineOptions.class);
  pipelineOptions.setMaxBundleSize(10000000L);
  pipelineOptions.setMaxBundleTimeMills(maxBundleTimeMills);

  final List<Integer> bundleOutput = new ArrayList<>();

  final DoFnTransform<String, String> doFnTransform =
    new DoFnTransform<>(
      new BundleTestDoFn(bundleOutput),
      NULL_INPUT_CODER,
      NULL_OUTPUT_CODERS,
      outputTag,
      Collections.emptyList(),
      WindowingStrategy.globalDefault(),
      pipelineOptions,
      DisplayData.none(),
      DoFnSchemaInformation.create(),
      Collections.emptyMap());

  final Transform.Context context = mock(Transform.Context.class);
  final OutputCollector<WindowedValue<String>> oc = new TestOutputCollector<>();

  long startTime = System.currentTimeMillis();
  doFnTransform.prepare(context, oc);

  int count = 0;
  while (bundleOutput.isEmpty()) {
    doFnTransform.onData(WindowedValue.valueInGlobalWindow("a"));
    count += 1;
    try {
      Thread.sleep(10);
    } catch (InterruptedException e) {
      e.printStackTrace();
      throw new RuntimeException(e);
    }
  }

  long endTime = System.currentTimeMillis();
  assertEquals(count, (int) bundleOutput.get(0));
  assertTrue(endTime - startTime >= maxBundleTimeMills);

  doFnTransform.close();
}

Example 4

Source File: DoFnTransformTest.java From incubator-nemo with Apache License 2.0

4 votes

@Test
@SuppressWarnings("unchecked")
public void testMultiOutputOutput() {

  TupleTag<String> mainOutput = new TupleTag<>("main-output");
  TupleTag<String> additionalOutput1 = new TupleTag<>("output-1");
  TupleTag<String> additionalOutput2 = new TupleTag<>("output-2");

  ImmutableList<TupleTag<?>> tags = ImmutableList.of(additionalOutput1, additionalOutput2);

  ImmutableMap<String, String> tagsMap =
    ImmutableMap.<String, String>builder()
      .put(additionalOutput1.getId(), additionalOutput1.getId())
      .put(additionalOutput2.getId(), additionalOutput2.getId())
      .build();

  final DoFnTransform<String, String> doFnTransform =
    new DoFnTransform<>(
      new MultiOutputDoFn(additionalOutput1, additionalOutput2),
      NULL_INPUT_CODER,
      NULL_OUTPUT_CODERS,
      mainOutput,
      tags,
      WindowingStrategy.globalDefault(),
      PipelineOptionsFactory.as(NemoPipelineOptions.class),
      DisplayData.none(),
      DoFnSchemaInformation.create(),
      Collections.emptyMap());

  // mock context
  final Transform.Context context = mock(Transform.Context.class);

  final OutputCollector<WindowedValue<String>> oc = new TestOutputCollector<>();
  doFnTransform.prepare(context, oc);

  doFnTransform.onData(WindowedValue.valueInGlobalWindow("one"));
  doFnTransform.onData(WindowedValue.valueInGlobalWindow("two"));
  doFnTransform.onData(WindowedValue.valueInGlobalWindow("hello"));

  // main output
  assertEquals(WindowedValue.valueInGlobalWindow("got: hello"),
    ((TestOutputCollector<String>) oc).outputs.get(0));

  // additional output 1
  assertTrue(((TestOutputCollector<String>) oc).getTaggedOutputs().contains(
    new Tuple<>(additionalOutput1.getId(), WindowedValue.valueInGlobalWindow("extra: one"))
  ));
  assertTrue(((TestOutputCollector<String>) oc).getTaggedOutputs().contains(
    new Tuple<>(additionalOutput1.getId(), WindowedValue.valueInGlobalWindow("got: hello"))
  ));

  // additional output 2
  assertTrue(((TestOutputCollector<String>) oc).getTaggedOutputs().contains(
    new Tuple<>(additionalOutput2.getId(), WindowedValue.valueInGlobalWindow("extra: two"))
  ));
  assertTrue(((TestOutputCollector<String>) oc).getTaggedOutputs().contains(
    new Tuple<>(additionalOutput2.getId(), WindowedValue.valueInGlobalWindow("got: hello"))
  ));

  doFnTransform.close();
}

Example 5

Source File: DoFnTransformTest.java From incubator-nemo with Apache License 2.0

4 votes

@Test
public void testSideInputs() {
  // mock context
  final Transform.Context context = mock(Transform.Context.class);
  TupleTag<Tuple<String, Iterable<String>>> outputTag = new TupleTag<>("main-output");

  WindowedValue<String> firstElement = WindowedValue.valueInGlobalWindow("first");
  WindowedValue<String> secondElement = WindowedValue.valueInGlobalWindow("second");

  SideInputElement firstSideinput = new SideInputElement<>(0, ImmutableList.of("1"));
  SideInputElement secondSideinput = new SideInputElement(1, ImmutableList.of("2"));

  final Map<Integer, PCollectionView<?>> sideInputMap = new HashMap<>();
  sideInputMap.put(firstSideinput.getSideInputIndex(), view1);
  sideInputMap.put(secondSideinput.getSideInputIndex(), view2);
  final PushBackDoFnTransform<String, String> doFnTransform =
    new PushBackDoFnTransform(
      new SimpleSideInputDoFn<String>(view1, view2),
      NULL_INPUT_CODER,
      NULL_OUTPUT_CODERS,
      outputTag,
      Collections.emptyList(),
      WindowingStrategy.globalDefault(),
      sideInputMap, /* side inputs */
      PipelineOptionsFactory.as(NemoPipelineOptions.class),
      DisplayData.none(),
      DoFnSchemaInformation.create(),
      Collections.emptyMap());

  final TestOutputCollector<String> oc = new TestOutputCollector<>();
  doFnTransform.prepare(context, oc);

  // Main input first, Side inputs later
  doFnTransform.onData(firstElement);

  doFnTransform.onData(WindowedValue.valueInGlobalWindow(firstSideinput));
  doFnTransform.onData(WindowedValue.valueInGlobalWindow(secondSideinput));
  assertEquals(
    WindowedValue.valueInGlobalWindow(
      concat(firstElement.getValue(), firstSideinput.getSideInputValue(), secondSideinput.getSideInputValue())),
    oc.getOutput().get(0));

  // Side inputs first, Main input later
  doFnTransform.onData(secondElement);
  assertEquals(
    WindowedValue.valueInGlobalWindow(
      concat(secondElement.getValue(), firstSideinput.getSideInputValue(), secondSideinput.getSideInputValue())),
    oc.getOutput().get(1));

  // There should be only 2 final outputs
  assertEquals(2, oc.getOutput().size());

  // The side inputs should be "READY"
  assertTrue(doFnTransform.getSideInputReader().isReady(view1, GlobalWindow.INSTANCE));
  assertTrue(doFnTransform.getSideInputReader().isReady(view2, GlobalWindow.INSTANCE));

  // This watermark should remove the side inputs. (Now should be "NOT READY")
  doFnTransform.onWatermark(new Watermark(GlobalWindow.TIMESTAMP_MAX_VALUE.getMillis()));
  Iterable materializedSideInput1 = doFnTransform.getSideInputReader().get(view1, GlobalWindow.INSTANCE);
  Iterable materializedSideInput2 = doFnTransform.getSideInputReader().get(view2, GlobalWindow.INSTANCE);
  assertFalse(materializedSideInput1.iterator().hasNext());
  assertFalse(materializedSideInput2.iterator().hasNext());

  // There should be only 2 final outputs
  doFnTransform.close();
  assertEquals(2, oc.getOutput().size());
}