Java Code Examples for org.apache.beam.sdk.transforms.windowing.Window#into()

The following examples show how to use org.apache.beam.sdk.transforms.windowing.Window#into() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: WindowedWordCount.java    From incubator-nemo with Apache License 2.0 5 votes vote down vote up 
/**
 * Main function for the MR BEAM program.
 *
 * @param args arguments.
 */
public static void main(final String[] args) {
  final String outputFilePath = args[0];
  final String windowType = args[1];

  final Window<KV<String, Long>> windowFn;
  if (windowType.equals("fixed")) {
    windowFn = Window.<KV<String, Long>>into(FixedWindows.of(Duration.standardSeconds(5)));
  } else {
    windowFn = Window.<KV<String, Long>>into(SlidingWindows.of(Duration.standardSeconds(10))
      .every(Duration.standardSeconds(5)));
  }

  final PipelineOptions options = NemoPipelineOptionsFactory.create();
  options.setJobName("WindowedWordCount");

  final Pipeline p = Pipeline.create(options);

  getSource(p, args)
    .apply(windowFn)
    .apply(Sum.longsPerKey())
    .apply(MapElements.<KV<String, Long>, String>via(new SimpleFunction<KV<String, Long>, String>() {
      @Override
      public String apply(final KV<String, Long> kv) {
        return kv.getKey() + ": " + kv.getValue();
      }
    }))
    .apply(new WriteOneFilePerWindow(outputFilePath, 1));

  p.run().waitUntilFinish();
}
Example 2
Source File: WindowedBroadcast.java    From incubator-nemo with Apache License 2.0 5 votes vote down vote up 
/**
 * Main function for the MR BEAM program.
 *
 * @param args arguments.
 */
public static void main(final String[] args) {
  final String outputFilePath = args[0];

  final Window<Long> windowFn = Window
    .<Long>into(SlidingWindows.of(Duration.standardSeconds(2))
      .every(Duration.standardSeconds(1)));

  final PipelineOptions options = NemoPipelineOptionsFactory.create();
  options.setJobName("WindowedBroadcast");

  final Pipeline p = Pipeline.create(options);

  final PCollection<Long> windowedElements = getSource(p).apply(windowFn);
  final PCollectionView<List<Long>> windowedView = windowedElements.apply(View.asList());

  windowedElements.apply(ParDo.of(new DoFn<Long, String>() {
      @ProcessElement
      public void processElement(final ProcessContext c) {
        final Long anElementInTheWindow = c.element();
        final List<Long> allElementsInTheWindow = c.sideInput(windowedView);
        System.out.println(anElementInTheWindow + " / " + allElementsInTheWindow);
        if (!allElementsInTheWindow.contains(anElementInTheWindow)) {
          throw new RuntimeException(anElementInTheWindow + " not in " + allElementsInTheWindow.toString());
        } else {
          c.output(anElementInTheWindow + " is in " + allElementsInTheWindow);
        }
      }
    }).withSideInputs(windowedView)
  ).apply(new WriteOneFilePerWindow(outputFilePath, 1));

  p.run().waitUntilFinish();
}
Example 3
Source File: TopTest.java    From beam with Apache License 2.0 5 votes vote down vote up 
@Test
public void testTopEmptyWithIncompatibleWindows() {
  p.enableAbandonedNodeEnforcement(false);

  Window<String> windowingFn = Window.into(FixedWindows.of(Duration.standardDays(10L)));
  PCollection<String> input = p.apply(Create.empty(StringUtf8Coder.of())).apply(windowingFn);

  expectedEx.expect(IllegalStateException.class);
  expectedEx.expectMessage("Top");
  expectedEx.expectMessage("GlobalWindows");
  expectedEx.expectMessage("withoutDefaults");
  expectedEx.expectMessage("asSingletonView");

  input.apply(Top.of(1, new OrderByLength()));
}
Example 4
Source File: WindowEvaluatorFactoryTest.java    From beam with Apache License 2.0 5 votes vote down vote up 
@Test
public void singleWindowFnSucceeds() throws Exception {
  Duration windowDuration = Duration.standardDays(7);
  Window<Long> transform = Window.into(FixedWindows.of(windowDuration));
  PCollection<Long> windowed = input.apply(transform);

  CommittedBundle<Long> inputBundle = createInputBundle();

  UncommittedBundle<Long> outputBundle = createOutputBundle(windowed, inputBundle);

  BoundedWindow firstSecondWindow = new IntervalWindow(EPOCH, EPOCH.plus(windowDuration));
  BoundedWindow thirdWindow = new IntervalWindow(EPOCH.minus(windowDuration), EPOCH);

  TransformResult<Long> result = runEvaluator(windowed, inputBundle);

  assertThat(Iterables.getOnlyElement(result.getOutputBundles()), Matchers.equalTo(outputBundle));
  CommittedBundle<Long> committed = outputBundle.commit(Instant.now());

  assertThat(
      committed.getElements(),
      containsInAnyOrder(
          // value in global window
          isSingleWindowedValue(3L, new Instant(2L), firstSecondWindow, NO_FIRING),

          // value in just interval window
          isSingleWindowedValue(2L, new Instant(-10L), thirdWindow, intervalWindowPane),

          // value in global window and two interval windows
          isSingleWindowedValue(
              1L, EPOCH.plus(Duration.standardDays(3)), firstSecondWindow, multiWindowPane),
          isSingleWindowedValue(
              1L, EPOCH.plus(Duration.standardDays(3)), firstSecondWindow, multiWindowPane),
          isSingleWindowedValue(
              1L, EPOCH.plus(Duration.standardDays(3)), firstSecondWindow, multiWindowPane)));
}
Example 5
Source File: WindowBuilder.java    From beam with Apache License 2.0 4 votes vote down vote up 
@Override
public <W extends BoundedWindow> WindowBuilder<T> windowBy(WindowFn<Object, W> windowFn) {
  checkState(window == null, "Window is already set.");
  window = Window.into(windowFn);
  return this;
}
Example 6
Source File: PAssert.java    From beam with Apache License 2.0 4 votes vote down vote up 
private <T> PTransform<PCollection<T>, PCollection<T>> window() {
  return Window.into(new GlobalWindows());
}
Example 7
Source File: PAssert.java    From beam with Apache License 2.0 4 votes vote down vote up 
@Override
public <T> PTransform<PCollection<T>, PCollection<T>> windowDummy() {
  return Window.into(windowFn);
}
Example 8
Source File: PAssert.java    From beam with Apache License 2.0 4 votes vote down vote up 
@Override
public <T> PTransform<PCollection<T>, PCollection<T>> windowActuals() {
  return Window.into(windowFn.intoOnlyExisting());
}
Example 9
Source File: WindowEvaluatorFactoryTest.java    From beam with Apache License 2.0 4 votes vote down vote up 
@Test
public void multipleWindowsWindowFnSucceeds() throws Exception {
  Duration windowDuration = Duration.standardDays(6);
  Duration slidingBy = Duration.standardDays(3);
  Window<Long> transform = Window.into(SlidingWindows.of(windowDuration).every(slidingBy));
  PCollection<Long> windowed = input.apply(transform);

  CommittedBundle<Long> inputBundle = createInputBundle();
  UncommittedBundle<Long> outputBundle = createOutputBundle(windowed, inputBundle);

  TransformResult<Long> result = runEvaluator(windowed, inputBundle);

  assertThat(Iterables.getOnlyElement(result.getOutputBundles()), Matchers.equalTo(outputBundle));
  CommittedBundle<Long> committed = outputBundle.commit(Instant.now());

  BoundedWindow w1 = new IntervalWindow(EPOCH, EPOCH.plus(windowDuration));
  BoundedWindow w2 =
      new IntervalWindow(EPOCH.plus(slidingBy), EPOCH.plus(slidingBy).plus(windowDuration));
  BoundedWindow wMinus1 = new IntervalWindow(EPOCH.minus(windowDuration), EPOCH);
  BoundedWindow wMinusSlide =
      new IntervalWindow(EPOCH.minus(windowDuration).plus(slidingBy), EPOCH.plus(slidingBy));

  assertThat(
      committed.getElements(),
      containsInAnyOrder(
          // Value in global window mapped to one windowed value in multiple windows
          isWindowedValue(
              valueInGlobalWindow.getValue(),
              valueInGlobalWindow.getTimestamp(),
              ImmutableSet.of(w1, wMinusSlide),
              NO_FIRING),

          // Value in interval window mapped to one windowed value in multiple windows
          isWindowedValue(
              valueInIntervalWindow.getValue(),
              valueInIntervalWindow.getTimestamp(),
              ImmutableSet.of(wMinus1, wMinusSlide),
              valueInIntervalWindow.getPane()),

          // Value in three windows mapped to three windowed values in the same multiple windows
          isWindowedValue(
              valueInGlobalAndTwoIntervalWindows.getValue(),
              valueInGlobalAndTwoIntervalWindows.getTimestamp(),
              ImmutableSet.of(w1, w2),
              valueInGlobalAndTwoIntervalWindows.getPane()),
          isWindowedValue(
              valueInGlobalAndTwoIntervalWindows.getValue(),
              valueInGlobalAndTwoIntervalWindows.getTimestamp(),
              ImmutableSet.of(w1, w2),
              valueInGlobalAndTwoIntervalWindows.getPane()),
          isWindowedValue(
              valueInGlobalAndTwoIntervalWindows.getValue(),
              valueInGlobalAndTwoIntervalWindows.getTimestamp(),
              ImmutableSet.of(w1, w2),
              valueInGlobalAndTwoIntervalWindows.getPane())));
}
Example 10
Source File: WindowEvaluatorFactoryTest.java    From beam with Apache License 2.0 4 votes vote down vote up 
@Test
public void referencesEarlierWindowsSucceeds() throws Exception {
  Window<Long> transform = Window.into(new EvaluatorTestWindowFn());
  PCollection<Long> windowed = input.apply(transform);

  CommittedBundle<Long> inputBundle = createInputBundle();
  UncommittedBundle<Long> outputBundle = createOutputBundle(windowed, inputBundle);

  TransformResult<Long> result = runEvaluator(windowed, inputBundle);

  assertThat(Iterables.getOnlyElement(result.getOutputBundles()), Matchers.equalTo(outputBundle));
  CommittedBundle<Long> committed = outputBundle.commit(Instant.now());

  assertThat(
      committed.getElements(),
      containsInAnyOrder(
          // Value in global window mapped to [timestamp, timestamp+1)
          isSingleWindowedValue(
              valueInGlobalWindow.getValue(),
              valueInGlobalWindow.getTimestamp(),
              new IntervalWindow(
                  valueInGlobalWindow.getTimestamp(),
                  valueInGlobalWindow.getTimestamp().plus(1L)),
              valueInGlobalWindow.getPane()),

          // Value in interval window mapped to the same window
          isWindowedValue(
              valueInIntervalWindow.getValue(),
              valueInIntervalWindow.getTimestamp(),
              valueInIntervalWindow.getWindows(),
              valueInIntervalWindow.getPane()),

          // Value in global window and two interval windows exploded and mapped in both ways
          isSingleWindowedValue(
              valueInGlobalAndTwoIntervalWindows.getValue(),
              valueInGlobalAndTwoIntervalWindows.getTimestamp(),
              new IntervalWindow(
                  valueInGlobalAndTwoIntervalWindows.getTimestamp(),
                  valueInGlobalAndTwoIntervalWindows.getTimestamp().plus(1L)),
              valueInGlobalAndTwoIntervalWindows.getPane()),
          isSingleWindowedValue(
              valueInGlobalAndTwoIntervalWindows.getValue(),
              valueInGlobalAndTwoIntervalWindows.getTimestamp(),
              intervalWindow1,
              valueInGlobalAndTwoIntervalWindows.getPane()),
          isSingleWindowedValue(
              valueInGlobalAndTwoIntervalWindows.getValue(),
              valueInGlobalAndTwoIntervalWindows.getTimestamp(),
              intervalWindow2,
              valueInGlobalAndTwoIntervalWindows.getPane())));
}