Java Code Examples for org.apache.beam.sdk.transforms.windowing.Window#into()
The following examples show how to use
org.apache.beam.sdk.transforms.windowing.Window#into() .
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: WindowedWordCount.java From incubator-nemo with Apache License 2.0 | 5 votes |
/** * Main function for the MR BEAM program. * * @param args arguments. */ public static void main(final String[] args) { final String outputFilePath = args[0]; final String windowType = args[1]; final Window<KV<String, Long>> windowFn; if (windowType.equals("fixed")) { windowFn = Window.<KV<String, Long>>into(FixedWindows.of(Duration.standardSeconds(5))); } else { windowFn = Window.<KV<String, Long>>into(SlidingWindows.of(Duration.standardSeconds(10)) .every(Duration.standardSeconds(5))); } final PipelineOptions options = NemoPipelineOptionsFactory.create(); options.setJobName("WindowedWordCount"); final Pipeline p = Pipeline.create(options); getSource(p, args) .apply(windowFn) .apply(Sum.longsPerKey()) .apply(MapElements.<KV<String, Long>, String>via(new SimpleFunction<KV<String, Long>, String>() { @Override public String apply(final KV<String, Long> kv) { return kv.getKey() + ": " + kv.getValue(); } })) .apply(new WriteOneFilePerWindow(outputFilePath, 1)); p.run().waitUntilFinish(); }
Example 2
Source File: WindowedBroadcast.java From incubator-nemo with Apache License 2.0 | 5 votes |
/** * Main function for the MR BEAM program. * * @param args arguments. */ public static void main(final String[] args) { final String outputFilePath = args[0]; final Window<Long> windowFn = Window .<Long>into(SlidingWindows.of(Duration.standardSeconds(2)) .every(Duration.standardSeconds(1))); final PipelineOptions options = NemoPipelineOptionsFactory.create(); options.setJobName("WindowedBroadcast"); final Pipeline p = Pipeline.create(options); final PCollection<Long> windowedElements = getSource(p).apply(windowFn); final PCollectionView<List<Long>> windowedView = windowedElements.apply(View.asList()); windowedElements.apply(ParDo.of(new DoFn<Long, String>() { @ProcessElement public void processElement(final ProcessContext c) { final Long anElementInTheWindow = c.element(); final List<Long> allElementsInTheWindow = c.sideInput(windowedView); System.out.println(anElementInTheWindow + " / " + allElementsInTheWindow); if (!allElementsInTheWindow.contains(anElementInTheWindow)) { throw new RuntimeException(anElementInTheWindow + " not in " + allElementsInTheWindow.toString()); } else { c.output(anElementInTheWindow + " is in " + allElementsInTheWindow); } } }).withSideInputs(windowedView) ).apply(new WriteOneFilePerWindow(outputFilePath, 1)); p.run().waitUntilFinish(); }
Example 3
Source File: TopTest.java From beam with Apache License 2.0 | 5 votes |
@Test public void testTopEmptyWithIncompatibleWindows() { p.enableAbandonedNodeEnforcement(false); Window<String> windowingFn = Window.into(FixedWindows.of(Duration.standardDays(10L))); PCollection<String> input = p.apply(Create.empty(StringUtf8Coder.of())).apply(windowingFn); expectedEx.expect(IllegalStateException.class); expectedEx.expectMessage("Top"); expectedEx.expectMessage("GlobalWindows"); expectedEx.expectMessage("withoutDefaults"); expectedEx.expectMessage("asSingletonView"); input.apply(Top.of(1, new OrderByLength())); }
Example 4
Source File: WindowEvaluatorFactoryTest.java From beam with Apache License 2.0 | 5 votes |
@Test public void singleWindowFnSucceeds() throws Exception { Duration windowDuration = Duration.standardDays(7); Window<Long> transform = Window.into(FixedWindows.of(windowDuration)); PCollection<Long> windowed = input.apply(transform); CommittedBundle<Long> inputBundle = createInputBundle(); UncommittedBundle<Long> outputBundle = createOutputBundle(windowed, inputBundle); BoundedWindow firstSecondWindow = new IntervalWindow(EPOCH, EPOCH.plus(windowDuration)); BoundedWindow thirdWindow = new IntervalWindow(EPOCH.minus(windowDuration), EPOCH); TransformResult<Long> result = runEvaluator(windowed, inputBundle); assertThat(Iterables.getOnlyElement(result.getOutputBundles()), Matchers.equalTo(outputBundle)); CommittedBundle<Long> committed = outputBundle.commit(Instant.now()); assertThat( committed.getElements(), containsInAnyOrder( // value in global window isSingleWindowedValue(3L, new Instant(2L), firstSecondWindow, NO_FIRING), // value in just interval window isSingleWindowedValue(2L, new Instant(-10L), thirdWindow, intervalWindowPane), // value in global window and two interval windows isSingleWindowedValue( 1L, EPOCH.plus(Duration.standardDays(3)), firstSecondWindow, multiWindowPane), isSingleWindowedValue( 1L, EPOCH.plus(Duration.standardDays(3)), firstSecondWindow, multiWindowPane), isSingleWindowedValue( 1L, EPOCH.plus(Duration.standardDays(3)), firstSecondWindow, multiWindowPane))); }
Example 5
Source File: WindowBuilder.java From beam with Apache License 2.0 | 4 votes |
@Override public <W extends BoundedWindow> WindowBuilder<T> windowBy(WindowFn<Object, W> windowFn) { checkState(window == null, "Window is already set."); window = Window.into(windowFn); return this; }
Example 6
Source File: PAssert.java From beam with Apache License 2.0 | 4 votes |
private <T> PTransform<PCollection<T>, PCollection<T>> window() { return Window.into(new GlobalWindows()); }
Example 7
Source File: PAssert.java From beam with Apache License 2.0 | 4 votes |
@Override public <T> PTransform<PCollection<T>, PCollection<T>> windowDummy() { return Window.into(windowFn); }
Example 8
Source File: PAssert.java From beam with Apache License 2.0 | 4 votes |
@Override public <T> PTransform<PCollection<T>, PCollection<T>> windowActuals() { return Window.into(windowFn.intoOnlyExisting()); }
Example 9
Source File: WindowEvaluatorFactoryTest.java From beam with Apache License 2.0 | 4 votes |
@Test public void multipleWindowsWindowFnSucceeds() throws Exception { Duration windowDuration = Duration.standardDays(6); Duration slidingBy = Duration.standardDays(3); Window<Long> transform = Window.into(SlidingWindows.of(windowDuration).every(slidingBy)); PCollection<Long> windowed = input.apply(transform); CommittedBundle<Long> inputBundle = createInputBundle(); UncommittedBundle<Long> outputBundle = createOutputBundle(windowed, inputBundle); TransformResult<Long> result = runEvaluator(windowed, inputBundle); assertThat(Iterables.getOnlyElement(result.getOutputBundles()), Matchers.equalTo(outputBundle)); CommittedBundle<Long> committed = outputBundle.commit(Instant.now()); BoundedWindow w1 = new IntervalWindow(EPOCH, EPOCH.plus(windowDuration)); BoundedWindow w2 = new IntervalWindow(EPOCH.plus(slidingBy), EPOCH.plus(slidingBy).plus(windowDuration)); BoundedWindow wMinus1 = new IntervalWindow(EPOCH.minus(windowDuration), EPOCH); BoundedWindow wMinusSlide = new IntervalWindow(EPOCH.minus(windowDuration).plus(slidingBy), EPOCH.plus(slidingBy)); assertThat( committed.getElements(), containsInAnyOrder( // Value in global window mapped to one windowed value in multiple windows isWindowedValue( valueInGlobalWindow.getValue(), valueInGlobalWindow.getTimestamp(), ImmutableSet.of(w1, wMinusSlide), NO_FIRING), // Value in interval window mapped to one windowed value in multiple windows isWindowedValue( valueInIntervalWindow.getValue(), valueInIntervalWindow.getTimestamp(), ImmutableSet.of(wMinus1, wMinusSlide), valueInIntervalWindow.getPane()), // Value in three windows mapped to three windowed values in the same multiple windows isWindowedValue( valueInGlobalAndTwoIntervalWindows.getValue(), valueInGlobalAndTwoIntervalWindows.getTimestamp(), ImmutableSet.of(w1, w2), valueInGlobalAndTwoIntervalWindows.getPane()), isWindowedValue( valueInGlobalAndTwoIntervalWindows.getValue(), valueInGlobalAndTwoIntervalWindows.getTimestamp(), ImmutableSet.of(w1, w2), valueInGlobalAndTwoIntervalWindows.getPane()), isWindowedValue( valueInGlobalAndTwoIntervalWindows.getValue(), valueInGlobalAndTwoIntervalWindows.getTimestamp(), ImmutableSet.of(w1, w2), valueInGlobalAndTwoIntervalWindows.getPane()))); }
Example 10
Source File: WindowEvaluatorFactoryTest.java From beam with Apache License 2.0 | 4 votes |
@Test public void referencesEarlierWindowsSucceeds() throws Exception { Window<Long> transform = Window.into(new EvaluatorTestWindowFn()); PCollection<Long> windowed = input.apply(transform); CommittedBundle<Long> inputBundle = createInputBundle(); UncommittedBundle<Long> outputBundle = createOutputBundle(windowed, inputBundle); TransformResult<Long> result = runEvaluator(windowed, inputBundle); assertThat(Iterables.getOnlyElement(result.getOutputBundles()), Matchers.equalTo(outputBundle)); CommittedBundle<Long> committed = outputBundle.commit(Instant.now()); assertThat( committed.getElements(), containsInAnyOrder( // Value in global window mapped to [timestamp, timestamp+1) isSingleWindowedValue( valueInGlobalWindow.getValue(), valueInGlobalWindow.getTimestamp(), new IntervalWindow( valueInGlobalWindow.getTimestamp(), valueInGlobalWindow.getTimestamp().plus(1L)), valueInGlobalWindow.getPane()), // Value in interval window mapped to the same window isWindowedValue( valueInIntervalWindow.getValue(), valueInIntervalWindow.getTimestamp(), valueInIntervalWindow.getWindows(), valueInIntervalWindow.getPane()), // Value in global window and two interval windows exploded and mapped in both ways isSingleWindowedValue( valueInGlobalAndTwoIntervalWindows.getValue(), valueInGlobalAndTwoIntervalWindows.getTimestamp(), new IntervalWindow( valueInGlobalAndTwoIntervalWindows.getTimestamp(), valueInGlobalAndTwoIntervalWindows.getTimestamp().plus(1L)), valueInGlobalAndTwoIntervalWindows.getPane()), isSingleWindowedValue( valueInGlobalAndTwoIntervalWindows.getValue(), valueInGlobalAndTwoIntervalWindows.getTimestamp(), intervalWindow1, valueInGlobalAndTwoIntervalWindows.getPane()), isSingleWindowedValue( valueInGlobalAndTwoIntervalWindows.getValue(), valueInGlobalAndTwoIntervalWindows.getTimestamp(), intervalWindow2, valueInGlobalAndTwoIntervalWindows.getPane()))); }