org.apache.beam.sdk.transforms.windowing.FixedWindows Java Examples
The following examples show how to use
org.apache.beam.sdk.transforms.windowing.FixedWindows.
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: LateDataUtilsTest.java From beam with Apache License 2.0 | 6 votes |
@Test public void garbageCollectionTimeAfterEndOfGlobalWindowWithLateness() { FixedWindows windowFn = FixedWindows.of(Duration.standardMinutes(5)); Duration allowedLateness = Duration.millis(Long.MAX_VALUE); WindowingStrategy<?, ?> strategy = WindowingStrategy.globalDefault() .withWindowFn(windowFn) .withAllowedLateness(allowedLateness); IntervalWindow window = windowFn.assignWindow(new Instant(-100)); assertThat( window.maxTimestamp().plus(allowedLateness), Matchers.greaterThan(GlobalWindow.INSTANCE.maxTimestamp())); assertThat( LateDataUtils.garbageCollectionTime(window, strategy), equalTo(GlobalWindow.INSTANCE.maxTimestamp())); }
Example #2
Source File: GroupByKeyTest.java From beam with Apache License 2.0 | 6 votes |
@Test @Category(NeedsRunner.class) public void testIdentityWindowFnPropagation() { List<KV<String, Integer>> ungroupedPairs = Arrays.asList(); PCollection<KV<String, Integer>> input = p.apply( Create.of(ungroupedPairs) .withCoder(KvCoder.of(StringUtf8Coder.of(), BigEndianIntegerCoder.of()))) .apply(Window.into(FixedWindows.of(Duration.standardMinutes(1)))); PCollection<KV<String, Iterable<Integer>>> output = input.apply(GroupByKey.create()); p.run(); Assert.assertTrue( output .getWindowingStrategy() .getWindowFn() .isCompatible(FixedWindows.of(Duration.standardMinutes(1)))); }
Example #3
Source File: AfterWatermarkStateMachineTest.java From beam with Apache License 2.0 | 6 votes |
@Test public void testEarlyAndAtWatermark() throws Exception { tester = TriggerStateMachineTester.forTrigger( AfterWatermarkStateMachine.pastEndOfWindow().withEarlyFirings(mockEarly), FixedWindows.of(Duration.millis(100))); injectElements(1); IntervalWindow window = new IntervalWindow(new Instant(0), new Instant(100)); testRunningAsTrigger(mockEarly, window); // Fire due to watermark when(mockEarly.shouldFire(anyTriggerContext())).thenReturn(false); tester.advanceInputWatermark(new Instant(100)); assertTrue(tester.shouldFire(window)); tester.fireIfShouldFire(window); assertTrue(tester.isMarkedFinished(window)); }
Example #4
Source File: BeamSqlDslJoinTest.java From beam with Apache License 2.0 | 6 votes |
@Test public void testJoinsUnboundedWithinWindowsWithDefaultTrigger() throws Exception { String sql = "SELECT o1.order_id, o1.price, o1.site_id, o2.order_id, o2.price, o2.site_id " + "FROM ORDER_DETAILS1 o1" + " JOIN ORDER_DETAILS2 o2" + " on " + " o1.order_id=o2.site_id AND o2.price=o1.site_id"; PCollection<Row> orders = ordersUnbounded() .apply("window", Window.into(FixedWindows.of(Duration.standardSeconds(50)))); PCollectionTuple inputs = tuple("ORDER_DETAILS1", orders, "ORDER_DETAILS2", orders); PAssert.that(inputs.apply("sql", SqlTransform.query(sql))) .containsInAnyOrder( TestUtils.RowsBuilder.of(RESULT_ROW_TYPE) .addRows(1, 2, 2, 2, 2, 1, 1, 4, 3, 3, 3, 1) .getRows()); pipeline.run(); }
Example #5
Source File: FlattenTest.java From beam with Apache License 2.0 | 6 votes |
@Test public void testIncompatibleWindowFnPropagationFailure() { p.enableAbandonedNodeEnforcement(false); PCollection<String> input1 = p.apply("CreateInput1", Create.of("Input1")) .apply("Window1", Window.into(FixedWindows.of(Duration.standardMinutes(1)))); PCollection<String> input2 = p.apply("CreateInput2", Create.of("Input2")) .apply("Window2", Window.into(FixedWindows.of(Duration.standardMinutes(2)))); try { PCollectionList.of(input1).and(input2).apply(Flatten.pCollections()); Assert.fail("Exception should have been thrown"); } catch (IllegalStateException e) { Assert.assertTrue( e.getMessage().startsWith("Inputs to Flatten had incompatible window windowFns")); } }
Example #6
Source File: WriteFilesTest.java From beam with Apache License 2.0 | 6 votes |
/** Test a WriteFiles with a windowed PCollection. */ @Test @Category(NeedsRunner.class) public void testWriteWindowed() throws IOException { List<String> inputs = Arrays.asList( "Critical canary", "Apprehensive eagle", "Intimidating pigeon", "Pedantic gull", "Frisky finch"); runWrite( inputs, new WindowAndReshuffle<>(Window.into(FixedWindows.of(Duration.millis(2)))), getBaseOutputFilename(), WriteFiles.to(makeSimpleSink())); }
Example #7
Source File: AfterFirstStateMachineTest.java From beam with Apache License 2.0 | 6 votes |
@Test public void testOnlyT1ShouldFireFixedWindows() throws Exception { tester = TriggerStateMachineTester.forTrigger( AfterFirstStateMachine.of(mockTrigger1, mockTrigger2), FixedWindows.of(Duration.millis(10))); tester.injectElements(1); IntervalWindow window = new IntervalWindow(new Instant(1), new Instant(11)); when(mockTrigger1.shouldFire(anyTriggerContext())).thenReturn(true); when(mockTrigger2.shouldFire(anyTriggerContext())).thenReturn(false); assertTrue(tester.shouldFire(window)); // should fire tester.fireIfShouldFire(window); assertTrue(tester.isMarkedFinished(window)); }
Example #8
Source File: StreamWordCount.java From beam-starter with Apache License 2.0 | 6 votes |
public static void main(String[] args) throws Exception { Options options = PipelineOptionsFactory.fromArgs(args).withValidation() .as(Options.class); options.setRunner(FlinkRunner.class); Pipeline p = Pipeline.create(options); KafkaIO.Read<byte[], String> kafkaIOReader = KafkaIO.read() .withBootstrapServers("192.168.99.100:32771") .withTopics(Arrays.asList("beam".split(","))) .updateConsumerProperties(ImmutableMap.of("auto.offset.reset", (Object)"earliest")) .withValueCoder(StringUtf8Coder.of()); p.apply(kafkaIOReader.withoutMetadata()) .apply(Values.<String>create()) .apply(Window.<String>into( FixedWindows.of(Duration.standardMinutes(options.getWindowSize())))) .apply(new CountWords()) .apply(MapElements.via(new FormatAsTextFn())) .apply("WriteCounts", TextIO.Write.to(options.getOutput())); p.run(); }
Example #9
Source File: LeaderBoard.java From beam with Apache License 2.0 | 6 votes |
@Override public PCollection<KV<String, Integer>> expand(PCollection<GameActionInfo> infos) { return infos .apply( "LeaderboardTeamFixedWindows", Window.<GameActionInfo>into(FixedWindows.of(teamWindowDuration)) // We will get early (speculative) results as well as cumulative // processing of late data. .triggering( AfterWatermark.pastEndOfWindow() .withEarlyFirings( AfterProcessingTime.pastFirstElementInPane() .plusDelayOf(FIVE_MINUTES)) .withLateFirings( AfterProcessingTime.pastFirstElementInPane() .plusDelayOf(TEN_MINUTES))) .withAllowedLateness(allowedLateness) .accumulatingFiredPanes()) // Extract and sum teamname/score pairs from the event data. .apply("ExtractTeamScore", new ExtractAndSumScore("team")); }
Example #10
Source File: PortablePipelineDotRendererTest.java From beam with Apache License 2.0 | 6 votes |
@Test public void testCompositePipeline() { p.apply(Create.timestamped(TimestampedValue.of(KV.of(1, 1), new Instant(1)))) .apply(Window.into(FixedWindows.of(Duration.millis(10)))) .apply(Sum.integersPerKey()); assertEquals( "digraph {" + " rankdir=LR" + " 0 [label=\"Create.TimestampedValues\\n\"]" + " 1 [label=\"Window.Into()\\n\"]" + " 0 -> 1 [style=solid label=\"Create.TimestampedValues/ParDo(ConvertTimestamps)/ParMultiDo(ConvertTimestamps).output\"]" + " 2 [label=\"Combine.perKey(SumInteger)\\nbeam:transform:combine_per_key:v1\"]" + " 1 -> 2 [style=solid label=\"Window.Into()/Window.Assign.out\"]" + "}", PortablePipelineDotRenderer.toDotString(PipelineTranslation.toProto(p)) .replaceAll(System.lineSeparator(), "")); }
Example #11
Source File: WriteFilesTest.java From beam with Apache License 2.0 | 6 votes |
@Test @Category(NeedsRunner.class) public void testWriteNoSpilling() throws IOException { List<String> inputs = Lists.newArrayList(); for (int i = 0; i < 100; ++i) { inputs.add("mambo_number_" + i); } runWrite( inputs, Window.into(FixedWindows.of(Duration.millis(1))), getBaseOutputFilename(), WriteFiles.to(makeSimpleSink()) .withMaxNumWritersPerBundle(2) .withWindowedWrites() .withNoSpilling()); }
Example #12
Source File: ReduceFnRunnerTest.java From beam with Apache License 2.0 | 6 votes |
/** * When the watermark passes the end-of-window and window expiration time in a single update, this * tests that it does not crash. */ @Test public void testFixedWindowsEowAndGcTogether() throws Exception { ReduceFnTester<Integer, Iterable<Integer>, IntervalWindow> tester = ReduceFnTester.nonCombining( FixedWindows.of(Duration.millis(10)), DefaultTriggerStateMachine.of(), AccumulationMode.ACCUMULATING_FIRED_PANES, Duration.millis(50), ClosingBehavior.FIRE_ALWAYS); tester.setAutoAdvanceOutputWatermark(true); tester.advanceInputWatermark(new Instant(0)); injectElement(tester, 1); tester.advanceInputWatermark(new Instant(100)); assertThat( tester.extractOutput(), contains( isSingleWindowedValue( contains(1), 1, 0, 10, PaneInfo.createPane(true, true, Timing.ON_TIME)))); }
Example #13
Source File: SumByKeyTest.java From beam with Apache License 2.0 | 6 votes |
@Test public void testBuild_Windowing() { final PCollection<String> dataset = TestUtils.createMockDataset(TypeDescriptors.strings()); final PCollection<KV<String, Long>> counted = SumByKey.of(dataset) .keyBy(s -> s) .valueBy(s -> 1L) .windowBy(FixedWindows.of(org.joda.time.Duration.standardHours(1))) .triggeredBy(DefaultTrigger.of()) .discardingFiredPanes() .withAllowedLateness(Duration.millis(1000)) .output(); final SumByKey sum = (SumByKey) TestUtils.getProducer(counted); assertTrue(sum.getWindow().isPresent()); @SuppressWarnings("unchecked") final WindowDesc<?> windowDesc = WindowDesc.of((Window) sum.getWindow().get()); assertEquals( FixedWindows.of(org.joda.time.Duration.standardHours(1)), windowDesc.getWindowFn()); assertEquals(DefaultTrigger.of(), windowDesc.getTrigger()); assertEquals(AccumulationMode.DISCARDING_FIRED_PANES, windowDesc.getAccumulationMode()); assertEquals(Duration.millis(1000), windowDesc.getAllowedLateness()); }
Example #14
Source File: ReduceByKeyTest.java From beam with Apache License 2.0 | 6 votes |
@Test public void testBuild_sortedValues() { final PCollection<String> dataset = TestUtils.createMockDataset(TypeDescriptors.strings()); final PCollection<KV<String, List<Long>>> reduced = ReduceByKey.of(dataset) .keyBy(s -> s) .valueBy(s -> 1L) .reduceBy(s -> s.collect(Collectors.toList())) .withSortedValues(Long::compare) .windowBy(FixedWindows.of(Duration.standardHours(1))) .triggeredBy(DefaultTrigger.of()) .accumulationMode(AccumulationMode.DISCARDING_FIRED_PANES) .output(); final ReduceByKey reduce = (ReduceByKey) TestUtils.getProducer(reduced); assertTrue(reduce.getValueComparator().isPresent()); }
Example #15
Source File: ReduceByKeyTest.java From beam with Apache License 2.0 | 6 votes |
@Test public void testBuild_Windowing() { final PCollection<String> dataset = TestUtils.createMockDataset(TypeDescriptors.strings()); final PCollection<KV<String, Long>> reduced = ReduceByKey.of(dataset) .keyBy(s -> s) .valueBy(s -> 1L) .combineBy(Sums.ofLongs()) .windowBy(FixedWindows.of(Duration.standardHours(1))) .triggeredBy(DefaultTrigger.of()) .accumulationMode(AccumulationMode.DISCARDING_FIRED_PANES) .output(); final ReduceByKey reduce = (ReduceByKey) TestUtils.getProducer(reduced); assertTrue(reduce.getWindow().isPresent()); @SuppressWarnings("unchecked") final Window<? extends BoundedWindow> window = (Window) reduce.getWindow().get(); assertEquals(FixedWindows.of(org.joda.time.Duration.standardHours(1)), window.getWindowFn()); assertEquals(DefaultTrigger.of(), WindowDesc.of(window).getTrigger()); assertSame( AccumulationMode.DISCARDING_FIRED_PANES, WindowDesc.of(window).getAccumulationMode()); assertFalse(reduce.getValueComparator().isPresent()); }
Example #16
Source File: ReshuffleTest.java From beam with Apache License 2.0 | 6 votes |
@Test @Category(ValidatesRunner.class) public void testReshuffleAfterSlidingWindowsAndGroupByKey() { PCollection<KV<String, Iterable<Integer>>> input = pipeline .apply( Create.of(GBK_TESTABLE_KVS) .withCoder(KvCoder.of(StringUtf8Coder.of(), VarIntCoder.of()))) .apply(Window.into(FixedWindows.of(Duration.standardMinutes(10L)))) .apply(GroupByKey.create()); PCollection<KV<String, Iterable<Integer>>> output = input.apply(Reshuffle.of()); PAssert.that(output).satisfies(new AssertThatHasExpectedContents()); assertEquals(input.getWindowingStrategy(), output.getWindowingStrategy()); pipeline.run(); }
Example #17
Source File: DocumentationExamplesTest.java From beam with Apache License 2.0 | 6 votes |
@Test public void windowingSection() { PCollection<Integer> input = pipeline.apply(Create.of(1, 2, 3, 4)).setTypeDescriptor(TypeDescriptors.integers()); PCollection<KV<Integer, Long>> countedElements = CountByKey.of(input) .keyBy(e -> e) .windowBy(FixedWindows.of(Duration.standardSeconds(1))) .triggeredBy(DefaultTrigger.of()) .discardingFiredPanes() .withAllowedLateness(Duration.standardSeconds(5)) .withOnTimeBehavior(OnTimeBehavior.FIRE_IF_NON_EMPTY) .withTimestampCombiner(TimestampCombiner.EARLIEST) .output(); pipeline.run(); }
Example #18
Source File: GroupByKeyTest.java From beam with Apache License 2.0 | 6 votes |
/** * Tests that when two elements are combined via a GroupByKey their output timestamp agrees with * the windowing function customized to actually be the same as the default, the earlier of the * two values. */ @Test @Category(ValidatesRunner.class) public void testTimestampCombinerEarliest() { p.apply( Create.timestamped( TimestampedValue.of(KV.of(0, "hello"), new Instant(0)), TimestampedValue.of(KV.of(0, "goodbye"), new Instant(10)))) .apply( Window.<KV<Integer, String>>into(FixedWindows.of(Duration.standardMinutes(10))) .withTimestampCombiner(TimestampCombiner.EARLIEST)) .apply(GroupByKey.create()) .apply(ParDo.of(new AssertTimestamp(new Instant(0)))); p.run(); }
Example #19
Source File: ReduceFnRunnerTest.java From beam with Apache License 2.0 | 5 votes |
@Test public void testOnElementBufferingAccumulating() throws Exception { // Test basic execution of a trigger using a non-combining window set and accumulating mode. ReduceFnTester<Integer, Iterable<Integer>, IntervalWindow> tester = ReduceFnTester.nonCombining( FixedWindows.of(Duration.millis(10)), mockTriggerStateMachine, AccumulationMode.ACCUMULATING_FIRED_PANES, Duration.millis(100), ClosingBehavior.FIRE_IF_NON_EMPTY); injectElement(tester, 1); // Fires {1, 2} when(mockTriggerStateMachine.shouldFire(anyTriggerContext())).thenReturn(true); injectElement(tester, 2); // Fires {1, 2, 3} because we are in accumulating mode when(mockTriggerStateMachine.shouldFire(anyTriggerContext())).thenReturn(true); triggerShouldFinish(mockTriggerStateMachine); injectElement(tester, 3); // This element shouldn't be seen, because the trigger has finished injectElement(tester, 4); assertThat( tester.extractOutput(), contains( isSingleWindowedValue(containsInAnyOrder(1, 2), 1, 0, 10), isSingleWindowedValue(containsInAnyOrder(1, 2, 3), 3, 0, 10))); assertTrue(tester.isMarkedFinished(firstWindow)); tester.assertHasOnlyGlobalAndFinishedSetsFor(firstWindow); }
Example #20
Source File: GroupAlsoByWindowParDoFnFactoryTest.java From beam with Apache License 2.0 | 5 votes |
@Test public void testJavaWindowingStrategyDeserialization() throws Exception { WindowFn windowFn = FixedWindows.of(Duration.millis(17)); WindowingStrategy windowingStrategy = WindowingStrategy.of(windowFn); assertThat(windowingStrategy.getWindowFn(), equalTo(windowFn)); }
Example #21
Source File: GroupAlsoByWindowProperties.java From beam with Apache License 2.0 | 5 votes |
/** * Tests that the given GABW implementation correctly groups elements that fall into overlapping * windows that are not merged. */ public static void groupsIntoOverlappingNonmergingWindows( GroupAlsoByWindowDoFnFactory<String, String, Iterable<String>> gabwFactory) throws Exception { WindowingStrategy<?, IntervalWindow> windowingStrategy = WindowingStrategy.of(FixedWindows.of(Duration.millis(10))); List<WindowedValue<KV<String, Iterable<String>>>> result = runGABW( gabwFactory, windowingStrategy, "key", WindowedValue.of("v1", new Instant(1), Arrays.asList(window(0, 5)), PaneInfo.NO_FIRING), WindowedValue.of("v2", new Instant(4), Arrays.asList(window(1, 5)), PaneInfo.NO_FIRING), WindowedValue.of( "v3", new Instant(4), Arrays.asList(window(0, 5)), PaneInfo.NO_FIRING)); assertThat(result, hasSize(2)); TimestampedValue<KV<String, Iterable<String>>> item0 = getOnlyElementInWindow(result, window(0, 5)); assertThat(item0.getValue().getValue(), containsInAnyOrder("v1", "v3")); assertThat(item0.getTimestamp(), equalTo(window(1, 5).maxTimestamp())); TimestampedValue<KV<String, Iterable<String>>> item1 = getOnlyElementInWindow(result, window(1, 5)); assertThat(item1.getValue().getValue(), contains("v2")); assertThat(item1.getTimestamp(), equalTo(window(0, 5).maxTimestamp())); }
Example #22
Source File: BeamModel.java From streamingbook with Apache License 2.0 | 5 votes |
@Override public PCollection<String> expand(PCollection<KV<String, Integer>> input) { return input .apply(Window.<KV<String, Integer>>into(FixedWindows.of(TWO_MINUTES))) .apply(Sum.integersPerKey()) .apply(ParDo.of(new FormatAsStrings())); }
Example #23
Source File: ParDoTest.java From beam with Apache License 2.0 | 5 votes |
@Test @Category(ValidatesRunner.class) public void testWindowingInStartAndFinishBundle() { final FixedWindows windowFn = FixedWindows.of(Duration.millis(1)); PCollection<String> output = pipeline .apply(Create.timestamped(TimestampedValue.of("elem", new Instant(1)))) .apply(Window.into(windowFn)) .apply( ParDo.of( new DoFn<String, String>() { @ProcessElement public void processElement( @Element String element, @Timestamp Instant timestamp, OutputReceiver<String> r) { r.output(element); System.out.println("Process: " + element + ":" + timestamp.getMillis()); } @FinishBundle public void finishBundle(FinishBundleContext c) { Instant ts = new Instant(3); c.output("finish", ts, windowFn.assignWindow(ts)); System.out.println("Finish: 3"); } })) .apply(ParDo.of(new PrintingDoFn())); PAssert.that(output).satisfies(new Checker()); pipeline.run(); }
Example #24
Source File: DistinctTest.java From beam with Apache License 2.0 | 5 votes |
@Test public void testSimpleDuplicatesWithStreamStrategyOldest() { execute( new AbstractTestCase<KV<String, Long>, String>() { @Override public List<String> getUnorderedOutput() { return Arrays.asList("2", "1", "3"); } @Override protected PCollection<String> getOutput(PCollection<KV<String, Long>> input) { input = AssignEventTime.of(input).using(KV::getValue).output(); PCollection<KV<String, Long>> distinct = Distinct.of(input) .projected( in -> in.getKey().substring(0, 1), Distinct.SelectionPolicy.OLDEST, TypeDescriptors.strings()) .windowBy(FixedWindows.of(org.joda.time.Duration.standardSeconds(1))) .triggeredBy(DefaultTrigger.of()) .discardingFiredPanes() .output(); return MapElements.of(distinct).using(KV::getKey).output(); } @Override protected List<KV<String, Long>> getInput() { return asTimedList(100, "1", "2", "3", "3.", "2.", "1."); } @Override protected TypeDescriptor<KV<String, Long>> getInputType() { return TypeDescriptors.kvs(TypeDescriptors.strings(), TypeDescriptors.longs()); } }); }
Example #25
Source File: TopTest.java From beam with Apache License 2.0 | 5 votes |
@Test public void testTopEmptyWithIncompatibleWindows() { p.enableAbandonedNodeEnforcement(false); Window<String> windowingFn = Window.into(FixedWindows.of(Duration.standardDays(10L))); PCollection<String> input = p.apply(Create.empty(StringUtf8Coder.of())).apply(windowingFn); expectedEx.expect(IllegalStateException.class); expectedEx.expectMessage("Top"); expectedEx.expectMessage("GlobalWindows"); expectedEx.expectMessage("withoutDefaults"); expectedEx.expectMessage("asSingletonView"); input.apply(Top.of(1, new OrderByLength())); }
Example #26
Source File: ReduceFnRunnerTest.java From beam with Apache License 2.0 | 5 votes |
/** Tests that a processing time timer does not cause window GC. */ @Test public void testProcessingTimeTimerDoesNotGc() throws Exception { WindowingStrategy<?, IntervalWindow> strategy = WindowingStrategy.of((WindowFn<?, IntervalWindow>) FixedWindows.of(Duration.millis(100))) .withTimestampCombiner(TimestampCombiner.EARLIEST) .withMode(AccumulationMode.ACCUMULATING_FIRED_PANES) .withAllowedLateness(Duration.ZERO) .withTrigger( Repeatedly.forever( AfterProcessingTime.pastFirstElementInPane().plusDelayOf(Duration.millis(10)))); ReduceFnTester<Integer, Integer, IntervalWindow> tester = ReduceFnTester.combining(strategy, Sum.ofIntegers(), VarIntCoder.of()); tester.advanceProcessingTime(new Instant(5000)); injectElement(tester, 2); // processing timer @ 5000 + 10; EOW timer @ 100 injectElement(tester, 5); tester.advanceProcessingTime(new Instant(10000)); tester.assertHasOnlyGlobalAndStateFor(new IntervalWindow(new Instant(0), new Instant(100))); assertThat( tester.extractOutput(), contains( isSingleWindowedValue( equalTo(7), 2, 0, 100, PaneInfo.createPane(true, false, Timing.EARLY, 0, 0)))); }
Example #27
Source File: WindowIntoTranslationTest.java From beam with Apache License 2.0 | 5 votes |
@Parameters(name = "{index}: {0}") public static Iterable<WindowFn<?, ?>> data() { // This pipeline exists for construction, not to run any test. return ImmutableList.<WindowFn<?, ?>>builder() .add(FixedWindows.of(Duration.standardMinutes(10L))) .add(new GlobalWindows()) .add(Sessions.withGapDuration(Duration.standardMinutes(15L))) .add(SlidingWindows.of(Duration.standardMinutes(5L)).every(Duration.standardMinutes(1L))) .add(new CustomWindows()) .build(); }
Example #28
Source File: LateDataUtilsTest.java From beam with Apache License 2.0 | 5 votes |
@Test public void beforeEndOfGlobalWindowSame() { FixedWindows windowFn = FixedWindows.of(Duration.standardMinutes(5)); Duration allowedLateness = Duration.standardMinutes(2); WindowingStrategy<?, ?> strategy = WindowingStrategy.globalDefault() .withWindowFn(windowFn) .withAllowedLateness(allowedLateness); IntervalWindow window = windowFn.assignWindow(new Instant(10)); assertThat( LateDataUtils.garbageCollectionTime(window, strategy), equalTo(window.maxTimestamp().plus(allowedLateness))); }
Example #29
Source File: RepeatedlyStateMachineTest.java From beam with Apache License 2.0 | 5 votes |
/** Tests that onElement correctly passes the data on to the subtrigger. */ @Test public void testOnElement() throws Exception { setUp(FixedWindows.of(Duration.millis(10))); tester.injectElements(37); verify(mockTrigger).onElement(Mockito.any()); }
Example #30
Source File: ReduceFnRunnerTest.java From beam with Apache License 2.0 | 5 votes |
@Test public void noEmptyPanesFinalAlways() throws Exception { ReduceFnTester<Integer, Iterable<Integer>, IntervalWindow> tester = ReduceFnTester.nonCombining( WindowingStrategy.of(FixedWindows.of(Duration.millis(10))) .withTrigger( Repeatedly.forever( AfterFirst.of( AfterPane.elementCountAtLeast(2), AfterWatermark.pastEndOfWindow()))) .withMode(AccumulationMode.ACCUMULATING_FIRED_PANES) .withAllowedLateness(Duration.millis(100)) .withTimestampCombiner(TimestampCombiner.EARLIEST) .withClosingBehavior(ClosingBehavior.FIRE_ALWAYS)); tester.advanceInputWatermark(new Instant(0)); tester.injectElements( TimestampedValue.of(1, new Instant(1)), TimestampedValue.of(2, new Instant(2))); tester.advanceInputWatermark(new Instant(20)); tester.advanceInputWatermark(new Instant(250)); List<WindowedValue<Iterable<Integer>>> output = tester.extractOutput(); assertThat( output, contains( // Trigger with 2 elements isSingleWindowedValue(containsInAnyOrder(1, 2), 1, 0, 10), // Trigger for the empty on time pane isSingleWindowedValue(containsInAnyOrder(1, 2), 9, 0, 10), // Trigger for the final pane isSingleWindowedValue(containsInAnyOrder(1, 2), 9, 0, 10))); }