org.apache.beam.sdk.transforms.windowing.AfterWatermark Java Examples

@Test
public void testGroupByKeyFinishingEndOfWindowEarlyFiringsTriggerNotOk() {
  PCollection<KV<String, String>> input =
      p.apply(Create.of(KV.of("hello", "goodbye")))
          .apply(
              Window.<KV<String, String>>configure()
                  .discardingFiredPanes()
                  .triggering(
                      AfterWatermark.pastEndOfWindow()
                          .withEarlyFirings(AfterPane.elementCountAtLeast(1)))
                  .withAllowedLateness(Duration.millis(10)));

  thrown.expect(IllegalArgumentException.class);
  thrown.expectMessage("Unsafe trigger");
  input.apply(GroupByKey.create());
}
 

@Override
public PCollection<KV<String, Integer>> expand(PCollection<GameActionInfo> infos) {
  return infos
      .apply(
          "LeaderboardTeamFixedWindows",
          Window.<GameActionInfo>into(FixedWindows.of(teamWindowDuration))
              // We will get early (speculative) results as well as cumulative
              // processing of late data.
              .triggering(
                  AfterWatermark.pastEndOfWindow()
                      .withEarlyFirings(
                          AfterProcessingTime.pastFirstElementInPane()
                              .plusDelayOf(FIVE_MINUTES))
                      .withLateFirings(
                          AfterProcessingTime.pastFirstElementInPane()
                              .plusDelayOf(TEN_MINUTES)))
              .withAllowedLateness(allowedLateness)
              .accumulatingFiredPanes())
      // Extract and sum teamname/score pairs from the event data.
      .apply("ExtractTeamScore", new ExtractAndSumScore("team"));
}
 

@Override
public PCollection<KV<String, Integer>> expand(PCollection<GameActionInfo> infos) {
  return infos
      .apply(
          "LeaderboardTeamFixedWindows",
          Window.<GameActionInfo>into(FixedWindows.of(teamWindowDuration))
              // We will get early (speculative) results as well as cumulative
              // processing of late data.
              .triggering(
                  AfterWatermark.pastEndOfWindow()
                      .withEarlyFirings(
                          AfterProcessingTime.pastFirstElementInPane()
                              .plusDelayOf(FIVE_MINUTES))
                      .withLateFirings(
                          AfterProcessingTime.pastFirstElementInPane()
                              .plusDelayOf(TEN_MINUTES)))
              .withAllowedLateness(allowedLateness)
              .accumulatingFiredPanes())
      // Extract and sum teamname/score pairs from the event data.
      .apply("ExtractTeamScore", new ExtractAndSumScore("team"));
}
 

/**
 * Tests that if end-of-window and GC timers come in together, that the pane is correctly marked
 * as final.
 */
@Test
public void testCombiningAccumulatingEventTime() throws Exception {
  WindowingStrategy<?, IntervalWindow> strategy =
      WindowingStrategy.of((WindowFn<?, IntervalWindow>) FixedWindows.of(Duration.millis(100)))
          .withTimestampCombiner(TimestampCombiner.EARLIEST)
          .withMode(AccumulationMode.ACCUMULATING_FIRED_PANES)
          .withAllowedLateness(Duration.millis(1))
          .withTrigger(Repeatedly.forever(AfterWatermark.pastEndOfWindow()));

  ReduceFnTester<Integer, Integer, IntervalWindow> tester =
      ReduceFnTester.combining(strategy, Sum.ofIntegers(), VarIntCoder.of());

  injectElement(tester, 2); // processing timer @ 5000 + 10; EOW timer @ 100
  injectElement(tester, 5);

  tester.advanceInputWatermark(new Instant(1000));

  assertThat(
      tester.extractOutput(),
      contains(
          isSingleWindowedValue(
              equalTo(7), 2, 0, 100, PaneInfo.createPane(true, true, Timing.ON_TIME, 0, 0))));
}
 

@Test
public void testWatermarkHoldForLateNewWindow() throws Exception {
  Duration allowedLateness = Duration.standardMinutes(1);
  Duration gapDuration = Duration.millis(10);
  ReduceFnTester<Integer, Iterable<Integer>, IntervalWindow> tester =
      ReduceFnTester.nonCombining(
          WindowingStrategy.of(Sessions.withGapDuration(gapDuration))
              .withMode(AccumulationMode.DISCARDING_FIRED_PANES)
              .withTrigger(
                  Repeatedly.forever(
                      AfterWatermark.pastEndOfWindow()
                          .withLateFirings(AfterPane.elementCountAtLeast(1))))
              .withAllowedLateness(allowedLateness));
  tester.setAutoAdvanceOutputWatermark(false);

  assertEquals(null, tester.getWatermarkHold());
  assertEquals(null, tester.getOutputWatermark());
  tester.advanceInputWatermark(new Instant(40));
  injectElements(tester, 1);
  assertThat(tester.getWatermarkHold(), nullValue());
  injectElements(tester, 10);
  assertThat(tester.getWatermarkHold(), nullValue());
}
 

static PCollection<Long> applyTransform(PCollection<String> events) {
  return events
      .apply(
          Window.<String>into(FixedWindows.of(Duration.standardDays(1)))
              .triggering(
                  AfterWatermark.pastEndOfWindow()
                  .withEarlyFirings(
                      AfterProcessingTime.pastFirstElementInPane()))
              .withAllowedLateness(Duration.ZERO)
              .discardingFiredPanes())

      .apply(Combine.globally(Count.<String>combineFn()).withoutDefaults());
}
 

@Test
public void testGroupByKeyFinishingEndOfWindowTriggerNotOk() {
  PCollection<KV<String, String>> input =
      p.apply(Create.of(KV.of("hello", "goodbye")))
          .apply(
              Window.<KV<String, String>>configure()
                  .discardingFiredPanes()
                  .triggering(AfterWatermark.pastEndOfWindow())
                  .withAllowedLateness(Duration.millis(10)));

  thrown.expect(IllegalArgumentException.class);
  thrown.expectMessage("Unsafe trigger");
  input.apply(GroupByKey.create());
}
 

@Test
public void testGroupByKeyFinishingEndOfWindowEarlyFiringsTriggerOk() {
  PCollection<KV<String, String>> input =
      p.apply(Create.of(KV.of("hello", "goodbye")))
          .apply(
              Window.<KV<String, String>>configure()
                  .discardingFiredPanes()
                  .triggering(
                      AfterWatermark.pastEndOfWindow()
                          .withEarlyFirings(AfterPane.elementCountAtLeast(1)))
                  .withAllowedLateness(Duration.ZERO));

  // OK
  input.apply(GroupByKey.create());
}
 

@Test
public void testGroupByKeyFinishingEndOfWindowTriggerOk() {
  PCollection<KV<String, String>> input =
      p.apply(Create.of(KV.of("hello", "goodbye")))
          .apply(
              Window.<KV<String, String>>configure()
                  .discardingFiredPanes()
                  .triggering(AfterWatermark.pastEndOfWindow())
                  .withAllowedLateness(Duration.ZERO));

  // OK
  input.apply(GroupByKey.create());
}
 

/**
 * Regression test: when all values are emitted by a speculative trigger, caused a null KV when
 * the on-time firing occurred.
 */
@Test
@Category({NeedsRunner.class, UsesTestStreamWithProcessingTime.class})
public void testTriggeredDistinctRepresentativeValuesEmpty() {
  Instant base = new Instant(0);
  TestStream<KV<Integer, String>> values =
      TestStream.create(KvCoder.of(VarIntCoder.of(), StringUtf8Coder.of()))
          .advanceWatermarkTo(base)
          .addElements(TimestampedValue.of(KV.of(1, "k1"), base))
          .advanceProcessingTime(Duration.standardMinutes(1))
          .advanceWatermarkToInfinity();

  PCollection<KV<Integer, String>> distinctValues =
      triggeredDistinctRepresentativePipeline
          .apply(values)
          .apply(
              Window.<KV<Integer, String>>into(FixedWindows.of(Duration.standardMinutes(1)))
                  .triggering(
                      AfterWatermark.pastEndOfWindow()
                          .withEarlyFirings(
                              AfterProcessingTime.pastFirstElementInPane()
                                  .plusDelayOf(Duration.standardSeconds(30))))
                  .withAllowedLateness(Duration.ZERO)
                  .discardingFiredPanes())
          .apply(
              Distinct.withRepresentativeValueFn(new Keys<Integer>())
                  .withRepresentativeType(TypeDescriptor.of(Integer.class)));

  PAssert.that(distinctValues).containsInAnyOrder(KV.of(1, "k1"));
  triggeredDistinctRepresentativePipeline.run();
}
 

@Test
public void testRejectsNonGlobalWindowsWithRepeatingTrigger() throws Exception {

  String sql =
      "SELECT o1.order_id, o1.price, o1.site_id, o2.order_id, o2.price, o2.site_id  "
          + "FROM ORDER_DETAILS1 o1"
          + " JOIN ORDER_DETAILS2 o2"
          + " on "
          + " o1.order_id=o2.site_id AND o2.price=o1.site_id";

  PCollection<Row> orders =
      ordersUnbounded()
          .apply(
              "window",
              Window.<Row>into(FixedWindows.of(Duration.standardSeconds(203)))
                  .triggering(Repeatedly.forever(AfterWatermark.pastEndOfWindow()))
                  .withAllowedLateness(Duration.standardMinutes(2))
                  .accumulatingFiredPanes());
  PCollectionTuple inputs = tuple("ORDER_DETAILS1", orders, "ORDER_DETAILS2", orders);

  thrown.expect(UnsupportedOperationException.class);
  thrown.expectMessage(
      stringContainsInOrder(Arrays.asList("once per window", "default trigger")));

  inputs.apply("sql", SqlTransform.query(sql));

  pipeline.run();
}
 

@Test
@Category(NeedsRunner.class)
public void singlePaneSingleReifiedPane() {
  PCollection<Iterable<ValueInSingleWindow<Iterable<Long>>>> accumulatedPanes =
      p.apply(GenerateSequence.from(0).to(20000))
          .apply(WithTimestamps.of(input -> new Instant(input * 10)))
          .apply(
              Window.<Long>into(FixedWindows.of(Duration.standardMinutes(1)))
                  .triggering(AfterWatermark.pastEndOfWindow())
                  .withAllowedLateness(Duration.ZERO)
                  .discardingFiredPanes())
          .apply(WithKeys.<Void, Long>of((Void) null).withKeyType(new TypeDescriptor<Void>() {}))
          .apply(GroupByKey.create())
          .apply(Values.create())
          .apply(GatherAllPanes.globally());

  PAssert.that(accumulatedPanes)
      .satisfies(
          input -> {
            for (Iterable<ValueInSingleWindow<Iterable<Long>>> windowedInput : input) {
              if (Iterables.size(windowedInput) > 1) {
                fail("Expected all windows to have exactly one pane, got " + windowedInput);
                return null;
              }
            }
            return null;
          });

  p.run();
}
 

@Test
public void testBuild_OptionalWindowing() {
  final Pipeline pipeline = TestUtils.createTestPipeline();
  final PCollection<String> left =
      TestUtils.createMockDataset(pipeline, TypeDescriptors.strings());
  final PCollection<String> right =
      TestUtils.createMockDataset(pipeline, TypeDescriptors.strings());
  final PCollection<KV<Integer, String>> joined =
      Join.named("Join1")
          .of(left, right)
          .by(String::length, String::length)
          .using((String l, String r, Collector<String> c) -> c.collect(l + r))
          .applyIf(
              true,
              b ->
                  b.windowBy(FixedWindows.of(org.joda.time.Duration.standardHours(1)))
                      .triggeredBy(AfterWatermark.pastEndOfWindow())
                      .accumulationMode(AccumulationMode.DISCARDING_FIRED_PANES))
          .output();
  final Join join = (Join) TestUtils.getProducer(joined);
  assertTrue(join.getWindow().isPresent());
  final Window<?> window = (Window) join.getWindow().get();
  assertEquals(FixedWindows.of(org.joda.time.Duration.standardHours(1)), window.getWindowFn());
  assertEquals(AfterWatermark.pastEndOfWindow(), WindowDesc.of(window).getTrigger());
  assertEquals(
      AccumulationMode.DISCARDING_FIRED_PANES, WindowDesc.of(window).getAccumulationMode());
}
 

@Test
public void testBuild_Windowing() {
  final Pipeline pipeline = TestUtils.createTestPipeline();
  final PCollection<String> left =
      TestUtils.createMockDataset(pipeline, TypeDescriptors.strings());
  final PCollection<String> right =
      TestUtils.createMockDataset(pipeline, TypeDescriptors.strings());
  final PCollection<KV<Integer, String>> joined =
      Join.named("Join1")
          .of(left, right)
          .by(String::length, String::length)
          .using((String l, String r, Collector<String> c) -> c.collect(l + r))
          .windowBy(FixedWindows.of(org.joda.time.Duration.standardHours(1)))
          .triggeredBy(AfterWatermark.pastEndOfWindow())
          .discardingFiredPanes()
          .withAllowedLateness(Duration.millis(1000))
          .output();
  final Join join = (Join) TestUtils.getProducer(joined);
  assertTrue(join.getWindow().isPresent());
  @SuppressWarnings("unchecked")
  final WindowDesc<?> windowDesc = WindowDesc.of((Window) join.getWindow().get());
  assertEquals(
      FixedWindows.of(org.joda.time.Duration.standardHours(1)), windowDesc.getWindowFn());
  assertEquals(AfterWatermark.pastEndOfWindow(), windowDesc.getTrigger());
  assertEquals(AccumulationMode.DISCARDING_FIRED_PANES, windowDesc.getAccumulationMode());
  assertEquals(Duration.millis(1000), windowDesc.getAllowedLateness());
}
 

@Test
public void testRejectsGlobalWindowsWithEndOfWindowTrigger() throws Exception {

  String sql =
      "SELECT o1.order_id, o1.price, o1.site_id, o2.order_id, o2.price, o2.site_id  "
          + "FROM ORDER_DETAILS1 o1"
          + " JOIN ORDER_DETAILS2 o2"
          + " on "
          + " o1.order_id=o2.site_id AND o2.price=o1.site_id";

  PCollection<Row> orders =
      ordersUnbounded()
          .apply(
              "window",
              Window.<Row>into(new GlobalWindows())
                  .triggering(AfterWatermark.pastEndOfWindow())
                  .withAllowedLateness(Duration.ZERO)
                  .accumulatingFiredPanes());
  PCollectionTuple inputs = tuple("ORDER_DETAILS1", orders, "ORDER_DETAILS2", orders);

  thrown.expect(UnsupportedOperationException.class);
  thrown.expectMessage(
      stringContainsInOrder(Arrays.asList("once per window", "default trigger")));

  inputs.apply("sql", SqlTransform.query(sql));

  pipeline.run();
}
 

static PCollection<Long> applyTransform(PCollection<String> events) {
  return events
      .apply(
          Window.<String>into(FixedWindows.of(Duration.standardDays(1)))
              .triggering(
                  AfterWatermark.pastEndOfWindow()
                      .withEarlyFirings(
                          AfterProcessingTime.pastFirstElementInPane()))
              .withAllowedLateness(Duration.ZERO)
              .accumulatingFiredPanes())

      .apply(Combine.globally(Count.<String>combineFn()).withoutDefaults());
}
 

@Override
public PCollection<String> expand(PCollection<KV<String, Integer>> input) {
    return input
        .apply(Window.<KV<String, Integer>>into(FixedWindows.of(TWO_MINUTES))
               .triggering(AfterWatermark.pastEndOfWindow()
                           .withEarlyFirings(AfterProcessingTime.pastFirstElementInPane().plusDelayOf(ONE_MINUTE))
                           .withLateFirings(AfterPane.elementCountAtLeast(1)))
               .withAllowedLateness(Duration.standardDays(1000))
               .discardingFiredPanes())
        .apply(Sum.integersPerKey())
        .apply(ParDo.of(new FormatAsStrings()));
}
 

@Override
public PCollection<String> expand(PCollection<KV<String, Integer>> input) {
    return input
        .apply(Window.<KV<String, Integer>>into(FixedWindows.of(TWO_MINUTES))
               .triggering(AfterWatermark.pastEndOfWindow()
                           .withEarlyFirings(AfterProcessingTime.pastFirstElementInPane().plusDelayOf(ONE_MINUTE))
                           .withLateFirings(AfterPane.elementCountAtLeast(1)))
               .withAllowedLateness(Duration.standardDays(1000))
               .accumulatingFiredPanes())
        .apply(Sum.integersPerKey())
        .apply(ParDo.of(new FormatAsStrings()));
}
 

@Test
public void testGroupByKeyEndOfWindowLateFiringsOk() {
  PCollection<KV<String, String>> input =
      p.apply(Create.of(KV.of("hello", "goodbye")))
          .apply(
              Window.<KV<String, String>>configure()
                  .discardingFiredPanes()
                  .triggering(
                      AfterWatermark.pastEndOfWindow()
                          .withLateFirings(AfterPane.elementCountAtLeast(1)))
                  .withAllowedLateness(Duration.millis(10)));

  // OK
  input.apply(GroupByKey.create());
}
 

@Test
@Category(NeedsRunner.class)
public void multiplePanesMultipleReifiedPane() {
  PCollection<Long> someElems = p.apply("someLongs", GenerateSequence.from(0).to(20000));
  PCollection<Long> otherElems = p.apply("otherLongs", GenerateSequence.from(0).to(20000));
  PCollection<Iterable<ValueInSingleWindow<Iterable<Long>>>> accumulatedPanes =
      PCollectionList.of(someElems)
          .and(otherElems)
          .apply(Flatten.pCollections())
          .apply(WithTimestamps.of(input -> new Instant(input * 10)))
          .apply(
              Window.<Long>into(FixedWindows.of(Duration.standardMinutes(1)))
                  .triggering(
                      AfterWatermark.pastEndOfWindow()
                          .withEarlyFirings(AfterPane.elementCountAtLeast(1)))
                  .withAllowedLateness(Duration.ZERO)
                  .discardingFiredPanes())
          .apply(WithKeys.<Void, Long>of((Void) null).withKeyType(new TypeDescriptor<Void>() {}))
          .apply(GroupByKey.create())
          .apply(Values.create())
          .apply(GatherAllPanes.globally());

  PAssert.that(accumulatedPanes)
      .satisfies(
          input -> {
            for (Iterable<ValueInSingleWindow<Iterable<Long>>> windowedInput : input) {
              if (Iterables.size(windowedInput) > 1) {
                return null;
              }
            }
            fail("Expected at least one window to have multiple panes");
            return null;
          });

  p.run();
}
 

@Test
@Category({NeedsRunner.class, UsesTestStreamWithProcessingTime.class})
public void testProcessingTimeTrigger() {
  TestStream<Long> source =
      TestStream.create(VarLongCoder.of())
          .addElements(
              TimestampedValue.of(1L, new Instant(1000L)),
              TimestampedValue.of(2L, new Instant(2000L)))
          .advanceProcessingTime(Duration.standardMinutes(12))
          .addElements(TimestampedValue.of(3L, new Instant(3000L)))
          .advanceProcessingTime(Duration.standardMinutes(6))
          .advanceWatermarkToInfinity();

  PCollection<Long> sum =
      p.apply(source)
          .apply(
              Window.<Long>configure()
                  .triggering(
                      AfterWatermark.pastEndOfWindow()
                          .withEarlyFirings(
                              AfterProcessingTime.pastFirstElementInPane()
                                  .plusDelayOf(Duration.standardMinutes(5))))
                  .accumulatingFiredPanes()
                  .withAllowedLateness(Duration.ZERO))
          .apply(Sum.longsGlobally());

  PAssert.that(sum).inEarlyGlobalWindowPanes().containsInAnyOrder(3L, 6L);

  p.run();
}
 

@Parameters(name = "{index}")
public static Iterable<Pipeline> testPipelines() {
  Pipeline trivialPipeline = Pipeline.create();
  trivialPipeline.apply(Create.of(1, 2, 3));

  Pipeline sideInputPipeline = Pipeline.create();
  final PCollectionView<String> singletonView =
      sideInputPipeline.apply(Create.of("foo")).apply(View.asSingleton());
  sideInputPipeline
      .apply(Create.of("main input"))
      .apply(
          ParDo.of(
                  new DoFn<String, String>() {
                    @ProcessElement
                    public void process(ProcessContext c) {
                      // actually never executed and no effect on translation
                      c.sideInput(singletonView);
                    }
                  })
              .withSideInputs(singletonView));

  Pipeline complexPipeline = Pipeline.create();
  BigEndianLongCoder customCoder = BigEndianLongCoder.of();
  PCollection<Long> elems = complexPipeline.apply(GenerateSequence.from(0L).to(207L));
  PCollection<Long> counted = elems.apply(Count.globally()).setCoder(customCoder);
  PCollection<Long> windowed =
      counted.apply(
          Window.<Long>into(FixedWindows.of(Duration.standardMinutes(7)))
              .triggering(
                  AfterWatermark.pastEndOfWindow()
                      .withLateFirings(AfterPane.elementCountAtLeast(19)))
              .accumulatingFiredPanes()
              .withAllowedLateness(Duration.standardMinutes(3L)));
  final WindowingStrategy<?, ?> windowedStrategy = windowed.getWindowingStrategy();
  PCollection<KV<String, Long>> keyed = windowed.apply(WithKeys.of("foo"));
  PCollection<KV<String, Iterable<Long>>> grouped = keyed.apply(GroupByKey.create());

  return ImmutableList.of(trivialPipeline, sideInputPipeline, complexPipeline);
}
 

@Parameters(name = "{index}: {0}")
public static Iterable<PCollection<?>> data() {
  Pipeline pipeline = TestPipeline.create();
  PCollection<Integer> ints = pipeline.apply("ints", Create.of(1, 2, 3));
  PCollection<Long> longs = pipeline.apply("unbounded longs", GenerateSequence.from(0));
  PCollection<Long> windowedLongs =
      longs.apply(
          "into fixed windows", Window.into(FixedWindows.of(Duration.standardMinutes(10L))));
  PCollection<KV<String, Iterable<String>>> groupedStrings =
      pipeline
          .apply(
              "kvs", Create.of(KV.of("foo", "spam"), KV.of("bar", "ham"), KV.of("baz", "eggs")))
          .apply("group", GroupByKey.create());
  PCollection<Long> coderLongs =
      pipeline
          .apply("counts with alternative coder", GenerateSequence.from(0).to(10))
          .setCoder(BigEndianLongCoder.of());
  pipeline
      .apply(
          "intsWithCustomCoder",
          Create.of(1, 2).withCoder(new AutoValue_PCollectionTranslationTest_CustomIntCoder()))
      .apply(
          "into custom windows",
          Window.into(new CustomWindows())
              .triggering(
                  AfterWatermark.pastEndOfWindow()
                      .withEarlyFirings(
                          AfterFirst.of(
                              AfterPane.elementCountAtLeast(5),
                              AfterProcessingTime.pastFirstElementInPane()
                                  .plusDelayOf(Duration.millis(227L)))))
              .accumulatingFiredPanes()
              .withAllowedLateness(Duration.standardMinutes(12L)));
  return ImmutableList.of(ints, longs, windowedLongs, coderLongs, groupedStrings);
}
 

@Test
public void testMergingLateWatermarkHolds() throws Exception {
  MetricsContainerImpl container = new MetricsContainerImpl("any");
  MetricsEnvironment.setCurrentContainer(container);
  Duration gapDuration = Duration.millis(10);
  Duration allowedLateness = Duration.standardMinutes(100);
  ReduceFnTester<Integer, Iterable<Integer>, IntervalWindow> tester =
      ReduceFnTester.nonCombining(
          WindowingStrategy.of(Sessions.withGapDuration(gapDuration))
              .withMode(AccumulationMode.DISCARDING_FIRED_PANES)
              .withTrigger(
                  Repeatedly.forever(
                      AfterWatermark.pastEndOfWindow()
                          .withLateFirings(AfterPane.elementCountAtLeast(10))))
              .withAllowedLateness(allowedLateness));
  tester.setAutoAdvanceOutputWatermark(false);

  // Input watermark -> null
  assertEquals(null, tester.getWatermarkHold());
  assertEquals(null, tester.getOutputWatermark());

  tester.advanceInputWatermark(new Instant(20));
  // Add two late elements that cause a window to merge.
  injectElements(tester, Arrays.asList(3));
  assertThat(tester.getWatermarkHold(), nullValue());
  injectElements(tester, Arrays.asList(4));
  Instant endOfWindow = new Instant(4).plus(gapDuration);
  // We expect a GC hold to be one less than the end of window plus the allowed lateness.
  Instant expectedGcHold = endOfWindow.plus(allowedLateness).minus(1);
  assertEquals(expectedGcHold, tester.getWatermarkHold());
  tester.advanceInputWatermark(new Instant(1000));
  assertEquals(expectedGcHold, tester.getWatermarkHold());
}
 

@Test
public void noEmptyPanesFinalIfNonEmpty() throws Exception {
  ReduceFnTester<Integer, Iterable<Integer>, IntervalWindow> tester =
      ReduceFnTester.nonCombining(
          WindowingStrategy.of(FixedWindows.of(Duration.millis(10)))
              .withTrigger(
                  Repeatedly.forever(
                      AfterFirst.of(
                          AfterPane.elementCountAtLeast(2), AfterWatermark.pastEndOfWindow())))
              .withMode(AccumulationMode.ACCUMULATING_FIRED_PANES)
              .withAllowedLateness(Duration.millis(100))
              .withTimestampCombiner(TimestampCombiner.EARLIEST)
              .withClosingBehavior(ClosingBehavior.FIRE_IF_NON_EMPTY));

  tester.advanceInputWatermark(new Instant(0));
  tester.injectElements(
      TimestampedValue.of(1, new Instant(1)), TimestampedValue.of(2, new Instant(2)));
  tester.advanceInputWatermark(new Instant(20));
  tester.advanceInputWatermark(new Instant(250));

  List<WindowedValue<Iterable<Integer>>> output = tester.extractOutput();
  assertThat(
      output,
      contains(
          // Trigger with 2 elements
          isSingleWindowedValue(containsInAnyOrder(1, 2), 1, 0, 10),
          // Trigger for the empty on time pane
          isSingleWindowedValue(containsInAnyOrder(1, 2), 9, 0, 10)));
}
 

@Test
public void noEmptyPanesFinalAlways() throws Exception {
  ReduceFnTester<Integer, Iterable<Integer>, IntervalWindow> tester =
      ReduceFnTester.nonCombining(
          WindowingStrategy.of(FixedWindows.of(Duration.millis(10)))
              .withTrigger(
                  Repeatedly.forever(
                      AfterFirst.of(
                          AfterPane.elementCountAtLeast(2), AfterWatermark.pastEndOfWindow())))
              .withMode(AccumulationMode.ACCUMULATING_FIRED_PANES)
              .withAllowedLateness(Duration.millis(100))
              .withTimestampCombiner(TimestampCombiner.EARLIEST)
              .withClosingBehavior(ClosingBehavior.FIRE_ALWAYS));

  tester.advanceInputWatermark(new Instant(0));
  tester.injectElements(
      TimestampedValue.of(1, new Instant(1)), TimestampedValue.of(2, new Instant(2)));
  tester.advanceInputWatermark(new Instant(20));
  tester.advanceInputWatermark(new Instant(250));

  List<WindowedValue<Iterable<Integer>>> output = tester.extractOutput();
  assertThat(
      output,
      contains(
          // Trigger with 2 elements
          isSingleWindowedValue(containsInAnyOrder(1, 2), 1, 0, 10),
          // Trigger for the empty on time pane
          isSingleWindowedValue(containsInAnyOrder(1, 2), 9, 0, 10),
          // Trigger for the final pane
          isSingleWindowedValue(containsInAnyOrder(1, 2), 9, 0, 10)));
}
 

@Test
public void testPaneInfoFinalAndOnTime() throws Exception {
  ReduceFnTester<Integer, Iterable<Integer>, IntervalWindow> tester =
      ReduceFnTester.nonCombining(
          WindowingStrategy.of(FixedWindows.of(Duration.millis(10)))
              .withTrigger(
                  Repeatedly.forever(AfterPane.elementCountAtLeast(2))
                      .orFinally(AfterWatermark.pastEndOfWindow()))
              .withMode(AccumulationMode.DISCARDING_FIRED_PANES)
              .withAllowedLateness(Duration.millis(100))
              .withClosingBehavior(ClosingBehavior.FIRE_ALWAYS));

  tester.advanceInputWatermark(new Instant(0));

  // Should trigger due to element count
  tester.injectElements(
      TimestampedValue.of(1, new Instant(1)), TimestampedValue.of(2, new Instant(2)));

  assertThat(
      tester.extractOutput(),
      contains(
          WindowMatchers.valueWithPaneInfo(
              PaneInfo.createPane(true, false, Timing.EARLY, 0, -1))));

  tester.advanceInputWatermark(new Instant(150));
  assertThat(
      tester.extractOutput(),
      contains(
          WindowMatchers.valueWithPaneInfo(
              PaneInfo.createPane(false, true, Timing.ON_TIME, 1, 0))));
}
 

/**
 * Test that it fires an empty on-time isFinished pane when OnTimeBehavior is FIRE_ALWAYS and
 * ClosingBehavior is FIRE_IF_NON_EMPTY.
 *
 * <p>This is a test just for backward compatibility.
 */
@Test
public void testEmptyOnTimeWithOnTimeBehaviorBackwardCompatibility() throws Exception {
  WindowingStrategy<?, IntervalWindow> strategy =
      WindowingStrategy.of((WindowFn<?, IntervalWindow>) FixedWindows.of(Duration.millis(10)))
          .withTimestampCombiner(TimestampCombiner.EARLIEST)
          .withTrigger(
              AfterWatermark.pastEndOfWindow().withEarlyFirings(AfterPane.elementCountAtLeast(1)))
          .withMode(AccumulationMode.ACCUMULATING_FIRED_PANES)
          .withAllowedLateness(Duration.ZERO)
          .withClosingBehavior(ClosingBehavior.FIRE_IF_NON_EMPTY);

  ReduceFnTester<Integer, Integer, IntervalWindow> tester =
      ReduceFnTester.combining(strategy, Sum.ofIntegers(), VarIntCoder.of());

  tester.advanceInputWatermark(new Instant(0));
  tester.advanceProcessingTime(new Instant(0));

  tester.injectElements(TimestampedValue.of(1, new Instant(1)));

  // Should fire empty on time isFinished pane
  tester.advanceInputWatermark(new Instant(11));

  List<WindowedValue<Integer>> output = tester.extractOutput();
  assertEquals(2, output.size());

  assertThat(
      output.get(0),
      WindowMatchers.valueWithPaneInfo(PaneInfo.createPane(true, false, Timing.EARLY, 0, -1)));
  assertThat(
      output.get(1),
      WindowMatchers.valueWithPaneInfo(PaneInfo.createPane(false, true, Timing.ON_TIME, 1, 0)));
}
 

/**
 * Late elements should still have a garbage collection hold set so that they can make a late pane
 * rather than be dropped due to lateness.
 */
@Test
public void setGarbageCollectionHoldOnLateElements() throws Exception {
  ReduceFnTester<Integer, Iterable<Integer>, IntervalWindow> tester =
      ReduceFnTester.nonCombining(
          WindowingStrategy.of(FixedWindows.of(Duration.millis(10)))
              .withTrigger(
                  AfterWatermark.pastEndOfWindow()
                      .withLateFirings(AfterPane.elementCountAtLeast(2)))
              .withMode(AccumulationMode.DISCARDING_FIRED_PANES)
              .withAllowedLateness(Duration.millis(100))
              .withClosingBehavior(ClosingBehavior.FIRE_IF_NON_EMPTY));

  tester.advanceInputWatermark(new Instant(0));
  tester.advanceOutputWatermark(new Instant(0));
  tester.injectElements(TimestampedValue.of(1, new Instant(1)));

  // Fire ON_TIME pane @ 9 with 1

  tester.advanceInputWatermark(new Instant(109));
  tester.advanceOutputWatermark(new Instant(109));
  tester.injectElements(TimestampedValue.of(2, new Instant(2)));
  // We should have set a garbage collection hold for the final pane.
  Instant hold = tester.getWatermarkHold();
  assertEquals(new Instant(109), hold);

  tester.advanceInputWatermark(new Instant(110));
  tester.advanceOutputWatermark(new Instant(110));

  // Fire final LATE pane @ 9 with 2

  List<WindowedValue<Iterable<Integer>>> output = tester.extractOutput();
  assertEquals(2, output.size());
}
 

/**
 * Test running of {@link Pipeline} which has two {@link POutput POutputs} and finishing the first
 * one triggers data being fed into the second one.
 */
@Test(timeout = 10000)
public void testTwoPOutputsInPipelineWithCascade() throws InterruptedException {

  StaticQueue<Integer> start = StaticQueue.of("start", VarIntCoder.of());
  StaticQueue<Integer> messages = StaticQueue.of("messages", VarIntCoder.of());

  Pipeline pipeline = getPipeline(false);
  pipeline.begin().apply("outputStartSignal", outputStartTo(start));
  PCollection<Integer> result =
      pipeline
          .apply("processMessages", messages.read())
          .apply(
              Window.<Integer>into(new GlobalWindows())
                  .triggering(AfterWatermark.pastEndOfWindow())
                  .discardingFiredPanes()
                  .withAllowedLateness(Duration.ZERO))
          .apply(Sum.integersGlobally());

  // the result should be 6, after the data will have been written
  PAssert.that(result).containsInAnyOrder(6);

  PipelineResult run = pipeline.run();

  // wait until a message has been written to the start queue
  while (start.take() == null) {}

  // and publish messages
  messages.add(1).add(2).add(3).terminate();

  run.waitUntilFinish();
}