org.apache.beam.sdk.transforms.windowing.DefaultTrigger Java Examples

@Test
public void windowingSection() {

  PCollection<Integer> input =
      pipeline.apply(Create.of(1, 2, 3, 4)).setTypeDescriptor(TypeDescriptors.integers());

  PCollection<KV<Integer, Long>> countedElements =
      CountByKey.of(input)
          .keyBy(e -> e)
          .windowBy(FixedWindows.of(Duration.standardSeconds(1)))
          .triggeredBy(DefaultTrigger.of())
          .discardingFiredPanes()
          .withAllowedLateness(Duration.standardSeconds(5))
          .withOnTimeBehavior(OnTimeBehavior.FIRE_IF_NON_EMPTY)
          .withTimestampCombiner(TimestampCombiner.EARLIEST)
          .output();

  pipeline.run();
}
 

private void verifyInputWindowing(PCollection<KV<KeyT, ValueT>> input) {
  if (input.isBounded().equals(PCollection.IsBounded.UNBOUNDED)) {
    checkArgument(
        !input.getWindowingStrategy().equals(WindowingStrategy.globalDefault()),
        "Cannot work with %s and GLOBAL %s",
        PCollection.IsBounded.UNBOUNDED,
        WindowingStrategy.class.getSimpleName());
    checkArgument(
        input.getWindowingStrategy().getTrigger().getClass().equals(DefaultTrigger.class),
        "Cannot work with %s trigger. Write works correctly only with %s",
        input.getWindowingStrategy().getTrigger().getClass().getSimpleName(),
        DefaultTrigger.class.getSimpleName());
    checkArgument(
        input.getWindowingStrategy().getAllowedLateness().equals(Duration.ZERO),
        "Write does not allow late data.");
  }
}
 

@Test
public void testWindow_applyIf() {
  final PCollection<String> dataset = TestUtils.createMockDataset(TypeDescriptors.strings());
  final PCollection<Triple<String, Long, Long>> result =
      TopPerKey.of(dataset)
          .keyBy(s -> s)
          .valueBy(s -> 1L)
          .scoreBy(s -> 1L)
          .applyIf(
              true,
              b ->
                  b.windowBy(FixedWindows.of(org.joda.time.Duration.standardHours(1)))
                      .triggeredBy(DefaultTrigger.of())
                      .accumulatingFiredPanes())
          .output();
  final TopPerKey tpk = (TopPerKey) TestUtils.getProducer(result);
  assertTrue(tpk.getWindow().isPresent());
  @SuppressWarnings("unchecked")
  final WindowDesc<?> windowDesc = WindowDesc.of((Window) tpk.getWindow().get());
  assertEquals(
      FixedWindows.of(org.joda.time.Duration.standardHours(1)), windowDesc.getWindowFn());
  assertEquals(DefaultTrigger.of(), windowDesc.getTrigger());
  assertEquals(AccumulationMode.ACCUMULATING_FIRED_PANES, windowDesc.getAccumulationMode());
}
 

@Test
public void testBuild_Windowing() {
  final PCollection<String> dataset = TestUtils.createMockDataset(TypeDescriptors.strings());
  final PCollection<Triple<String, Long, Long>> result =
      TopPerKey.of(dataset)
          .keyBy(s -> s)
          .valueBy(s -> 1L)
          .scoreBy(s -> 1L)
          .windowBy(FixedWindows.of(org.joda.time.Duration.standardHours(1)))
          .triggeredBy(DefaultTrigger.of())
          .accumulationMode(AccumulationMode.DISCARDING_FIRED_PANES)
          .output();
  final TopPerKey tpk = (TopPerKey) TestUtils.getProducer(result);
  assertTrue(tpk.getWindow().isPresent());
  @SuppressWarnings("unchecked")
  final WindowDesc<?> windowDesc = WindowDesc.of((Window) tpk.getWindow().get());
  assertEquals(
      FixedWindows.of(org.joda.time.Duration.standardHours(1)), windowDesc.getWindowFn());
  assertEquals(DefaultTrigger.of(), windowDesc.getTrigger());
  assertEquals(AccumulationMode.DISCARDING_FIRED_PANES, windowDesc.getAccumulationMode());
}
 

@Test
public void testWindow_applyIf() {
  final PCollection<String> dataset = TestUtils.createMockDataset(TypeDescriptors.strings());
  final PCollection<String> uniq =
      Distinct.of(dataset)
          .applyIf(
              true,
              b ->
                  b.windowBy(FixedWindows.of(Duration.standardHours(1)))
                      .triggeredBy(DefaultTrigger.of())
                      .discardingFiredPanes())
          .output();
  final Distinct distinct = (Distinct) TestUtils.getProducer(uniq);
  assertTrue(distinct.getWindow().isPresent());
  @SuppressWarnings("unchecked")
  final WindowDesc<?> windowDesc = WindowDesc.of((Window) distinct.getWindow().get());
  assertEquals(
      FixedWindows.of(org.joda.time.Duration.standardHours(1)), windowDesc.getWindowFn());
  assertEquals(DefaultTrigger.of(), windowDesc.getTrigger());
  assertEquals(AccumulationMode.DISCARDING_FIRED_PANES, windowDesc.getAccumulationMode());
}
 

@Test
public void testBuild_Windowing() {
  final PCollection<String> dataset = TestUtils.createMockDataset(TypeDescriptors.strings());
  final PCollection<String> uniq =
      Distinct.of(dataset)
          .windowBy(FixedWindows.of(org.joda.time.Duration.standardHours(1)))
          .triggeredBy(DefaultTrigger.of())
          .accumulationMode(AccumulationMode.DISCARDING_FIRED_PANES)
          .output();
  final Distinct distinct = (Distinct) TestUtils.getProducer(uniq);
  assertTrue(distinct.getWindow().isPresent());
  @SuppressWarnings("unchecked")
  final WindowDesc<?> windowDesc = WindowDesc.of((Window) distinct.getWindow().get());
  assertEquals(
      FixedWindows.of(org.joda.time.Duration.standardHours(1)), windowDesc.getWindowFn());
  assertEquals(DefaultTrigger.of(), windowDesc.getTrigger());
}
 

@Test
public void testBuild() {
  final PCollection<String> dataset = TestUtils.createMockDataset(TypeDescriptors.strings());
  final FixedWindows windowing = FixedWindows.of(org.joda.time.Duration.standardHours(1));
  final DefaultTrigger trigger = DefaultTrigger.of();
  final PCollection<String> uniq =
      Distinct.named("Distinct1")
          .of(dataset)
          .windowBy(windowing)
          .triggeredBy(trigger)
          .discardingFiredPanes()
          .withAllowedLateness(Duration.millis(1000))
          .output();
  final Distinct distinct = (Distinct) TestUtils.getProducer(uniq);
  assertTrue(distinct.getName().isPresent());
  assertEquals("Distinct1", distinct.getName().get());

  assertTrue(distinct.getWindow().isPresent());
  @SuppressWarnings("unchecked")
  final WindowDesc<?> windowDesc = WindowDesc.of((Window) distinct.getWindow().get());
  assertEquals(windowing, windowDesc.getWindowFn());
  assertEquals(trigger, windowDesc.getTrigger());
  assertEquals(AccumulationMode.DISCARDING_FIRED_PANES, windowDesc.getAccumulationMode());
  assertEquals(Duration.millis(1000), windowDesc.getAllowedLateness());
}
 

@Test
public void testWindow_applyIf() {
  final PCollection<String> dataset = TestUtils.createMockDataset(TypeDescriptors.strings());
  final PCollection<KV<String, Long>> counted =
      SumByKey.of(dataset)
          .keyBy(s -> s)
          .valueBy(s -> 1L)
          .applyIf(
              true,
              b ->
                  b.windowBy(FixedWindows.of(org.joda.time.Duration.standardHours(1)))
                      .triggeredBy(DefaultTrigger.of())
                      .accumulationMode(AccumulationMode.DISCARDING_FIRED_PANES))
          .output();
  final SumByKey sum = (SumByKey) TestUtils.getProducer(counted);
  assertTrue(sum.getWindow().isPresent());
  final Window<?> window = (Window) sum.getWindow().get();
  assertEquals(FixedWindows.of(org.joda.time.Duration.standardHours(1)), window.getWindowFn());
  assertEquals(DefaultTrigger.of(), WindowDesc.of(window).getTrigger());
  assertEquals(
      AccumulationMode.DISCARDING_FIRED_PANES, WindowDesc.of(window).getAccumulationMode());
}
 

@Test
public void testBuild_Windowing() {
  final PCollection<String> dataset = TestUtils.createMockDataset(TypeDescriptors.strings());
  final PCollection<KV<String, Long>> counted =
      SumByKey.of(dataset)
          .keyBy(s -> s)
          .valueBy(s -> 1L)
          .windowBy(FixedWindows.of(org.joda.time.Duration.standardHours(1)))
          .triggeredBy(DefaultTrigger.of())
          .discardingFiredPanes()
          .withAllowedLateness(Duration.millis(1000))
          .output();
  final SumByKey sum = (SumByKey) TestUtils.getProducer(counted);
  assertTrue(sum.getWindow().isPresent());
  @SuppressWarnings("unchecked")
  final WindowDesc<?> windowDesc = WindowDesc.of((Window) sum.getWindow().get());
  assertEquals(
      FixedWindows.of(org.joda.time.Duration.standardHours(1)), windowDesc.getWindowFn());
  assertEquals(DefaultTrigger.of(), windowDesc.getTrigger());
  assertEquals(AccumulationMode.DISCARDING_FIRED_PANES, windowDesc.getAccumulationMode());
  assertEquals(Duration.millis(1000), windowDesc.getAllowedLateness());
}
 

@Test
public void testWindow_applyIf() {
  final PCollection<String> dataset = TestUtils.createMockDataset(TypeDescriptors.strings());
  final FixedWindows windowing = FixedWindows.of(org.joda.time.Duration.standardHours(1));
  final DefaultTrigger trigger = DefaultTrigger.of();
  final PCollection<KV<String, Long>> counted =
      CountByKey.named("CountByKey1")
          .of(dataset)
          .keyBy(s -> s)
          .applyIf(true, b -> b.windowBy(windowing).triggeredBy(trigger).discardingFiredPanes())
          .output();
  final CountByKey count = (CountByKey) TestUtils.getProducer(counted);
  assertTrue(count.getWindow().isPresent());
  final WindowDesc<?> desc = WindowDesc.of((Window<?>) count.getWindow().get());
  assertEquals(windowing, desc.getWindowFn());
  assertEquals(trigger, desc.getTrigger());
  assertEquals(AccumulationMode.DISCARDING_FIRED_PANES, desc.getAccumulationMode());
}
 

@Test
public void testBuild_Windowing() {
  final PCollection<String> dataset = TestUtils.createMockDataset(TypeDescriptors.strings());
  final PCollection<KV<String, Long>> counted =
      CountByKey.named("CountByKey1")
          .of(dataset)
          .keyBy(s -> s)
          .windowBy(FixedWindows.of(org.joda.time.Duration.standardHours(1)))
          .triggeredBy(DefaultTrigger.of())
          .accumulationMode(AccumulationMode.DISCARDING_FIRED_PANES)
          .output();
  final CountByKey count = (CountByKey) TestUtils.getProducer(counted);
  assertTrue(count.getWindow().isPresent());
  final WindowDesc<?> desc = WindowDesc.of((Window<?>) count.getWindow().get());
  assertEquals(FixedWindows.of(org.joda.time.Duration.standardHours(1)), desc.getWindowFn());
  assertEquals(DefaultTrigger.of(), desc.getTrigger());
  assertEquals(AccumulationMode.DISCARDING_FIRED_PANES, desc.getAccumulationMode());
}
 

@Test
public void testWindow_applyIfNot() {
  final PCollection<String> dataset = TestUtils.createMockDataset(TypeDescriptors.strings());
  final PCollection<KV<String, Long>> reduced =
      ReduceByKey.of(dataset)
          .keyBy(s -> s)
          .valueBy(s -> 1L)
          .combineBy(Sums.ofLongs())
          .applyIf(
              false,
              b ->
                  b.windowBy(FixedWindows.of(Duration.standardHours(1)))
                      .triggeredBy(DefaultTrigger.of())
                      .accumulationMode(AccumulationMode.DISCARDING_FIRED_PANES))
          .output();
  final ReduceByKey reduce = (ReduceByKey) TestUtils.getProducer(reduced);
  assertFalse(reduce.getWindow().isPresent());
}
 

@Test
public void testWindow_applyIf() {
  final PCollection<String> dataset = TestUtils.createMockDataset(TypeDescriptors.strings());
  final PCollection<KV<String, Long>> reduced =
      ReduceByKey.of(dataset)
          .keyBy(s -> s)
          .valueBy(s -> 1L)
          .combineBy(Sums.ofLongs())
          .applyIf(
              true,
              b ->
                  b.windowBy(FixedWindows.of(Duration.standardHours(1)))
                      .triggeredBy(DefaultTrigger.of())
                      .accumulationMode(AccumulationMode.DISCARDING_FIRED_PANES))
          .output();
  final ReduceByKey reduce = (ReduceByKey) TestUtils.getProducer(reduced);
  assertTrue(reduce.getWindow().isPresent());
  @SuppressWarnings("unchecked")
  final Window<? extends BoundedWindow> window = (Window) reduce.getWindow().get();
  assertEquals(FixedWindows.of(org.joda.time.Duration.standardHours(1)), window.getWindowFn());
  assertEquals(DefaultTrigger.of(), WindowDesc.of(window).getTrigger());
  assertSame(
      AccumulationMode.DISCARDING_FIRED_PANES, WindowDesc.of(window).getAccumulationMode());
}
 

@Test
public void testBuild_sortedValues() {
  final PCollection<String> dataset = TestUtils.createMockDataset(TypeDescriptors.strings());
  final PCollection<KV<String, List<Long>>> reduced =
      ReduceByKey.of(dataset)
          .keyBy(s -> s)
          .valueBy(s -> 1L)
          .reduceBy(s -> s.collect(Collectors.toList()))
          .withSortedValues(Long::compare)
          .windowBy(FixedWindows.of(Duration.standardHours(1)))
          .triggeredBy(DefaultTrigger.of())
          .accumulationMode(AccumulationMode.DISCARDING_FIRED_PANES)
          .output();
  final ReduceByKey reduce = (ReduceByKey) TestUtils.getProducer(reduced);
  assertTrue(reduce.getValueComparator().isPresent());
}
 

@Test
public void testBuild_Windowing() {
  final PCollection<String> dataset = TestUtils.createMockDataset(TypeDescriptors.strings());
  final PCollection<KV<String, Long>> reduced =
      ReduceByKey.of(dataset)
          .keyBy(s -> s)
          .valueBy(s -> 1L)
          .combineBy(Sums.ofLongs())
          .windowBy(FixedWindows.of(Duration.standardHours(1)))
          .triggeredBy(DefaultTrigger.of())
          .accumulationMode(AccumulationMode.DISCARDING_FIRED_PANES)
          .output();

  final ReduceByKey reduce = (ReduceByKey) TestUtils.getProducer(reduced);

  assertTrue(reduce.getWindow().isPresent());
  @SuppressWarnings("unchecked")
  final Window<? extends BoundedWindow> window = (Window) reduce.getWindow().get();
  assertEquals(FixedWindows.of(org.joda.time.Duration.standardHours(1)), window.getWindowFn());
  assertEquals(DefaultTrigger.of(), WindowDesc.of(window).getTrigger());
  assertSame(
      AccumulationMode.DISCARDING_FIRED_PANES, WindowDesc.of(window).getAccumulationMode());
  assertFalse(reduce.getValueComparator().isPresent());
}
 

@Test
public void testWindow_applyIf() {
  final PCollection<String> dataset = TestUtils.createMockDataset(TypeDescriptors.strings());
  final PCollection<Long> output =
      ReduceWindow.of(dataset)
          .reduceBy(e -> 1L)
          .withSortedValues(String::compareTo)
          .applyIf(
              true,
              b ->
                  b.windowBy(FixedWindows.of(org.joda.time.Duration.standardHours(1)))
                      .triggeredBy(DefaultTrigger.of())
                      .discardingFiredPanes())
          .output();
  final ReduceWindow rw = (ReduceWindow) TestUtils.getProducer(output);
  assertTrue(rw.getWindow().isPresent());
  @SuppressWarnings("unchecked")
  final WindowDesc<?> windowDesc = WindowDesc.of((Window) rw.getWindow().get());
  assertEquals(
      FixedWindows.of(org.joda.time.Duration.standardHours(1)), windowDesc.getWindowFn());
  assertEquals(DefaultTrigger.of(), windowDesc.getTrigger());
  assertEquals(AccumulationMode.DISCARDING_FIRED_PANES, windowDesc.getAccumulationMode());
}
 

@Test
public void testBuild() {
  final PCollection<String> dataset = TestUtils.createMockDataset(TypeDescriptors.strings());
  final FixedWindows windowing = FixedWindows.of(org.joda.time.Duration.standardHours(1));
  final DefaultTrigger trigger = DefaultTrigger.of();
  final PCollection<KV<String, Long>> counted =
      CountByKey.named("CountByKey1")
          .of(dataset)
          .keyBy(s -> s)
          .windowBy(windowing)
          .triggeredBy(trigger)
          .discardingFiredPanes()
          .withAllowedLateness(Duration.millis(1000))
          .output();
  final CountByKey count = (CountByKey) TestUtils.getProducer(counted);
  assertTrue(count.getName().isPresent());
  assertEquals("CountByKey1", count.getName().get());
  assertNotNull(count.getKeyExtractor());
  assertTrue(count.getWindow().isPresent());
  final WindowDesc<?> desc = WindowDesc.of((Window<?>) count.getWindow().get());
  assertEquals(windowing, desc.getWindowFn());
  assertEquals(trigger, desc.getTrigger());
  assertEquals(AccumulationMode.DISCARDING_FIRED_PANES, desc.getAccumulationMode());
  assertEquals(Duration.millis(1000), desc.getAllowedLateness());
}
 

@Test
public void testSumByKey() {
  execute(
      new AbstractTestCase<Integer, KV<Integer, Long>>() {
        @Override
        protected PCollection<KV<Integer, Long>> getOutput(PCollection<Integer> input) {
          final PCollection<Integer> inputWithTime =
              AssignEventTime.of(input).using(i -> 0).output();
          return SumByKey.of(inputWithTime)
              .keyBy(e -> e % 2)
              .valueBy(e -> (long) e)
              .windowBy(FixedWindows.of(org.joda.time.Duration.standardHours(1)))
              .triggeredBy(DefaultTrigger.of())
              .discardingFiredPanes()
              .output();
        }

        @Override
        protected List<Integer> getInput() {
          return Arrays.asList(1, 2, 3, 4, 5, 6, 7, 8, 9);
        }

        @Override
        protected TypeDescriptor<Integer> getInputType() {
          return null;
        }

        @Override
        public List<KV<Integer, Long>> getUnorderedOutput() {
          return Arrays.asList(KV.of(0, 20L), KV.of(1, 25L));
        }
      });
}
 

public static <T, W extends BoundedWindow> WindowingStrategy<T, W> of(WindowFn<T, W> windowFn) {
  return new WindowingStrategy<>(
      windowFn,
      DefaultTrigger.of(),
      false,
      AccumulationMode.DISCARDING_FIRED_PANES,
      false,
      DEFAULT_ALLOWED_LATENESS,
      false,
      TimestampCombiner.END_OF_WINDOW,
      false,
      ClosingBehavior.FIRE_IF_NON_EMPTY,
      OnTimeBehavior.FIRE_ALWAYS);
}
 

private static <T, W extends BoundedWindow> void validateWindowStrategy(
    WindowingStrategy<T, W> strategy) {
  if (!strategy.getWindowFn().isNonMerging()
      && (!strategy.getTrigger().getClass().equals(DefaultTrigger.class)
          || strategy.getAllowedLateness().isLongerThan(Duration.ZERO))) {
    throw new UnsupportedOperationException(
        String.format(
            "%s does not support merging windowing strategies, except when using the default "
                + "trigger and zero allowed lateness.",
            Distinct.class.getSimpleName()));
  }
}
 

public static void applicableTo(PCollection<?> input) {
  WindowingStrategy<?, ?> windowingStrategy = input.getWindowingStrategy();
  // Verify that the input PCollection is bounded, or that there is windowing/triggering being
  // used. Without this, the watermark (at end of global window) will never be reached.
  if (windowingStrategy.getWindowFn() instanceof GlobalWindows
      && windowingStrategy.getTrigger() instanceof DefaultTrigger
      && input.isBounded() != IsBounded.BOUNDED) {
    throw new IllegalStateException(
        "GroupByKey cannot be applied to non-bounded PCollection in the GlobalWindow without a"
            + " trigger. Use a Window.into or Window.triggering transform prior to GroupByKey.");
  }

  // Validate the window merge function.
  if (windowingStrategy.getWindowFn() instanceof InvalidWindows) {
    String cause = ((InvalidWindows<?>) windowingStrategy.getWindowFn()).getCause();
    throw new IllegalStateException(
        "GroupByKey must have a valid Window merge function.  " + "Invalid because: " + cause);
  }

  // Validate that the trigger does not finish before garbage collection time
  if (!triggerIsSafe(windowingStrategy)) {
    throw new IllegalArgumentException(
        String.format(
            "Unsafe trigger '%s' may lose data, did you mean to wrap it in"
                + "`Repeatedly.forever(...)`?%nSee "
                + "https://s.apache.org/finishing-triggers-drop-data "
                + "for details.",
            windowingStrategy.getTrigger()));
  }
}
 

@Test
public void testUnsupportedGlobalWindowWithDefaultTrigger() {
  exceptions.expect(UnsupportedOperationException.class);

  pipeline.enableAbandonedNodeEnforcement(false);

  PCollection<Row> input =
      unboundedInput1.apply(
          "unboundedInput1.globalWindow",
          Window.<Row>into(new GlobalWindows()).triggering(DefaultTrigger.of()));

  String sql = "SELECT f_int2, COUNT(*) AS `size` FROM PCOLLECTION GROUP BY f_int2";

  input.apply("testUnsupportedGlobalWindows", SqlTransform.query(sql));
}
 

/**
 * Performs the same check as {@link GroupByKey}, provides more context in exception.
 *
 * <p>Verifies that the input PCollection is bounded, or that there is windowing/triggering
 * being used. Without this, the watermark (at end of global window) will never be reached.
 *
 * <p>Throws {@link UnsupportedOperationException} if validation fails.
 */
private void validateWindowIsSupported(PCollection<Row> upstream) {
  WindowingStrategy<?, ?> windowingStrategy = upstream.getWindowingStrategy();
  if (windowingStrategy.getWindowFn() instanceof GlobalWindows
      && windowingStrategy.getTrigger() instanceof DefaultTrigger
      && upstream.isBounded() != BOUNDED) {

    throw new UnsupportedOperationException(
        "Please explicitly specify windowing in SQL query using HOP/TUMBLE/SESSION functions "
            + "(default trigger will be used in this case). "
            + "Unbounded input with global windowing and default trigger is not supported "
            + "in Beam SQL aggregations. "
            + "See GroupByKey section in Beam Programming Guide");
  }
}
 

private boolean triggersOncePerWindow(WindowingStrategy windowingStrategy) {
  Trigger trigger = windowingStrategy.getTrigger();

  return !(windowingStrategy.getWindowFn() instanceof GlobalWindows)
      && trigger instanceof DefaultTrigger
      && ZERO.equals(windowingStrategy.getAllowedLateness());
}
 

@Test
public void testSimpleDuplicatesWithStreamStrategyNewest() {
  execute(
      new AbstractTestCase<KV<String, Long>, String>() {

        @Override
        public List<String> getUnorderedOutput() {
          return Arrays.asList("2.", "1.", "3.");
        }

        @Override
        protected PCollection<String> getOutput(PCollection<KV<String, Long>> input) {
          input = AssignEventTime.of(input).using(KV::getValue).output();
          PCollection<KV<String, Long>> distinct =
              Distinct.of(input)
                  .projected(
                      in -> in.getKey().substring(0, 1),
                      Distinct.SelectionPolicy.NEWEST,
                      TypeDescriptors.strings())
                  .windowBy(FixedWindows.of(org.joda.time.Duration.standardSeconds(1)))
                  .triggeredBy(DefaultTrigger.of())
                  .discardingFiredPanes()
                  .output();
          return MapElements.of(distinct).using(KV::getKey).output();
        }

        @Override
        protected List<KV<String, Long>> getInput() {
          return asTimedList(100, "1", "2", "3", "3.", "2.", "1.");
        }

        @Override
        protected TypeDescriptor<KV<String, Long>> getInputType() {
          return TypeDescriptors.kvs(TypeDescriptors.strings(), TypeDescriptors.longs());
        }
      });
}
 

@Test
public void testSimpleDuplicatesWithStreamStrategyOldest() {
  execute(
      new AbstractTestCase<KV<String, Long>, String>() {

        @Override
        public List<String> getUnorderedOutput() {
          return Arrays.asList("2", "1", "3");
        }

        @Override
        protected PCollection<String> getOutput(PCollection<KV<String, Long>> input) {
          input = AssignEventTime.of(input).using(KV::getValue).output();
          PCollection<KV<String, Long>> distinct =
              Distinct.of(input)
                  .projected(
                      in -> in.getKey().substring(0, 1),
                      Distinct.SelectionPolicy.OLDEST,
                      TypeDescriptors.strings())
                  .windowBy(FixedWindows.of(org.joda.time.Duration.standardSeconds(1)))
                  .triggeredBy(DefaultTrigger.of())
                  .discardingFiredPanes()
                  .output();
          return MapElements.of(distinct).using(KV::getKey).output();
        }

        @Override
        protected List<KV<String, Long>> getInput() {
          return asTimedList(100, "1", "2", "3", "3.", "2.", "1.");
        }

        @Override
        protected TypeDescriptor<KV<String, Long>> getInputType() {
          return TypeDescriptors.kvs(TypeDescriptors.strings(), TypeDescriptors.longs());
        }
      });
}
 

@Test
public void testSimpleDuplicatesWithStream() {
  execute(
      new AbstractTestCase<KV<Integer, Long>, Integer>() {

        @Override
        public List<Integer> getUnorderedOutput() {
          return Arrays.asList(2, 1, 3);
        }

        @Override
        protected PCollection<Integer> getOutput(PCollection<KV<Integer, Long>> input) {
          input = AssignEventTime.of(input).using(KV::getValue).output();
          PCollection<KV<Integer, Long>> distinct =
              Distinct.of(input)
                  .projected(KV::getKey)
                  .windowBy(FixedWindows.of(org.joda.time.Duration.standardSeconds(1)))
                  .triggeredBy(DefaultTrigger.of())
                  .discardingFiredPanes()
                  .output();
          return MapElements.of(distinct).using(KV::getKey).output();
        }

        @Override
        protected List<KV<Integer, Long>> getInput() {
          List<KV<Integer, Long>> first = asTimedList(100, 1, 2, 3, 3, 2, 1);
          first.addAll(asTimedList(100, 1, 2, 3, 3, 2, 1));
          return first;
        }

        @Override
        protected TypeDescriptor<KV<Integer, Long>> getInputType() {
          return TypeDescriptors.kvs(TypeDescriptors.integers(), TypeDescriptors.longs());
        }
      });
}
 

/**
 * Tests that with the default trigger we will not produce two ON_TIME panes, even if there are
 * two outputs that are both candidates.
 */
@Test
public void testOnlyOneOnTimePane() throws Exception {
  WindowingStrategy<?, IntervalWindow> strategy =
      WindowingStrategy.of((WindowFn<?, IntervalWindow>) FixedWindows.of(Duration.millis(10)))
          .withTrigger(DefaultTrigger.of())
          .withMode(AccumulationMode.ACCUMULATING_FIRED_PANES)
          .withAllowedLateness(Duration.millis(100));

  ReduceFnTester<Integer, Integer, IntervalWindow> tester =
      ReduceFnTester.combining(strategy, Sum.ofIntegers(), VarIntCoder.of());

  tester.advanceInputWatermark(new Instant(0));

  int value1 = 1;
  int value2 = 3;

  // A single element that should be in the ON_TIME output
  tester.injectElements(TimestampedValue.of(value1, new Instant(1)));

  // Should fire ON_TIME
  tester.advanceInputWatermark(new Instant(10));

  // The DefaultTrigger should cause output labeled LATE, even though it does not have to be
  // labeled as such.
  tester.injectElements(TimestampedValue.of(value2, new Instant(3)));

  List<WindowedValue<Integer>> output = tester.extractOutput();
  assertEquals(2, output.size());

  assertThat(output.get(0), isWindowedValue(equalTo(value1)));
  assertThat(output.get(1), isWindowedValue(equalTo(value1 + value2)));

  assertThat(
      output.get(0),
      WindowMatchers.valueWithPaneInfo(PaneInfo.createPane(true, false, Timing.ON_TIME, 0, 0)));
  assertThat(
      output.get(1),
      WindowMatchers.valueWithPaneInfo(PaneInfo.createPane(false, false, Timing.LATE, 1, 1)));
}
 

private <K, V> void groupByKeyHelper(
    GroupByKey<K, V> transform, TranslationContext context) {
  StepTranslationContext stepContext = context.addStep(transform, "GroupByKey");
  PCollection<KV<K, V>> input = context.getInput(transform);
  stepContext.addInput(PropertyNames.PARALLEL_INPUT, input);
  stepContext.addOutput(PropertyNames.OUTPUT, context.getOutput(transform));

  WindowingStrategy<?, ?> windowingStrategy = input.getWindowingStrategy();
  boolean isStreaming =
      context.getPipelineOptions().as(StreamingOptions.class).isStreaming();
  boolean allowCombinerLifting =
      windowingStrategy.getWindowFn().isNonMerging()
          && windowingStrategy.getWindowFn().assignsToOneWindow();
  if (isStreaming) {
    allowCombinerLifting &= transform.fewKeys();
    // TODO: Allow combiner lifting on the non-default trigger, as appropriate.
    allowCombinerLifting &= (windowingStrategy.getTrigger() instanceof DefaultTrigger);
  }
  stepContext.addInput(PropertyNames.DISALLOW_COMBINER_LIFTING, !allowCombinerLifting);
  stepContext.addInput(
      PropertyNames.SERIALIZED_FN,
      byteArrayToJsonString(
          serializeWindowingStrategy(windowingStrategy, context.getPipelineOptions())));
  stepContext.addInput(
      PropertyNames.IS_MERGING_WINDOW_FN,
      !windowingStrategy.getWindowFn().isNonMerging());
}
 

@Test
public void testReduceWithAttachedWindowingMoreWindows() {
  execute(
      new AbstractTestCase<Integer, Integer>() {
        @Override
        protected PCollection<Integer> getOutput(PCollection<Integer> input) {
          PCollection<Integer> withEventTime =
              AssignEventTime.of(input).using(i -> 1000L * i).output();

          PCollection<Integer> first =
              ReduceWindow.named("first-reduce")
                  .of(withEventTime)
                  .combineBy(Sums.ofInts())
                  .windowBy(FixedWindows.of(org.joda.time.Duration.standardSeconds(5)))
                  .triggeredBy(DefaultTrigger.of())
                  .discardingFiredPanes()
                  .output();

          return ReduceWindow.named("second-reduce").of(first).combineBy(Sums.ofInts()).output();
        }

        @Override
        protected List<Integer> getInput() {
          return Arrays.asList(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 100);
        }

        @Override
        protected TypeDescriptor<Integer> getInputType() {
          return TypeDescriptors.integers();
        }

        @Override
        public List<Integer> getUnorderedOutput() {
          return Arrays.asList(10, 35, 10, 100);
        }
      });
}