org.apache.beam.sdk.transforms.windowing.PaneInfo Java Examples

/** Construct an {@link OutputFile} for {@code pane} in {@code window} for {@code shard}. */
private OutputFile outputFileFor(BoundedWindow window, String shard, PaneInfo pane) {
  @Nullable
  String filename =
      outputPath == null
          ? null
          : String.format(
              "%s/LOG-%s-%s-%03d-%s-%x",
              outputPath,
              window.maxTimestamp(),
              shard,
              pane.getIndex(),
              timingToString(pane.getTiming()),
              ThreadLocalRandom.current().nextLong());
  return new OutputFile(
      window.maxTimestamp(), shard, pane.getIndex(), pane.getTiming(), filename);
}
 

@Override
public ResourceId windowedFilename(
    int shardNumber,
    int numShards,
    BoundedWindow window,
    PaneInfo paneInfo,
    OutputFileHints outputFileHints) {
  String filenamePrefix =
      outputFilePrefix.isDirectory() ? "" : firstNonNull(outputFilePrefix.getFilename(), "");

  IntervalWindow interval = (IntervalWindow) window;
  String windowStr =
      String.format("%s-%s", interval.start().toString(), interval.end().toString());
  String filename =
      String.format(
          "%s-%s-%s-of-%s-pane-%s%s%s.avro",
          filenamePrefix,
          windowStr,
          shardNumber,
          numShards,
          paneInfo.getIndex(),
          paneInfo.isLast() ? "-last" : "",
          outputFileHints.getSuggestedFilenameSuffix());
  return outputFilePrefix.getCurrentDirectory().resolve(filename, RESOLVE_FILE);
}
 

/**
 * Tests that if end-of-window and GC timers come in together, that the pane is correctly marked
 * as final.
 */
@Test
public void testCombiningAccumulatingEventTime() throws Exception {
  WindowingStrategy<?, IntervalWindow> strategy =
      WindowingStrategy.of((WindowFn<?, IntervalWindow>) FixedWindows.of(Duration.millis(100)))
          .withTimestampCombiner(TimestampCombiner.EARLIEST)
          .withMode(AccumulationMode.ACCUMULATING_FIRED_PANES)
          .withAllowedLateness(Duration.millis(1))
          .withTrigger(Repeatedly.forever(AfterWatermark.pastEndOfWindow()));

  ReduceFnTester<Integer, Integer, IntervalWindow> tester =
      ReduceFnTester.combining(strategy, Sum.ofIntegers(), VarIntCoder.of());

  injectElement(tester, 2); // processing timer @ 5000 + 10; EOW timer @ 100
  injectElement(tester, 5);

  tester.advanceInputWatermark(new Instant(1000));

  assertThat(
      tester.extractOutput(),
      contains(
          isSingleWindowedValue(
              equalTo(7), 2, 0, 100, PaneInfo.createPane(true, true, Timing.ON_TIME, 0, 0))));
}
 

/** Emit the current element from the given Reader. The reader is guaranteed to have data. */
private void emitElement(
    SourceContext<WindowedValue<ValueWithRecordId<OutputT>>> ctx,
    UnboundedSource.UnboundedReader<OutputT> reader) {
  // make sure that reader state update and element emission are atomic
  // with respect to snapshots
  OutputT item = reader.getCurrent();
  byte[] recordId = reader.getCurrentRecordId();
  Instant timestamp = reader.getCurrentTimestamp();

  WindowedValue<ValueWithRecordId<OutputT>> windowedValue =
      WindowedValue.of(
          new ValueWithRecordId<>(item, recordId),
          timestamp,
          GlobalWindow.INSTANCE,
          PaneInfo.NO_FIRING);
  ctx.collect(windowedValue);
}
 

@Test
public void mutationAfterAddCreateBundleThrows() {
  UncommittedBundle<byte[]> intermediate = factory.createBundle(transformed);

  byte[] array = new byte[] {4, 8, 12};
  WindowedValue<byte[]> windowedArray =
      WindowedValue.of(
          array,
          new Instant(891L),
          new IntervalWindow(new Instant(0), new Instant(1000)),
          PaneInfo.ON_TIME_AND_ONLY_FIRING);
  intermediate.add(windowedArray);

  array[2] = -3;
  thrown.expect(IllegalMutationException.class);
  thrown.expectMessage("Values must not be mutated in any way after being output");
  intermediate.commit(Instant.now());
}
 

/**
 * When the watermark passes the end-of-window and window expiration time in a single update, this
 * tests that it does not crash.
 */
@Test
public void testFixedWindowsEowAndGcTogetherFireIfNonEmpty() throws Exception {
  ReduceFnTester<Integer, Iterable<Integer>, IntervalWindow> tester =
      ReduceFnTester.nonCombining(
          FixedWindows.of(Duration.millis(10)),
          DefaultTriggerStateMachine.of(),
          AccumulationMode.ACCUMULATING_FIRED_PANES,
          Duration.millis(50),
          ClosingBehavior.FIRE_IF_NON_EMPTY);

  tester.setAutoAdvanceOutputWatermark(true);

  tester.advanceInputWatermark(new Instant(0));
  injectElement(tester, 1);
  tester.advanceInputWatermark(new Instant(100));

  List<WindowedValue<Iterable<Integer>>> output = tester.extractOutput();
  assertThat(
      output,
      contains(
          isSingleWindowedValue(
              contains(1), 1, 0, 10, PaneInfo.createPane(true, true, Timing.ON_TIME))));
}
 

private <V> void addElement(
    InputMessageBundle.Builder messageBundle,
    Collection<IntervalWindow> windows,
    Instant timestamp,
    Coder<V> valueCoder,
    V value)
    throws IOException {
  @SuppressWarnings({"unchecked", "rawtypes"})
  Coder<Collection<? extends BoundedWindow>> windowsCoder =
      (Coder) CollectionCoder.of(windowCoder);

  ByteString.Output dataOutput = ByteString.newOutput();
  valueCoder.encode(value, dataOutput, Context.OUTER);
  messageBundle
      .addMessagesBuilder()
      .setMetadata(WindmillSink.encodeMetadata(windowsCoder, windows, PaneInfo.NO_FIRING))
      .setData(dataOutput.toByteString())
      .setTimestamp(WindmillTimeUtils.harnessToWindmillTimestamp(timestamp));
}
 

@Override
public void encode(WindowedAccumulator<InputT, ValueT, AccumT, ?> value, OutputStream outStream)
    throws CoderException, IOException {
  if (type.isMapBased()) {
    wrap.encode(((MapBasedWindowedAccumulator<?, ?, AccumT, ?>) value).map.values(), outStream);
  } else {
    SingleWindowWindowedAccumulator<?, ?, AccumT> swwa =
        (SingleWindowWindowedAccumulator<?, ?, AccumT>) value;
    if (swwa.isEmpty()) {
      outStream.write(0);
    } else {
      outStream.write(1);
      accumCoder.encode(
          WindowedValue.of(
              swwa.windowAccumulator, swwa.accTimestamp, swwa.accWindow, PaneInfo.NO_FIRING),
          outStream);
    }
  }
}
 

@Test
public void testIsWindowedValueReorderedWindows() {
  long timestamp = 100;
  long windowStart = 0;
  long windowEnd = 200;
  long windowStart2 = 50;
  long windowEnd2 = 150;

  assertThat(
      WindowedValue.of(
          "hello",
          new Instant(timestamp),
          ImmutableList.of(
              new IntervalWindow(new Instant(windowStart), new Instant(windowEnd)),
              new IntervalWindow(new Instant(windowStart2), new Instant(windowEnd2))),
          PaneInfo.NO_FIRING),
      WindowMatchers.isWindowedValue(
          "hello",
          new Instant(timestamp),
          ImmutableList.of(
              new IntervalWindow(new Instant(windowStart), new Instant(windowEnd)),
              new IntervalWindow(new Instant(windowStart2), new Instant(windowEnd2))),
          PaneInfo.NO_FIRING));
}
 

@Override
public ResourceId windowedFilename(
    int shardNumber,
    int numShards,
    BoundedWindow window,
    PaneInfo paneInfo,
    OutputFileHints outputFileHints) {
  IntervalWindow intervalWindow = (IntervalWindow) window;
  String filename =
      String.format(
          "%s-%s-of-%s%s",
          filenamePrefixForWindow(intervalWindow),
          shardNumber,
          numShards,
          outputFileHints.getSuggestedFilenameSuffix());
  return baseFilename
      .getCurrentDirectory()
      .resolve(filename, StandardResolveOptions.RESOLVE_FILE);
}
 

FnApiTimerMap(
    String timerFamilyId,
    K userKey,
    BoundedWindow boundedWindow,
    Instant elementTimestampOrTimerHoldTimestamp,
    Instant elementTimestampOrTimerFireTimestamp,
    PaneInfo paneInfo) {
  this.timerFamilyId = timerFamilyId;
  this.userKey = userKey;
  this.elementTimestampOrTimerHoldTimestamp = elementTimestampOrTimerHoldTimestamp;
  this.elementTimestampOrTimerFireTimestamp = elementTimestampOrTimerFireTimestamp;
  this.boundedWindow = boundedWindow;
  this.paneInfo = paneInfo;
  this.timeDomain =
      translateTimeDomain(
          parDoPayload.getTimerFamilySpecsMap().get(timerFamilyId).getTimeDomain());
}
 

public long insertAll(
    TableReference ref, List<TableRow> rowList, @Nullable List<String> insertIdList)
    throws IOException, InterruptedException {
  List<ValueInSingleWindow<TableRow>> windowedRows = Lists.newArrayList();
  for (TableRow row : rowList) {
    windowedRows.add(
        ValueInSingleWindow.of(
            row,
            GlobalWindow.TIMESTAMP_MAX_VALUE,
            GlobalWindow.INSTANCE,
            PaneInfo.ON_TIME_AND_ONLY_FIRING));
  }
  return insertAll(
      ref,
      windowedRows,
      insertIdList,
      InsertRetryPolicy.alwaysRetry(),
      null,
      null,
      false,
      false,
      false);
}
 

@Test
public void processElementSideInputReadyAllWindows() {
  when(reader.isReady(Mockito.eq(singletonView), Mockito.any(BoundedWindow.class)))
      .thenReturn(true);

  ImmutableList<PCollectionView<?>> views = ImmutableList.of(singletonView);
  SimplePushbackSideInputDoFnRunner<Integer, Integer> runner = createRunner(views);

  WindowedValue<Integer> multiWindow =
      WindowedValue.of(
          2,
          new Instant(-2),
          ImmutableList.of(
              new IntervalWindow(new Instant(-500L), new Instant(0L)),
              new IntervalWindow(BoundedWindow.TIMESTAMP_MIN_VALUE, new Instant(250L)),
              GlobalWindow.INSTANCE),
          PaneInfo.ON_TIME_AND_ONLY_FIRING);
  Iterable<WindowedValue<Integer>> multiWindowPushback =
      runner.processElementInReadyWindows(multiWindow);
  assertThat(multiWindowPushback, emptyIterable());
  assertThat(
      underlying.inputElems,
      containsInAnyOrder(ImmutableList.copyOf(multiWindow.explodeWindows()).toArray()));
}
 

@Override
public PCollection<ValueInSingleWindow<T>> expand(PCollection<T> input) {
  return input
      .apply(
          ParDo.of(
              new DoFn<T, ValueInSingleWindow<T>>() {
                @ProcessElement
                public void processElement(
                    @Element T element,
                    @Timestamp Instant timestamp,
                    BoundedWindow window,
                    PaneInfo pane,
                    OutputReceiver<ValueInSingleWindow<T>> r) {
                  r.outputWithTimestamp(
                      ValueInSingleWindow.of(element, timestamp, window, pane), timestamp);
                }
              }))
      .setCoder(
          ValueInSingleWindow.Coder.of(
              input.getCoder(), input.getWindowingStrategy().getWindowFn().windowCoder()));
}
 

@Test
public void allPanesMultiplePanes() {
  SerializableFunction<Iterable<ValueInSingleWindow<Integer>>, Iterable<Integer>> extractor =
      PaneExtractors.allPanes();
  Iterable<ValueInSingleWindow<Integer>> onlyOnTime =
      ImmutableList.of(
          ValueInSingleWindow.of(
              8,
              new Instant(0L),
              GlobalWindow.INSTANCE,
              PaneInfo.createPane(false, false, Timing.LATE, 2L, 1L)),
          ValueInSingleWindow.of(
              4,
              new Instant(0L),
              GlobalWindow.INSTANCE,
              PaneInfo.createPane(false, false, Timing.ON_TIME, 1L, 0L)),
          ValueInSingleWindow.of(
              1,
              new Instant(0L),
              GlobalWindow.INSTANCE,
              PaneInfo.createPane(true, false, Timing.EARLY)));

  assertThat(extractor.apply(onlyOnTime), containsInAnyOrder(4, 8, 1));
}
 

/**
 * Tests that for a simple sequence of elements on the same key, the given GABW implementation
 * correctly groups them according to fixed windows.
 */
public static void groupsElementsIntoFixedWindows(
    GroupAlsoByWindowDoFnFactory<String, String, Iterable<String>> gabwFactory) throws Exception {

  WindowingStrategy<?, IntervalWindow> windowingStrategy =
      WindowingStrategy.of(FixedWindows.of(Duration.millis(10)));

  List<WindowedValue<KV<String, Iterable<String>>>> result =
      runGABW(
          gabwFactory,
          windowingStrategy,
          "key",
          WindowedValue.of(
              "v1", new Instant(1), Arrays.asList(window(0, 10)), PaneInfo.NO_FIRING),
          WindowedValue.of(
              "v2", new Instant(2), Arrays.asList(window(0, 10)), PaneInfo.NO_FIRING),
          WindowedValue.of(
              "v3", new Instant(13), Arrays.asList(window(10, 20)), PaneInfo.NO_FIRING));

  assertThat(result, hasSize(2));

  TimestampedValue<KV<String, Iterable<String>>> item0 =
      getOnlyElementInWindow(result, window(0, 10));
  assertThat(item0.getValue().getValue(), containsInAnyOrder("v1", "v2"));
  assertThat(item0.getTimestamp(), equalTo(window(0, 10).maxTimestamp()));

  TimestampedValue<KV<String, Iterable<String>>> item1 =
      getOnlyElementInWindow(result, window(10, 20));
  assertThat(item1.getValue().getValue(), contains("v3"));
  assertThat(item1.getTimestamp(), equalTo(window(10, 20).maxTimestamp()));
}
 

@Override
public void outputWindowedValue(
    OutputT output,
    Instant timestamp,
    Collection<? extends BoundedWindow> windows,
    PaneInfo pane) {
  outputWindowedValue(tester.getMainOutputTag(), output, timestamp, windows, pane);
}
 

@Test
public void testBasicDoFnAllParameters() throws Exception {
  DoFnSignature sig =
      DoFnSignatures.getSignature(
          new DoFn<String, String>() {
            @ProcessElement
            public void process(
                @Element String element,
                @Timestamp Instant timestamp,
                BoundedWindow window,
                PaneInfo paneInfo,
                OutputReceiver<String> receiver,
                PipelineOptions options,
                @SideInput("tag1") String input1,
                @SideInput("tag2") Integer input2,
                BundleFinalizer bundleFinalizer) {}
          }.getClass());

  assertThat(sig.processElement().extraParameters().size(), equalTo(9));
  assertThat(sig.processElement().extraParameters().get(0), instanceOf(ElementParameter.class));
  assertThat(sig.processElement().extraParameters().get(1), instanceOf(TimestampParameter.class));
  assertThat(sig.processElement().extraParameters().get(2), instanceOf(WindowParameter.class));
  assertThat(sig.processElement().extraParameters().get(3), instanceOf(PaneInfoParameter.class));
  assertThat(
      sig.processElement().extraParameters().get(4), instanceOf(OutputReceiverParameter.class));
  assertThat(
      sig.processElement().extraParameters().get(5), instanceOf(PipelineOptionsParameter.class));
  assertThat(sig.processElement().extraParameters().get(6), instanceOf(SideInputParameter.class));
  assertThat(sig.processElement().extraParameters().get(7), instanceOf(SideInputParameter.class));
  assertThat(
      sig.processElement().extraParameters().get(8), instanceOf(BundleFinalizerParameter.class));
}
 

/**
 * It is possible for a session window's trigger to be closed at the point at which the (merged)
 * session window is garbage collected. Make sure we don't accidentally assume the window is still
 * active.
 */
@Test
public void testMergingWithCloseBeforeGC() throws Exception {
  ReduceFnTester<Integer, Iterable<Integer>, IntervalWindow> tester =
      ReduceFnTester.nonCombining(
          Sessions.withGapDuration(Duration.millis(10)),
          mockTriggerStateMachine,
          AccumulationMode.DISCARDING_FIRED_PANES,
          Duration.millis(50),
          ClosingBehavior.FIRE_IF_NON_EMPTY);

  // Two elements in two overlapping session windows.
  tester.injectElements(
      TimestampedValue.of(1, new Instant(1)), // in [1, 11)
      TimestampedValue.of(10, new Instant(10))); // in [10, 20)

  // Close the trigger, but the gargbage collection timer is still pending.
  when(mockTriggerStateMachine.shouldFire(anyTriggerContext())).thenReturn(true);
  triggerShouldFinish(mockTriggerStateMachine);
  tester.advanceInputWatermark(new Instant(30));

  // Now the garbage collection timer will fire, finding the trigger already closed.
  tester.advanceInputWatermark(new Instant(100));

  List<WindowedValue<Iterable<Integer>>> output = tester.extractOutput();
  assertThat(output.size(), equalTo(1));
  assertThat(
      output.get(0),
      isSingleWindowedValue(
          containsInAnyOrder(1, 10),
          1, // timestamp
          1, // window start
          20)); // window end
  assertThat(
      output.get(0).getPane(), equalTo(PaneInfo.createPane(true, true, Timing.ON_TIME, 0, 0)));
}
 

@Experimental(Kind.FILESYSTEM)
public FileResult(
    ResourceId tempFilename,
    int shard,
    BoundedWindow window,
    PaneInfo paneInfo,
    DestinationT destination) {
  checkArgument(window != null, "window can not be null");
  checkArgument(paneInfo != null, "paneInfo can not be null");
  this.tempFilename = tempFilename;
  this.shard = shard;
  this.window = window;
  this.paneInfo = paneInfo;
  this.destination = destination;
}
 

@Override
public void outputWindowedValue(
    KV<K, Iterable<V>> output,
    Instant timestamp,
    Collection<? extends BoundedWindow> windows,
    PaneInfo pane) {
  outputs.add(WindowedValue.of(output, timestamp, windows, pane));
}
 

@Override
public void encode(ValueInSingleWindow<T> windowedElem, OutputStream outStream, Context context)
    throws IOException {
  InstantCoder.of().encode(windowedElem.getTimestamp(), outStream);
  windowCoder.encode(windowedElem.getWindow(), outStream);
  PaneInfo.PaneInfoCoder.INSTANCE.encode(windowedElem.getPane(), outStream);
  valueCoder.encode(windowedElem.getValue(), outStream, context);
}
 

@Override
public WindowedValue<T> nextRecord(WindowedValue<T> t) throws IOException {
  if (inputAvailable) {
    final T current = reader.getCurrent();
    final Instant timestamp = reader.getCurrentTimestamp();
    // advance reader to have a record ready next time
    inputAvailable = readerInvoker.invokeAdvance(reader);
    return WindowedValue.of(current, timestamp, GlobalWindow.INSTANCE, PaneInfo.NO_FIRING);
  }

  return null;
}
 

@Override
public void outputWindowedValue(
    KV<K, OutputT> output,
    Instant timestamp,
    Collection<? extends BoundedWindow> windows,
    PaneInfo pane) {
  this.output.add(WindowedValue.of(output, timestamp, windows, pane));
}
 

/**
 * Generates the filename. MUST use each argument and return different values for each
 * combination of the arguments.
 */
String getFilename(
    BoundedWindow window,
    PaneInfo pane,
    int numShards,
    int shardIndex,
    Compression compression);
 

private WindowedValue<String> createDatum(String element, long timestamp) {
  return WindowedValue.of(
      element,
      new Instant(timestamp),
      Arrays.asList(createWindow(timestamp)),
      PaneInfo.NO_FIRING);
}
 

@Override
public PCollection<KV<K, ValueInSingleWindow<V>>> expand(PCollection<KV<K, V>> input) {
  KvCoder<K, V> coder = (KvCoder<K, V>) input.getCoder();
  return input
      .apply(
          ParDo.of(
              new DoFn<KV<K, V>, KV<K, ValueInSingleWindow<V>>>() {
                @ProcessElement
                public void processElement(
                    @Element KV<K, V> element,
                    @Timestamp Instant timestamp,
                    BoundedWindow window,
                    PaneInfo pane,
                    OutputReceiver<KV<K, ValueInSingleWindow<V>>> r) {
                  r.output(
                      KV.of(
                          element.getKey(),
                          ValueInSingleWindow.of(element.getValue(), timestamp, window, pane)));
                }
              }))
      .setCoder(
          KvCoder.of(
              coder.getKeyCoder(),
              ValueInSingleWindow.Coder.of(
                  coder.getValueCoder(),
                  input.getWindowingStrategy().getWindowFn().windowCoder())));
}
 

/** Tests that the given GABW implementation correctly groups elements into merged sessions. */
public static void groupsElementsInMergedSessions(
    GroupAlsoByWindowDoFnFactory<String, String, Iterable<String>> gabwFactory) throws Exception {

  WindowingStrategy<?, IntervalWindow> windowingStrategy =
      WindowingStrategy.of(Sessions.withGapDuration(Duration.millis(10)));

  List<WindowedValue<KV<String, Iterable<String>>>> result =
      runGABW(
          gabwFactory,
          windowingStrategy,
          "key",
          WindowedValue.of(
              "v1", new Instant(0), Arrays.asList(window(0, 10)), PaneInfo.NO_FIRING),
          WindowedValue.of(
              "v2", new Instant(5), Arrays.asList(window(5, 15)), PaneInfo.NO_FIRING),
          WindowedValue.of(
              "v3", new Instant(15), Arrays.asList(window(15, 25)), PaneInfo.NO_FIRING));

  assertThat(result, hasSize(2));

  TimestampedValue<KV<String, Iterable<String>>> item0 =
      getOnlyElementInWindow(result, window(0, 15));
  assertThat(item0.getValue().getValue(), containsInAnyOrder("v1", "v2"));
  assertThat(item0.getTimestamp(), equalTo(window(0, 15).maxTimestamp()));

  TimestampedValue<KV<String, Iterable<String>>> item1 =
      getOnlyElementInWindow(result, window(15, 25));
  assertThat(item1.getValue().getValue(), contains("v3"));
  assertThat(item1.getTimestamp(), equalTo(window(15, 25).maxTimestamp()));
}
 

private String paneInfoToString(PaneInfo paneInfo) {
  String paneString = String.format("pane-%d", paneInfo.getIndex());
  if (paneInfo.getTiming() == Timing.LATE) {
    paneString = String.format("%s-late", paneString);
  }
  if (paneInfo.isLast()) {
    paneString = String.format("%s-last", paneString);
  }
  return paneString;
}
 

/**
 * Tests that for a simple sequence of elements on the same key, the given GABW implementation
 * correctly groups them according to fixed windows and also sets the output timestamp according
 * to the policy {@link TimestampCombiner#LATEST}.
 */
public static void groupsElementsIntoFixedWindowsWithLatestTimestamp(
    GroupAlsoByWindowDoFnFactory<String, String, Iterable<String>> gabwFactory) throws Exception {

  WindowingStrategy<?, IntervalWindow> windowingStrategy =
      WindowingStrategy.of(FixedWindows.of(Duration.millis(10)))
          .withTimestampCombiner(TimestampCombiner.LATEST);

  List<WindowedValue<KV<String, Iterable<String>>>> result =
      runGABW(
          gabwFactory,
          windowingStrategy,
          "k",
          WindowedValue.of(
              "v1", new Instant(1), Arrays.asList(window(0, 10)), PaneInfo.NO_FIRING),
          WindowedValue.of(
              "v2", new Instant(2), Arrays.asList(window(0, 10)), PaneInfo.NO_FIRING),
          WindowedValue.of(
              "v3", new Instant(13), Arrays.asList(window(10, 20)), PaneInfo.NO_FIRING));

  assertThat(result, hasSize(2));

  TimestampedValue<KV<String, Iterable<String>>> item0 =
      getOnlyElementInWindow(result, window(0, 10));
  assertThat(item0.getValue().getValue(), containsInAnyOrder("v1", "v2"));
  assertThat(item0.getTimestamp(), equalTo(new Instant(2)));

  TimestampedValue<KV<String, Iterable<String>>> item1 =
      getOnlyElementInWindow(result, window(10, 20));
  assertThat(item1.getValue().getValue(), contains("v3"));
  assertThat(item1.getTimestamp(), equalTo(new Instant(13)));
}