Java Code Examples for org.apache.beam.sdk.transforms.windowing.PaneInfo#getNonSpeculativeIndex()

The following examples show how to use org.apache.beam.sdk.transforms.windowing.PaneInfo#getNonSpeculativeIndex() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example 1

Source File: StateAndTimers.java From streamingbook with Apache License 2.0

6 votes

@ProcessElement
 public void processElement(@Element Attribution attribution,
@Timestamp Instant timestamp,
BoundedWindow window,
PaneInfo pane,
OutputReceiver<String> output) {
     StringBuilder builder = new StringBuilder(String.format(
         "%s: %s %s %-7s", Utils.formatWindow(window), attribution,
         Utils.formatTime(timestamp), pane.getTiming()));
     if (pane.getTiming() != PaneInfo.Timing.UNKNOWN) {
         builder.append(String.format(" index=%d", pane.getIndex()));
         if (pane.getNonSpeculativeIndex() > -1)
             builder.append(" onTimeIndex=" + pane.getNonSpeculativeIndex());
         if (pane.isFirst())
             builder.append(" isFirst");
         if (pane.isLast())
             builder.append(" isLast");
     }
     output.output(builder.toString());
 }

Example 2

Source File: BeamModel.java From streamingbook with Apache License 2.0

6 votes

@ProcessElement
public void processElement(@Element KV<String, Integer> kv,
                           @Timestamp Instant timestamp,
                           BoundedWindow window,
                           PaneInfo pane,
                           OutputReceiver<String> output) {
    StringBuilder builder = new StringBuilder(String.format(
        "%s: %s:%-2d %s %-7s index=%d",
        Utils.formatWindow(window), kv.getKey(), kv.getValue(),
        Utils.formatTime(timestamp), pane.getTiming(), pane.getIndex()));
    if (pane.getNonSpeculativeIndex() > -1)
        builder.append(" onTimeIndex=" + pane.getNonSpeculativeIndex());
    if (pane.isFirst())
        builder.append(" isFirst");
    if (pane.isLast())
        builder.append(" isLast");
    output.output(builder.toString());
}

Example 3

Source File: PaneInfoTracker.java From beam with Apache License 2.0

4 votes

private <W> PaneInfo describePane(
    Object key, Instant windowMaxTimestamp, PaneInfo previousPane, boolean isFinal) {
  boolean isFirst = previousPane == null;
  Timing previousTiming = isFirst ? null : previousPane.getTiming();
  long index = isFirst ? 0 : previousPane.getIndex() + 1;
  long nonSpeculativeIndex = isFirst ? 0 : previousPane.getNonSpeculativeIndex() + 1;
  Instant outputWM = timerInternals.currentOutputWatermarkTime();
  Instant inputWM = timerInternals.currentInputWatermarkTime();

  // True if it is not possible to assign the element representing this pane a timestamp
  // which will make an ON_TIME pane for any following computation.
  // Ie true if the element's latest possible timestamp is before the current output watermark.
  boolean isLateForOutput = outputWM != null && windowMaxTimestamp.isBefore(outputWM);

  // True if all emitted panes (if any) were EARLY panes.
  // Once the ON_TIME pane has fired, all following panes must be considered LATE even
  // if the output watermark is behind the end of the window.
  boolean onlyEarlyPanesSoFar = previousTiming == null || previousTiming == Timing.EARLY;

  // True is the input watermark hasn't passed the window's max timestamp.
  boolean isEarlyForInput = !inputWM.isAfter(windowMaxTimestamp);

  Timing timing;
  if (isLateForOutput || !onlyEarlyPanesSoFar) {
    // The output watermark has already passed the end of this window, or we have already
    // emitted a non-EARLY pane. Irrespective of how this pane was triggered we must
    // consider this pane LATE.
    timing = Timing.LATE;
  } else if (isEarlyForInput) {
    // This is an EARLY firing.
    timing = Timing.EARLY;
    nonSpeculativeIndex = -1;
  } else {
    // This is the unique ON_TIME firing for the window.
    timing = Timing.ON_TIME;
  }

  WindowTracing.debug(
      "describePane: {} pane (prev was {}) for key:{}; windowMaxTimestamp:{}; "
          + "inputWatermark:{}; outputWatermark:{}; isLateForOutput:{}",
      timing,
      previousTiming,
      key,
      windowMaxTimestamp,
      inputWM,
      outputWM,
      isLateForOutput);

  if (previousPane != null) {
    // Timing transitions should follow EARLY* ON_TIME? LATE*
    switch (previousTiming) {
      case EARLY:
        checkState(
            timing == Timing.EARLY || timing == Timing.ON_TIME || timing == Timing.LATE,
            "EARLY cannot transition to %s",
            timing);
        break;
      case ON_TIME:
        checkState(timing == Timing.LATE, "ON_TIME cannot transition to %s", timing);
        break;
      case LATE:
        checkState(timing == Timing.LATE, "LATE cannot transtion to %s", timing);
        break;
      case UNKNOWN:
        break;
    }
    checkState(!previousPane.isLast(), "Last pane was not last after all.");
  }

  return PaneInfo.createPane(isFirst, isFinal, timing, index, nonSpeculativeIndex);
}