Java Code Examples for org.apache.beam.sdk.util.WindowedValue#explodeWindows()

@Override
public Iterable<WindowedValue<InputT>> processElementInReadyWindows(WindowedValue<InputT> elem) {
  if (views.isEmpty()) {
    // When there are no side inputs, we can preserve the compressed representation.
    underlying.processElement(elem);
    return Collections.emptyList();
  }
  ImmutableList.Builder<WindowedValue<InputT>> pushedBack = ImmutableList.builder();
  for (WindowedValue<InputT> windowElem : elem.explodeWindows()) {
    BoundedWindow mainInputWindow = Iterables.getOnlyElement(windowElem.getWindows());
    if (isReady(mainInputWindow)) {
      // When there are any side inputs, we have to process the element in each window
      // individually, to disambiguate access to per-window side inputs.
      underlying.processElement(windowElem);
    } else {
      notReadyWindows.add(mainInputWindow);
      pushedBack.add(windowElem);
    }
  }
  return pushedBack.build();
}
 

@Override
public void processElement(WindowedValue<InputT> input) {

  // StatefulDoFnRunner always observes windows, so we need to explode
  for (WindowedValue<InputT> value : input.explodeWindows()) {
    BoundedWindow window = value.getWindows().iterator().next();
    if (isLate(window)) {
      // The element is too late for this window.
      reportDroppedElement(value, window);
    } else if (requiresTimeSortedInput) {
      processElementOrdered(window, value);
    } else {
      processElementUnordered(window, value);
    }
  }
}
 

@Override
public void flatMap(
    WindowedValue<KV<K, InputT>> inWithMultipleWindows,
    Collector<WindowedValue<SingletonKeyedWorkItem<K, InputT>>> out)
    throws Exception {

  // we need to wrap each one work item per window for now
  // since otherwise the PushbackSideInputRunner will not correctly
  // determine whether side inputs are ready
  //
  // this is tracked as https://issues.apache.org/jira/browse/BEAM-1850
  for (WindowedValue<KV<K, InputT>> in : inWithMultipleWindows.explodeWindows()) {
    SingletonKeyedWorkItem<K, InputT> workItem =
        new SingletonKeyedWorkItem<>(
            in.getValue().getKey(), in.withValue(in.getValue().getValue()));

    out.collect(WindowedValue.valueInGlobalWindow(workItem));
  }
}
 

@Override
public void flatMap(
    WindowedValue<KV<K, InputT>> inWithMultipleWindows,
    Collector<WindowedValue<SingletonKeyedWorkItem<K, InputT>>> out)
    throws Exception {

  // we need to wrap each one work item per window for now
  // since otherwise the PushbackSideInputRunner will not correctly
  // determine whether side inputs are ready
  //
  // this is tracked as https://issues.apache.org/jira/browse/BEAM-1850
  for (WindowedValue<KV<K, InputT>> in : inWithMultipleWindows.explodeWindows()) {
    SingletonKeyedWorkItem<K, InputT> workItem =
        new SingletonKeyedWorkItem<>(
            in.getValue().getKey(), in.withValue(in.getValue().getValue()));

    out.collect(in.withValue(workItem));
  }
}
 

@Override
public void processElement(WindowedValue<InputT> compressedElem) {
  if (observesWindow) {
    for (WindowedValue<InputT> elem : compressedElem.explodeWindows()) {
      invokeProcessElement(elem);
    }
  } else {
    invokeProcessElement(compressedElem);
  }
}
 

@Override
public void add(WindowedValue<InputT> value, SparkCombineFn<InputT, ValueT, AccumT, ?> context)
    throws Exception {
  for (WindowedValue<InputT> v : value.explodeWindows()) {
    SparkCombineContext ctx = context.ctxtForValue(v);
    BoundedWindow window = getWindow(v);
    TimestampCombiner combiner = context.windowingStrategy.getTimestampCombiner();
    Instant windowTimestamp =
        combiner.assign(
            window,
            context.windowingStrategy.getWindowFn().getOutputTime(v.getTimestamp(), window));
    map.compute(
        window,
        (w, windowAccumulator) -> {
          final AccumT acc;
          final Instant timestamp;
          if (windowAccumulator == null) {
            acc = context.combineFn.createAccumulator(ctx);
            timestamp = windowTimestamp;
          } else {
            acc = windowAccumulator.getValue();
            timestamp = windowAccumulator.getTimestamp();
          }
          AccumT result = context.combineFn.addInput(acc, toValue(v), ctx);
          Instant timestampCombined = combiner.combine(windowTimestamp, timestamp);
          return WindowedValue.of(result, timestampCombined, window, PaneInfo.NO_FIRING);
        });
  }
  mergeWindows(context);
}
 

@Override
public void processElement(
    WindowedValue<KV<K, V>> inputElement, OpEmitter<KeyedWorkItem<K, V>> emitter) {
  final KV<K, V> kv = inputElement.getValue();
  for (WindowedValue<KV<K, V>> windowedValue : inputElement.explodeWindows()) {
    final KeyedWorkItem<K, V> workItem =
        new SingletonKeyedWorkItem<>(kv.getKey(), windowedValue.withValue(kv.getValue()));
    emitter.emitElement(windowedValue.withValue(workItem));
  }
}
 

@Override
public void processElement(WindowedValue<InputT> compressedElem) {
  for (WindowedValue<InputT> elem : compressedElem.explodeWindows()) {
    if (!sideInputFetcher.storeIfBlocked(elem)) {
      simpleDoFnRunner.processElement(elem);
    }
  }
}
 

@Override
public void processElement(WindowedValue<InputT> elem) {
  if (elem.getWindows().size() <= 1 || sideInputReader.isEmpty()) {
    invokeProcessElement(elem);
  } else {
    // We could modify the windowed value (and the processContext) to
    // avoid repeated allocations, but this is more straightforward.
    for (WindowedValue<InputT> windowedValue : elem.explodeWindows()) {
      invokeProcessElement(windowedValue);
    }
  }
}
 

@Override
public void processElement(WindowedValue<InputT> compressedElement) throws Exception {
  for (WindowedValue<InputT> element : compressedElement.explodeWindows()) {
    Collection<? extends BoundedWindow> windows = assignWindows(windowFn, element);
    outputBundle.add(
        WindowedValue.of(
            element.getValue(), element.getTimestamp(), windows, element.getPane()));
  }
}