Java Code Examples for org.apache.beam.sdk.transforms.windowing.Sessions#withGapDuration()

public static WindowFn<?, ?> windowFnFromProto(FunctionSpec windowFnSpec) {
  try {
    String s = windowFnSpec.getUrn();
    if (s.equals(getUrn(GlobalWindowsPayload.Enum.PROPERTIES))) {
      return new GlobalWindows();
    } else if (s.equals(getUrn(FixedWindowsPayload.Enum.PROPERTIES))) {
      FixedWindowsPayload fixedParams = FixedWindowsPayload.parseFrom(windowFnSpec.getPayload());
      return FixedWindows.of(Duration.millis(Durations.toMillis(fixedParams.getSize())))
          .withOffset(Duration.millis(Timestamps.toMillis(fixedParams.getOffset())));
    } else if (s.equals(getUrn(SlidingWindowsPayload.Enum.PROPERTIES))) {
      SlidingWindowsPayload slidingParams =
          SlidingWindowsPayload.parseFrom(windowFnSpec.getPayload());
      return SlidingWindows.of(Duration.millis(Durations.toMillis(slidingParams.getSize())))
          .every(Duration.millis(Durations.toMillis(slidingParams.getPeriod())))
          .withOffset(Duration.millis(Timestamps.toMillis(slidingParams.getOffset())));
    } else if (s.equals(getUrn(SessionWindowsPayload.Enum.PROPERTIES))) {
      SessionWindowsPayload sessionParams =
          SessionWindowsPayload.parseFrom(windowFnSpec.getPayload());
      return Sessions.withGapDuration(
          Duration.millis(Durations.toMillis(sessionParams.getGapSize())));
    } else if (s.equals(SERIALIZED_JAVA_WINDOWFN_URN)) {
      return (WindowFn<?, ?>)
          SerializableUtils.deserializeFromByteArray(
              windowFnSpec.getPayload().toByteArray(), "WindowFn");
    } else {
      throw new IllegalArgumentException(
          "Unknown or unsupported WindowFn: " + windowFnSpec.getUrn());
    }
  } catch (InvalidProtocolBufferException e) {
    throw new IllegalArgumentException(
        String.format(
            "%s for %s with URN %s did not contain expected proto message for payload",
            FunctionSpec.class.getSimpleName(),
            WindowFn.class.getSimpleName(),
            windowFnSpec.getUrn()),
        e);
  }
}
 

@Before
public void setup() {
  windowFn = Sessions.withGapDuration(Duration.millis(10));
  state = InMemoryStateInternals.forKey("dummyKey");
  set = new MergingActiveWindowSet<>(windowFn, state);
  @SuppressWarnings("unchecked")
  ActiveWindowSet.MergeCallback<IntervalWindow> callback =
      mock(ActiveWindowSet.MergeCallback.class);
  this.callback = callback;
}
 

@Override public void handleStep( ILogChannel log, TransformMeta transformMeta, Map<String, PCollection<HopRow>> stepCollectionMap,
                                  Pipeline pipeline, IRowMeta inputRowMeta, List<TransformMeta> previousSteps,
                                  PCollection<HopRow> input ) throws HopException {

  BeamWindowMeta beamWindowMeta = (BeamWindowMeta) transformMeta.getTransform();

  if ( StringUtils.isEmpty( beamWindowMeta.getWindowType() ) ) {
    throw new HopException( "Please specify a window type in Beam Window transform '" + transformMeta.getName() + "'" );
  }

  String duration = pipelineMeta.environmentSubstitute( beamWindowMeta.getDuration() );
  long durationSeconds = Const.toLong( duration, -1L );

  PCollection<HopRow> stepPCollection;

  if ( BeamDefaults.WINDOW_TYPE_FIXED.equals( beamWindowMeta.getWindowType() ) ) {

    if ( durationSeconds <= 0 ) {
      throw new HopException( "Please specify a valid positive window size (duration) for Beam window transform '" + transformMeta.getName() + "'" );
    }

    FixedWindows fixedWindows = FixedWindows
      .of( Duration.standardSeconds( durationSeconds ) );
    stepPCollection = input.apply( Window.into( fixedWindows ) );

  } else if ( BeamDefaults.WINDOW_TYPE_SLIDING.equals( beamWindowMeta.getWindowType() ) ) {

    if ( durationSeconds <= 0 ) {
      throw new HopException( "Please specify a valid positive window size (duration) for Beam window transform '" + transformMeta.getName() + "'" );
    }

    String every = pipelineMeta.environmentSubstitute( beamWindowMeta.getEvery() );
    long everySeconds = Const.toLong( every, -1L );

    SlidingWindows slidingWindows = SlidingWindows
      .of( Duration.standardSeconds( durationSeconds ) )
      .every( Duration.standardSeconds( everySeconds ) );
    stepPCollection = input.apply( Window.into( slidingWindows ) );

  } else if ( BeamDefaults.WINDOW_TYPE_SESSION.equals( beamWindowMeta.getWindowType() ) ) {

    if ( durationSeconds < 600 ) {
      throw new HopException(
        "Please specify a window size (duration) of at least 600 (10 minutes) for Beam window transform '" + transformMeta.getName() + "'.  This is the minimum gap between session windows." );
    }

    Sessions sessionWindows = Sessions
      .withGapDuration( Duration.standardSeconds( durationSeconds ) );
    stepPCollection = input.apply( Window.into( sessionWindows ) );

  } else if ( BeamDefaults.WINDOW_TYPE_GLOBAL.equals( beamWindowMeta.getWindowType() ) ) {

    stepPCollection = input.apply( Window.into( new GlobalWindows() ) );

  } else {
    throw new HopException( "Beam Window type '" + beamWindowMeta.getWindowType() + " is not supported in transform '" + transformMeta.getName() + "'" );
  }

  // Now get window information about the window if we asked about it...
  //
  if ( StringUtils.isNotEmpty( beamWindowMeta.getStartWindowField() ) ||
    StringUtils.isNotEmpty( beamWindowMeta.getEndWindowField() ) ||
    StringUtils.isNotEmpty( beamWindowMeta.getMaxWindowField() ) ) {

    WindowInfoFn windowInfoFn = new WindowInfoFn(
      transformMeta.getName(),
      pipelineMeta.environmentSubstitute( beamWindowMeta.getMaxWindowField() ),
      pipelineMeta.environmentSubstitute( beamWindowMeta.getStartWindowField() ),
      pipelineMeta.environmentSubstitute( beamWindowMeta.getMaxWindowField() ),
      JsonRowMeta.toJson( inputRowMeta ),
      transformPluginClasses,
      xpPluginClasses
    );

    stepPCollection = stepPCollection.apply( ParDo.of( windowInfoFn ) );
  }

  // Save this in the map
  //
  stepCollectionMap.put( transformMeta.getName(), stepPCollection );
  log.logBasic( "Handled transform (WINDOW) : " + transformMeta.getName() + ", gets data from " + previousSteps.size() + " previous transform(s)" );
}
 

@Override public void handleStep( LogChannelInterface log, StepMeta stepMeta, Map<String, PCollection<KettleRow>> stepCollectionMap,
                                  Pipeline pipeline, RowMetaInterface inputRowMeta, List<StepMeta> previousSteps,
                                  PCollection<KettleRow> input ) throws KettleException {

  BeamWindowMeta beamWindowMeta = (BeamWindowMeta) stepMeta.getStepMetaInterface();

  if ( StringUtils.isEmpty( beamWindowMeta.getWindowType() ) ) {
    throw new KettleException( "Please specify a window type in Beam Window step '" + stepMeta.getName() + "'" );
  }

  String duration = transMeta.environmentSubstitute( beamWindowMeta.getDuration() );
  long durationSeconds = Const.toLong( duration, -1L );

  PCollection<KettleRow> stepPCollection;

  if ( BeamDefaults.WINDOW_TYPE_FIXED.equals( beamWindowMeta.getWindowType() ) ) {

    if ( durationSeconds <= 0 ) {
      throw new KettleException( "Please specify a valid positive window size (duration) for Beam window step '" + stepMeta.getName() + "'" );
    }

    FixedWindows fixedWindows = FixedWindows
      .of( Duration.standardSeconds( durationSeconds ) );
    stepPCollection = input.apply( Window.into( fixedWindows ) );

  } else if ( BeamDefaults.WINDOW_TYPE_SLIDING.equals( beamWindowMeta.getWindowType() ) ) {

    if ( durationSeconds <= 0 ) {
      throw new KettleException( "Please specify a valid positive window size (duration) for Beam window step '" + stepMeta.getName() + "'" );
    }

    String every = transMeta.environmentSubstitute( beamWindowMeta.getEvery() );
    long everySeconds = Const.toLong( every, -1L );

    SlidingWindows slidingWindows = SlidingWindows
      .of( Duration.standardSeconds( durationSeconds ) )
      .every( Duration.standardSeconds( everySeconds ) );
    stepPCollection = input.apply( Window.into( slidingWindows ) );

  } else if ( BeamDefaults.WINDOW_TYPE_SESSION.equals( beamWindowMeta.getWindowType() ) ) {

    if ( durationSeconds < 600 ) {
      throw new KettleException(
        "Please specify a window size (duration) of at least 600 (10 minutes) for Beam window step '" + stepMeta.getName() + "'.  This is the minimum gap between session windows." );
    }

    Sessions sessionWindows = Sessions
      .withGapDuration( Duration.standardSeconds( durationSeconds ) );
    stepPCollection = input.apply( Window.into( sessionWindows ) );

  } else if ( BeamDefaults.WINDOW_TYPE_GLOBAL.equals( beamWindowMeta.getWindowType() ) ) {

    stepPCollection = input.apply( Window.into( new GlobalWindows() ) );

  } else {
    throw new KettleException( "Beam Window type '" + beamWindowMeta.getWindowType() + " is not supported in step '" + stepMeta.getName() + "'" );
  }

  // Now get window information about the window if we asked about it...
  //
  if ( StringUtils.isNotEmpty( beamWindowMeta.getStartWindowField() ) ||
    StringUtils.isNotEmpty( beamWindowMeta.getEndWindowField() ) ||
    StringUtils.isNotEmpty( beamWindowMeta.getMaxWindowField() ) ) {

    WindowInfoFn windowInfoFn = new WindowInfoFn(
      stepMeta.getName(),
      transMeta.environmentSubstitute( beamWindowMeta.getMaxWindowField() ),
      transMeta.environmentSubstitute( beamWindowMeta.getStartWindowField() ),
      transMeta.environmentSubstitute( beamWindowMeta.getMaxWindowField() ),
      JsonRowMeta.toJson( inputRowMeta ),
      stepPluginClasses,
      xpPluginClasses
    );

    stepPCollection = stepPCollection.apply( ParDo.of( windowInfoFn ) );
  }

  // Save this in the map
  //
  stepCollectionMap.put( stepMeta.getName(), stepPCollection );
  log.logBasic( "Handled step (WINDOW) : " + stepMeta.getName() + ", gets data from " + previousSteps.size() + " previous step(s)" );
}
 

/**
 * Returns a {@link WindowFn} based on the SQL windowing function defined by {#code operatorKind}.
 * Supported {@link SqlKind}s:
 *
 * <ul>
 *   <li>{@link SqlKind#TUMBLE}, mapped to {@link FixedWindows};
 *   <li>{@link SqlKind#HOP}, mapped to {@link SlidingWindows};
 *   <li>{@link SqlKind#SESSION}, mapped to {@link Sessions};
 * </ul>
 *
 * <p>For example:
 *
 * <pre>{@code
 * SELECT event_timestamp, COUNT(*)
 * FROM PCOLLECTION
 * GROUP BY TUMBLE(event_timestamp, INTERVAL '1' HOUR)
 * }</pre>
 *
 * <p>SQL window functions support optional window_offset parameter which indicates a how window
 * definition is offset from the event time. Offset is zero if not specified.
 *
 * <p>Beam model does not support offset for session windows, so this method will throw {@link
 * UnsupportedOperationException} if offset is specified in SQL query for {@link SqlKind#SESSION}.
 */
private static @Nullable WindowFn createWindowFn(List<RexNode> parameters, SqlKind operatorKind) {
  switch (operatorKind) {
    case TUMBLE:

      // Fixed-size, non-intersecting time-based windows, for example:
      //   every hour aggregate elements from the previous hour;
      //
      // SQL Syntax:
      //   TUMBLE(monotonic_field, window_size [, window_offset])
      //
      // Example:
      //   TUMBLE(event_timestamp_field, INTERVAL '1' HOUR)

      FixedWindows fixedWindows = FixedWindows.of(durationParameter(parameters, 1));
      if (parameters.size() == 3) {
        fixedWindows = fixedWindows.withOffset(durationParameter(parameters, 2));
      }

      return fixedWindows;
    case HOP:

      // Sliding, fixed-size, intersecting time-based windows, for example:
      //   every minute aggregate elements from the previous hour;
      //
      // SQL Syntax:
      //   HOP(monotonic_field, emit_frequency, window_size [, window_offset])

      SlidingWindows slidingWindows =
          SlidingWindows.of(durationParameter(parameters, 2))
              .every(durationParameter(parameters, 1));

      if (parameters.size() == 4) {
        slidingWindows = slidingWindows.withOffset(durationParameter(parameters, 3));
      }

      return slidingWindows;
    case SESSION:

      // Session windows, for example:
      //   aggregate events after a gap of 1 minute of no events;
      //
      // SQL Syntax:
      //   SESSION(monotonic_field, session_gap)
      //
      // Example:
      //   SESSION(event_timestamp_field, INTERVAL '1' MINUTE)

      Sessions sessions = Sessions.withGapDuration(durationParameter(parameters, 1));
      if (parameters.size() == 3) {
        throw new UnsupportedOperationException(
            "Specifying alignment (offset) is not supported for session windows");
      }

      return sessions;
    default:
      return null;
  }
}