Java Code Examples for org.apache.beam.sdk.transforms.windowing.BoundedWindow#TIMESTAMP_MAX_VALUE

private <T> CommittedBundle<T> afterCommitGetElementsShouldHaveAddedElements(
    Iterable<WindowedValue<T>> elems) {
  UncommittedBundle<T> bundle = bundleFactory.createRootBundle();
  Collection<Matcher<? super WindowedValue<T>>> expectations = new ArrayList<>();
  Instant minElementTs = BoundedWindow.TIMESTAMP_MAX_VALUE;
  for (WindowedValue<T> elem : elems) {
    bundle.add(elem);
    expectations.add(equalTo(elem));
    if (elem.getTimestamp().isBefore(minElementTs)) {
      minElementTs = elem.getTimestamp();
    }
  }
  Matcher<Iterable<? extends WindowedValue<T>>> containsMatcher =
      containsInAnyOrder(expectations);
  Instant commitTime = Instant.now();
  CommittedBundle<T> committed = bundle.commit(commitTime);
  assertThat(committed.getElements(), containsMatcher);

  // Sanity check that the test is meaningful.
  assertThat(minElementTs, not(equalTo(commitTime)));
  assertThat(committed.getMinimumTimestamp(), equalTo(minElementTs));
  assertThat(committed.getSynchronizedProcessingOutputWatermark(), equalTo(commitTime));

  return committed;
}
 

public SynchronizedProcessingTimeInputWatermark(
    String name,
    Collection<? extends Watermark> inputWms,
    Consumer<TimerData> timerUpdateNotification) {

  this.name = name;
  this.inputWms = inputWms;
  this.pendingBundles = new HashSet<>();
  this.processingTimers = new HashMap<>();
  this.synchronizedProcessingTimers = new HashMap<>();
  this.existingTimers = new HashMap<>();
  this.pendingTimers = new TreeSet<>();
  Instant initialHold = BoundedWindow.TIMESTAMP_MAX_VALUE;
  for (Watermark wm : inputWms) {
    initialHold = INSTANT_ORDERING.min(initialHold, wm.get());
  }
  this.earliestHold = new AtomicReference<>(initialHold);
  this.timerUpdateNotification = timerUpdateNotification;
}
 

@Override
public void processSideInput(
    String id, WindowedValue<? extends Iterable<?>> elements, OpEmitter<OutT> emitter) {
  @SuppressWarnings("unchecked")
  final WindowedValue<Iterable<?>> retypedElements = (WindowedValue<Iterable<?>>) elements;

  final PCollectionView<?> view = idToViewMap.get(id);
  if (view == null) {
    throw new IllegalArgumentException("No mapping of id " + id + " to view.");
  }

  sideInputHandler.addSideInputValue(view, retypedElements);

  final List<WindowedValue<InT>> previousPushbackValues = new ArrayList<>(pushbackValues);
  pushbackWatermarkHold = BoundedWindow.TIMESTAMP_MAX_VALUE;
  pushbackValues.clear();

  for (final WindowedValue<InT> value : previousPushbackValues) {
    processElement(value, emitter);
  }

  // We may be able to advance the output watermark since we may have played some pushed back
  // events.
  processWatermark(this.inputWatermark, emitter);
}
 

/** Calculates new watermark value and returns it if it's greater than the previous one. */
Instant calculateNew(long currentOffset, Instant processingTime) {

  // the source has seen all elements so the watermark is "+infinity"
  if (currentOffset >= endOffset) {
    watermark = BoundedWindow.TIMESTAMP_MAX_VALUE;
    return watermark;
  }

  Instant newWatermark =
      findLowestEventTimeInAdvance(currentOffset, processingTime)
          .minus(Duration.millis(options.watermarkDriftMillis));

  if (newWatermark.getMillis() > watermark.getMillis()) {
    watermark = newWatermark;
  }

  return watermark;
}
 

private static Instant minTimestamp(Iterable<? extends WindowedValue<?>> elements) {
  Instant minTs = BoundedWindow.TIMESTAMP_MAX_VALUE;
  for (WindowedValue<?> element : elements) {
    if (element.getTimestamp().isBefore(minTs)) {
      minTs = element.getTimestamp();
    }
  }
  return minTs;
}
 

/**
 * Get the earliest watermark hold in this table. Ignores the contents of any underlying table.
 */
private Instant getEarliestWatermarkHold() {
  Instant earliest = BoundedWindow.TIMESTAMP_MAX_VALUE;
  for (State existingState : this.values()) {
    if (existingState instanceof WatermarkHoldState) {
      Instant hold = ((WatermarkHoldState) existingState).read();
      if (hold != null && hold.isBefore(earliest)) {
        earliest = hold;
      }
    }
  }
  return earliest;
}
 

private void onSortFlushTimer(BoundedWindow window, Instant timestamp) {
  StateInternals stateInternals = stepContext.stateInternals();
  StateNamespace namespace = StateNamespaces.window(windowCoder, window);
  BagState<WindowedValue<InputT>> sortBuffer = stateInternals.state(namespace, sortBufferTag);
  ValueState<Instant> minStampState = stateInternals.state(namespace, sortBufferMinStampTag);
  List<WindowedValue<InputT>> keep = new ArrayList<>();
  List<WindowedValue<InputT>> flush = new ArrayList<>();
  Instant newMinStamp = BoundedWindow.TIMESTAMP_MAX_VALUE;
  for (WindowedValue<InputT> e : sortBuffer.read()) {
    if (!e.getTimestamp().isAfter(timestamp)) {
      flush.add(e);
    } else {
      keep.add(e);
      if (e.getTimestamp().isBefore(newMinStamp)) {
        newMinStamp = e.getTimestamp();
      }
    }
  }
  flush.stream()
      .sorted(Comparator.comparing(WindowedValue::getTimestamp))
      .forEachOrdered(e -> processElementUnordered(window, e));
  sortBuffer.clear();
  keep.forEach(sortBuffer::add);
  minStampState.write(newMinStamp);
  if (newMinStamp.isBefore(BoundedWindow.TIMESTAMP_MAX_VALUE)) {
    setupFlushTimerAndWatermarkHold(namespace, window, newMinStamp);
  } else {
    clearWatermarkHold(namespace);
  }
}
 

@Override
public Instant getWatermark() {
  getWatermarkCalls++;
  if (index + 1 == elems.size() && TestUnboundedSource.this.advanceWatermarkToInfinity) {
    return BoundedWindow.TIMESTAMP_MAX_VALUE;
  } else {
    return new Instant(index + getWatermarkCalls);
  }
}
 

@Override
public Instant getWatermark() {
  if (current >= numMessagesPerShard - 1) {
    // we won't emit further data, signal this with the final watermark
    return new Instant(BoundedWindow.TIMESTAMP_MAX_VALUE);
  }

  // The watermark is a promise about future elements, and the timestamps of elements are
  // strictly increasing for this source.
  return new Instant(current + 1);
}
 

@Test
public void timerUpdateBuilderWithSetAtEndOfTime() {
  Instant timerStamp = BoundedWindow.TIMESTAMP_MAX_VALUE;
  TimerData tooFar =
      TimerData.of(StateNamespaces.global(), timerStamp, timerStamp, TimeDomain.EVENT_TIME);

  TimerUpdateBuilder builder = TimerUpdate.builder(StructuralKey.empty());
  thrown.expect(IllegalArgumentException.class);
  thrown.expectMessage(timerStamp.toString());
  builder.setTimer(tooFar);
}
 

/**
 * Convert a Windmill message timestamp to a harness timestamp.
 *
 * <p>For soundness we require the test {@code harness message timestamp >= harness output
 * watermark} to imply {@code windmill message timestamp >= windmill output watermark}. Thus we
 * round timestamps down and output watermarks up.
 */
public static Instant windmillToHarnessTimestamp(long timestampUs) {
  // Windmill should never send us an unknown timestamp.
  Preconditions.checkArgument(timestampUs != Long.MIN_VALUE);
  Instant result = new Instant(divideAndRoundDown(timestampUs, 1000));
  if (result.isAfter(BoundedWindow.TIMESTAMP_MAX_VALUE)) {
    // End of time.
    return BoundedWindow.TIMESTAMP_MAX_VALUE;
  }
  return result;
}
 

@Test
public void addElementsAtEndOfTimeThrows() {
  Instant timestamp = BoundedWindow.TIMESTAMP_MAX_VALUE;
  WindowedValue<Integer> value = WindowedValue.timestampedValueInGlobalWindow(1, timestamp);

  UncommittedBundle<Integer> bundle = bundleFactory.createRootBundle();
  thrown.expect(IllegalArgumentException.class);
  thrown.expectMessage(timestamp.toString());
  bundle.add(value);
}
 

@VisibleForTesting
synchronized Instant getEarliestTimerTimestamp() {
  if (pendingTimers.isEmpty()) {
    return BoundedWindow.TIMESTAMP_MAX_VALUE;
  } else {
    return getMinimumOutputTimestamp(pendingTimers);
  }
}
 

private UnboundedQueuedReader(final UnboundedQueuedInput source) {
    this.source = source;
    this.state = LoopState.lookup(source.stateId);
    if (this.state != null) {
        this.state.referenceCounting.incrementAndGet();
    } else {
        this.waterMarkProvider = () -> BoundedWindow.TIMESTAMP_MAX_VALUE;
    }
}
 

public static <InputT> Builder<InputT> withoutHold(AppliedPTransform<?, ?, ?> transform) {
  return new Builder(transform, BoundedWindow.TIMESTAMP_MAX_VALUE);
}
 

/** Like {@link #complete(List)}, but assigns the same timestamp to all new outputs. */
public static <OutputT> PollResult<OutputT> complete(
    Instant timestamp, List<OutputT> outputs) {
  return new PollResult<>(
      addTimestamp(timestamp, outputs), BoundedWindow.TIMESTAMP_MAX_VALUE);
}
 

@Override
public Instant getWatermark() {
  return done ? BoundedWindow.TIMESTAMP_MAX_VALUE : BoundedWindow.TIMESTAMP_MIN_VALUE;
}
 

@Override
public UnboundedReader<T> createReader(
    PipelineOptions options, @Nullable CheckpointMark checkpointMark) {
  return new UnboundedReader<T>() {

    private int currentIdx = -1;
    private boolean lastAdvanced = false;

    @Override
    public boolean start() {
      return advance();
    }

    @Override
    public boolean advance() {
      if (lastAdvanced) {
        // Idle for this call.
        numIdles.merge(uuid, 1, Integer::sum);
        lastAdvanced = false;
        return false;
      }
      if (currentIdx < data.size() - 1) {
        currentIdx++;
        lastAdvanced = true;
        return true;
      }
      return false;
    }

    @Override
    public Instant getWatermark() {
      if (currentIdx >= data.size() - 1) {
        return BoundedWindow.TIMESTAMP_MAX_VALUE;
      }
      return new Instant(currentIdx);
    }

    @Override
    public CheckpointMark getCheckpointMark() {
      return CheckpointMark.NOOP_CHECKPOINT_MARK;
    }

    @Override
    public UnboundedSource<T, ?> getCurrentSource() {
      return IdlingUnboundedSource.this;
    }

    @Override
    public T getCurrent() throws NoSuchElementException {
      if (currentIdx >= 0 && currentIdx < data.size()) {
        return data.get(currentIdx);
      }
      throw new NoSuchElementException();
    }

    @Override
    public Instant getCurrentTimestamp() throws NoSuchElementException {
      if (currentIdx >= 0 && currentIdx < data.size()) {
        return new Instant(currentIdx);
      }
      throw new NoSuchElementException();
    }

    @Override
    public void close() {
      // No-op.
    }
  };
}
 

@Override
public void open(
    Config config,
    Context context,
    Scheduler<KeyedTimerData<Void>> timerRegistry,
    OpEmitter<OutT> emitter) {
  this.inputWatermark = BoundedWindow.TIMESTAMP_MIN_VALUE;
  this.sideInputWatermark = BoundedWindow.TIMESTAMP_MIN_VALUE;
  this.pushbackWatermarkHold = BoundedWindow.TIMESTAMP_MAX_VALUE;
  this.currentBundleElementCount = new AtomicLong(0L);
  this.bundleStartTime = new AtomicLong(Long.MAX_VALUE);
  this.isBundleStarted = new AtomicBoolean(false);
  this.bundleWatermarkHold = null;

  final DoFnSignature signature = DoFnSignatures.getSignature(doFn.getClass());
  final SamzaExecutionContext samzaExecutionContext =
      (SamzaExecutionContext) context.getApplicationContainerContext();
  this.samzaPipelineOptions = samzaExecutionContext.getPipelineOptions();
  this.maxBundleSize = samzaPipelineOptions.getMaxBundleSize();
  this.maxBundleTimeMs = samzaPipelineOptions.getMaxBundleTimeMs();
  this.bundleTimerScheduler = timerRegistry;

  if (this.maxBundleSize > 1) {
    scheduleNextBundleCheck();
  }

  final SamzaStoreStateInternals.Factory<?> nonKeyedStateInternalsFactory =
      SamzaStoreStateInternals.createStateInternalFactory(
          transformId, null, context.getTaskContext(), samzaPipelineOptions, signature);

  this.timerInternalsFactory =
      SamzaTimerInternalsFactory.createTimerInternalFactory(
          keyCoder,
          (Scheduler) timerRegistry,
          getTimerStateId(signature),
          nonKeyedStateInternalsFactory,
          windowingStrategy,
          isBounded,
          samzaPipelineOptions);

  this.sideInputHandler =
      new SideInputHandler(sideInputs, nonKeyedStateInternalsFactory.stateInternalsForKey(null));

  if (isPortable) {
    // storing events within a bundle in states
    final BagState<WindowedValue<InT>> bundledEventsBagState =
        nonKeyedStateInternalsFactory
            .stateInternalsForKey(null)
            .state(StateNamespaces.global(), StateTags.bag(bundleStateId, windowedValueCoder));
    final ExecutableStage executableStage = ExecutableStage.fromPayload(stagePayload);
    stageBundleFactory = samzaExecutionContext.getJobBundleFactory().forStage(executableStage);
    this.fnRunner =
        SamzaDoFnRunners.createPortable(
            samzaPipelineOptions,
            bundledEventsBagState,
            outputManagerFactory.create(emitter),
            stageBundleFactory,
            mainOutputTag,
            idToTupleTagMap,
            context,
            transformFullName);
  } else {
    this.fnRunner =
        SamzaDoFnRunners.create(
            samzaPipelineOptions,
            doFn,
            windowingStrategy,
            transformFullName,
            transformId,
            context,
            mainOutputTag,
            sideInputHandler,
            timerInternalsFactory,
            keyCoder,
            outputManagerFactory.create(emitter),
            inputCoder,
            sideOutputTags,
            outputCoders,
            doFnSchemaInformation,
            sideInputMapping);
  }

  this.pushbackFnRunner =
      SimplePushbackSideInputDoFnRunner.create(fnRunner, sideInputs, sideInputHandler);
  this.pushbackValues = new ArrayList<>();

  final Iterator<SamzaDoFnInvokerRegistrar> invokerReg =
      ServiceLoader.load(SamzaDoFnInvokerRegistrar.class).iterator();
  if (!invokerReg.hasNext()) {
    // use the default invoker here
    doFnInvoker = DoFnInvokers.invokerFor(doFn);
  } else {
    doFnInvoker = Iterators.getOnlyElement(invokerReg).invokerFor(doFn, context);
  }

  doFnInvoker.invokeSetup();
}
 

/**
 * Constructs a {@link PollResult} with the given outputs and declares that there will be no
 * new outputs for the current input. The {@link PollFn} will not be called again for this
 * input.
 */
public static <OutputT> PollResult<OutputT> complete(
    List<TimestampedValue<OutputT>> outputs) {
  return new PollResult<>(outputs, BoundedWindow.TIMESTAMP_MAX_VALUE);
}