Java Code Examples for org.apache.beam.sdk.transforms.windowing.BoundedWindow#TIMESTAMP_MIN_VALUE

public AppliedPTransformInputWatermark(
    String name,
    Collection<? extends Watermark> inputWatermarks,
    Consumer<TimerData> timerUpdateNotification) {

  this.name = name;
  this.inputWatermarks = inputWatermarks;

  // The ordering must order elements by timestamp, and must not compare two distinct elements
  // as equal. This is built on the assumption that any element added as a pending element will
  // be consumed without modifications.
  //
  // The same logic is applied for pending timers
  Ordering<Bundle<?, ?>> pendingBundleComparator =
      new BundleByElementTimestampComparator().compound(Ordering.arbitrary());
  this.pendingElements = TreeMultiset.create(pendingBundleComparator);
  this.pendingTimers = TreeMultiset.create();
  this.objectTimers = new HashMap<>();
  this.existingTimers = new HashMap<>();
  this.currentWatermark = new AtomicReference<>(BoundedWindow.TIMESTAMP_MIN_VALUE);
  this.timerUpdateNotification = timerUpdateNotification;
}
 

@Test
public void testMonotonicallyIncreasingWatermarkEstimator() {
  TimestampObservingWatermarkEstimator<Instant> watermarkEstimator =
      new WatermarkEstimators.MonotonicallyIncreasing(BoundedWindow.TIMESTAMP_MIN_VALUE);
  assertEquals(BoundedWindow.TIMESTAMP_MIN_VALUE, watermarkEstimator.currentWatermark());
  watermarkEstimator.observeTimestamp(BoundedWindow.TIMESTAMP_MIN_VALUE);
  watermarkEstimator.observeTimestamp(
      BoundedWindow.TIMESTAMP_MIN_VALUE.plus(Duration.standardHours(2)));
  assertEquals(
      BoundedWindow.TIMESTAMP_MIN_VALUE.plus(Duration.standardHours(2)),
      watermarkEstimator.currentWatermark());

  // Make sure that even if the watermark goes backwards we report the "greatest" value we have
  // reported so far.
  watermarkEstimator.observeTimestamp(
      BoundedWindow.TIMESTAMP_MIN_VALUE.plus(Duration.standardHours(1)));
  assertEquals(
      BoundedWindow.TIMESTAMP_MIN_VALUE.plus(Duration.standardHours(2)),
      watermarkEstimator.currentWatermark());

  watermarkEstimator.observeTimestamp(BoundedWindow.TIMESTAMP_MAX_VALUE);
  assertEquals(BoundedWindow.TIMESTAMP_MAX_VALUE, watermarkEstimator.currentWatermark());
}
 

public SynchronizedProcessingTimeOutputWatermark(
    String name, SynchronizedProcessingTimeInputWatermark inputWm) {
  this.name = name;
  this.inputWm = inputWm;
  holds = new PerKeyHolds();
  this.latestRefresh = new AtomicReference<>(BoundedWindow.TIMESTAMP_MIN_VALUE);
}
 

public Simulator(NexmarkConfiguration configuration) {
  super(new WinningBidsSimulator(configuration).results());
  winningPricesByCategory = new ArrayList<>();
  lastTimestamp = BoundedWindow.TIMESTAMP_MIN_VALUE;
  windowStart = NexmarkUtils.BEGINNING_OF_TIME;
  lastSeenResults = new TreeMap<>();
}
 

public AppliedPTransformOutputWatermark(
    String name, AppliedPTransformInputWatermark inputWatermark) {
  this.name = name;
  this.inputWatermark = inputWatermark;
  holds = new PerKeyHolds();
  currentWatermark = new AtomicReference<>(BoundedWindow.TIMESTAMP_MIN_VALUE);
}
 

public WinningBidsSimulator(NexmarkConfiguration configuration) {
  super(NexmarkUtils.standardEventIterator(configuration));
  openAuctions = new TreeMap<>();
  closedAuctions = new TreeSet<>();
  bestBids = new TreeMap<>();
  bidsWithoutAuctions = new ArrayList<>();
  lastTimestamp = BoundedWindow.TIMESTAMP_MIN_VALUE;
}
 

private TransformWatermarks(
    ExecutableT executable,
    AppliedPTransformInputWatermark inputWatermark,
    AppliedPTransformOutputWatermark outputWatermark,
    SynchronizedProcessingTimeInputWatermark inputSynchProcessingWatermark,
    SynchronizedProcessingTimeOutputWatermark outputSynchProcessingWatermark) {
  this.executable = executable;
  this.inputWatermark = inputWatermark;
  this.outputWatermark = outputWatermark;

  this.synchronizedProcessingInputWatermark = inputSynchProcessingWatermark;
  this.synchronizedProcessingOutputWatermark = outputSynchProcessingWatermark;
  this.latestSynchronizedInputWm = BoundedWindow.TIMESTAMP_MIN_VALUE;
  this.latestSynchronizedOutputWm = BoundedWindow.TIMESTAMP_MIN_VALUE;
}
 

private Instant ensureTimestampWithinBounds(Instant timestamp) {
  if (timestamp.isBefore(BoundedWindow.TIMESTAMP_MIN_VALUE)) {
    timestamp = BoundedWindow.TIMESTAMP_MIN_VALUE;
  } else if (timestamp.isAfter(BoundedWindow.TIMESTAMP_MAX_VALUE)) {
    timestamp = BoundedWindow.TIMESTAMP_MAX_VALUE;
  }
  return timestamp;
}
 

@Test
public void processElementSideInputNotReadySomeWindows() {
  when(reader.isReady(Mockito.eq(singletonView), Mockito.eq(GlobalWindow.INSTANCE)))
      .thenReturn(false);
  when(reader.isReady(
          Mockito.eq(singletonView),
          org.mockito.AdditionalMatchers.not(Mockito.eq(GlobalWindow.INSTANCE))))
      .thenReturn(true);

  SimplePushbackSideInputDoFnRunner<Integer, Integer> runner =
      createRunner(ImmutableList.of(singletonView));

  IntervalWindow littleWindow = new IntervalWindow(new Instant(-500L), new Instant(0L));
  IntervalWindow bigWindow =
      new IntervalWindow(BoundedWindow.TIMESTAMP_MIN_VALUE, new Instant(250L));
  WindowedValue<Integer> multiWindow =
      WindowedValue.of(
          2,
          new Instant(-2),
          ImmutableList.of(littleWindow, bigWindow, GlobalWindow.INSTANCE),
          PaneInfo.NO_FIRING);
  Iterable<WindowedValue<Integer>> multiWindowPushback =
      runner.processElementInReadyWindows(multiWindow);
  assertThat(
      multiWindowPushback,
      containsInAnyOrder(WindowedValue.timestampedValueInGlobalWindow(2, new Instant(-2L))));
  assertThat(
      underlying.inputElems,
      containsInAnyOrder(
          WindowedValue.of(
              2, new Instant(-2), ImmutableList.of(littleWindow), PaneInfo.NO_FIRING),
          WindowedValue.of(2, new Instant(-2), ImmutableList.of(bigWindow), PaneInfo.NO_FIRING)));
}
 

PartitionState(
    TopicPartition partition, long nextOffset, TimestampPolicy<K, V> timestampPolicy) {
  this.topicPartition = partition;
  this.nextOffset = nextOffset;
  this.latestOffset = UNINITIALIZED_OFFSET;
  this.latestOffsetFetchTime = BoundedWindow.TIMESTAMP_MIN_VALUE;
  this.lastWatermark = BoundedWindow.TIMESTAMP_MIN_VALUE;
  this.timestampPolicy = timestampPolicy;
}
 

@Override
public UnboundedReader<T> createReader(PipelineOptions po, Checkpoint<T> cmt) {
  return new UnboundedReader<T>() {

    T read = cmt == null ? null : cmt.read;
    boolean finished = false;

    @Override
    public boolean start() throws IOException {
      return advance();
    }

    @Override
    public boolean advance() throws IOException {
      try {
        Optional<T> taken = queue.take();
        if (taken.isPresent()) {
          read = taken.get();
          return true;
        }
        finished = true;
        return false;
      } catch (InterruptedException ex) {
        throw new IOException(ex);
      }
    }

    @Override
    public Instant getWatermark() {
      if (finished) {
        return BoundedWindow.TIMESTAMP_MAX_VALUE;
      }
      return BoundedWindow.TIMESTAMP_MIN_VALUE;
    }

    @Override
    public CheckpointMark getCheckpointMark() {
      return new Checkpoint(read);
    }

    @Override
    public UnboundedSource<T, ?> getCurrentSource() {
      return StaticQueueSource.this;
    }

    @Override
    public T getCurrent() throws NoSuchElementException {
      return read;
    }

    @Override
    public Instant getCurrentTimestamp() {
      return getWatermark();
    }

    @Override
    public void close() throws IOException {
      // nop
    }
  };
}
 

@Override
public scala.Option<RDD<BoxedUnit>> compute(Time validTime) {
  // compute parent.
  scala.Option<RDD<Metadata>> parentRDDOpt = parent.getOrCompute(validTime);
  final MetricsContainerStepMapAccumulator metricsAccum = MetricsAccumulator.getInstance();
  long count = 0;
  SparkWatermarks sparkWatermark = null;
  Instant globalLowWatermarkForBatch = BoundedWindow.TIMESTAMP_MIN_VALUE;
  Instant globalHighWatermarkForBatch = BoundedWindow.TIMESTAMP_MIN_VALUE;
  long maxReadDuration = 0;
  if (parentRDDOpt.isDefined()) {
    JavaRDD<Metadata> parentRDD = parentRDDOpt.get().toJavaRDD();
    for (Metadata metadata : parentRDD.collect()) {
      count += metadata.getNumRecords();
      // compute the global input watermark - advance to latest of all partitions.
      Instant partitionLowWatermark = metadata.getLowWatermark();
      globalLowWatermarkForBatch =
          globalLowWatermarkForBatch.isBefore(partitionLowWatermark)
              ? partitionLowWatermark
              : globalLowWatermarkForBatch;
      Instant partitionHighWatermark = metadata.getHighWatermark();
      globalHighWatermarkForBatch =
          globalHighWatermarkForBatch.isBefore(partitionHighWatermark)
              ? partitionHighWatermark
              : globalHighWatermarkForBatch;
      // Update metrics reported in the read
      final Gauge gauge = Metrics.gauge(NAMESPACE, READ_DURATION_MILLIS);
      final MetricsContainer container = metadata.getMetricsContainers().getContainer(stepName);
      try (Closeable ignored = MetricsEnvironment.scopedMetricsContainer(container)) {
        final long readDurationMillis = metadata.getReadDurationMillis();
        if (readDurationMillis > maxReadDuration) {
          gauge.set(readDurationMillis);
        }
      } catch (IOException e) {
        throw new RuntimeException(e);
      }
      metricsAccum.value().updateAll(metadata.getMetricsContainers());
    }

    sparkWatermark =
        new SparkWatermarks(
            globalLowWatermarkForBatch,
            globalHighWatermarkForBatch,
            new Instant(validTime.milliseconds()));
    // add to watermark queue.
    GlobalWatermarkHolder.add(inputDStreamId, sparkWatermark);
  }
  // report - for RateEstimator and visibility.
  report(validTime, count, sparkWatermark);
  return scala.Option.empty();
}
 

/** Build the {@link TimerInternals} according to the feeding streams. */
public static SparkTimerInternals forStreamFromSources(
    List<Integer> sourceIds, Map<Integer, SparkWatermarks> watermarks) {
  // if watermarks are invalid for the specific ids, use defaults.
  if (watermarks == null
      || watermarks.isEmpty()
      || Collections.disjoint(sourceIds, watermarks.keySet())) {
    return new SparkTimerInternals(
        BoundedWindow.TIMESTAMP_MIN_VALUE, BoundedWindow.TIMESTAMP_MIN_VALUE, new Instant(0));
  }
  // there might be more than one stream feeding this stream, slowest WM is the right one.
  Instant slowestLowWatermark = BoundedWindow.TIMESTAMP_MAX_VALUE;
  Instant slowestHighWatermark = BoundedWindow.TIMESTAMP_MAX_VALUE;
  // synchronized processing time should clearly be synchronized.
  Instant synchronizedProcessingTime = null;
  for (Integer sourceId : sourceIds) {
    SparkWatermarks sparkWatermarks = watermarks.get(sourceId);
    if (sparkWatermarks != null) {
      // keep slowest WMs.
      slowestLowWatermark =
          slowestLowWatermark.isBefore(sparkWatermarks.getLowWatermark())
              ? slowestLowWatermark
              : sparkWatermarks.getLowWatermark();
      slowestHighWatermark =
          slowestHighWatermark.isBefore(sparkWatermarks.getHighWatermark())
              ? slowestHighWatermark
              : sparkWatermarks.getHighWatermark();
      if (synchronizedProcessingTime == null) {
        // firstime set.
        synchronizedProcessingTime = sparkWatermarks.getSynchronizedProcessingTime();
      } else {
        // assert on following.
        checkArgument(
            sparkWatermarks.getSynchronizedProcessingTime().equals(synchronizedProcessingTime),
            "Synchronized time is expected to keep synchronized across sources.");
      }
    }
  }
  return new SparkTimerInternals(
      slowestLowWatermark, slowestHighWatermark, synchronizedProcessingTime);
}
 

@Override
public Instant getWatermark() {
  return done ? BoundedWindow.TIMESTAMP_MAX_VALUE : BoundedWindow.TIMESTAMP_MIN_VALUE;
}
 

@Override
public Instant getTimestamp() {
  return BoundedWindow.TIMESTAMP_MIN_VALUE;
}
 

/** By default, returns the minimum possible timestamp. */
@Override
public Instant getCurrentTimestamp() throws NoSuchElementException {
  return BoundedWindow.TIMESTAMP_MIN_VALUE;
}
 

private Builder(Coder<T> coder) {
  this(coder, ImmutableList.of(), BoundedWindow.TIMESTAMP_MIN_VALUE);
}
 

public Simulator(NexmarkConfiguration configuration) {
  super(NexmarkUtils.standardEventIterator(configuration));
  highestBids = new ArrayList<>();
  windowStart = NexmarkUtils.BEGINNING_OF_TIME;
  lastTimestamp = BoundedWindow.TIMESTAMP_MIN_VALUE;
}
 

public Simulator(NexmarkConfiguration configuration) {
  super(new WinningBidsSimulator(configuration).results());
  winningBidsPerSeller = new TreeMap<>();
  totalWinningBidPricesPerSeller = new TreeMap<>();
  lastTimestamp = BoundedWindow.TIMESTAMP_MIN_VALUE;
}
 

@Override
public IntervalWindow assignWindow(Instant timestamp) {
  return new IntervalWindow(
      BoundedWindow.TIMESTAMP_MIN_VALUE, GlobalWindow.INSTANCE.maxTimestamp());
}