Java Code Examples for org.apache.beam.sdk.values.TimestampedValue#getTimestamp()

@Override
protected void run() {
  TimestampedValue<AuctionBid> timestampedWinningBid = nextInput();
  if (timestampedWinningBid == null) {
    prune(NexmarkUtils.END_OF_TIME);
    for (TimestampedValue<CategoryPrice> result : lastSeenResults.values()) {
      addResult(result);
    }
    allDone();
    return;
  }
  lastTimestamp = timestampedWinningBid.getTimestamp();
  Instant newWindowStart =
      windowStart(
          Duration.standardSeconds(configuration.windowSizeSec),
          Duration.standardSeconds(configuration.windowPeriodSec),
          lastTimestamp);
  prune(newWindowStart);
  captureWinningBid(
      timestampedWinningBid.getValue().auction,
      timestampedWinningBid.getValue().bid,
      lastTimestamp);
}
 

@Override
protected void run() {
  TimestampedValue<AuctionBid> timestampedWinningBid = nextInput();
  if (timestampedWinningBid == null) {
    for (Map.Entry<Long, Queue<Bid>> entry : winningBidsPerSeller.entrySet()) {
      long seller = entry.getKey();
      long count = entry.getValue().size();
      long total = totalWinningBidPricesPerSeller.get(seller);
      addResult(
          TimestampedValue.of(
              new SellerPrice(seller, Math.round((double) total / count)), lastTimestamp));
    }
    allDone();
    return;
  }

  lastTimestamp = timestampedWinningBid.getTimestamp();
  captureWinningBid(
      timestampedWinningBid.getValue().auction,
      timestampedWinningBid.getValue().bid,
      lastTimestamp);
}
 

@Override
public void run() {
  TimestampedValue<Event> timestampedEvent = nextInput();
  if (timestampedEvent == null) {
    // Drain the remaining windows.
    retireWindows(NexmarkUtils.END_OF_TIME);
    allDone();
    return;
  }

  Event event = timestampedEvent.getValue();
  if (event.bid == null) {
    // Ignore non-bid events.
    return;
  }
  Instant timestamp = timestampedEvent.getTimestamp();
  Instant newWindowStart =
      windowStart(
          Duration.standardSeconds(configuration.windowSizeSec),
          Duration.standardSeconds(configuration.windowPeriodSec),
          timestamp);
  // Capture results from any windows we can now retire.
  retireWindows(newWindowStart);
  // Capture current bid.
  captureBid(event.bid, timestamp);
}
 

/**
 * Assigns the given {@code timestampedValue} to windows using the specified {@code windowFn}, and
 * verifies that result of {@code windowFn.getOutputTimestamp} for each window is within the
 * proper bound. This version allows passing a {@link TimestampedValue} in case the value is
 * needed to assign windows.
 */
public static <T, W extends BoundedWindow> void validateNonInterferingOutputTimesWithValue(
    WindowFn<T, W> windowFn, TimestampedValue<T> timestampedValue) throws Exception {
  Collection<W> windows = assignedWindowsWithValue(windowFn, timestampedValue);

  Instant instant = timestampedValue.getTimestamp();
  for (W window : windows) {
    Instant outputTimestamp = windowFn.getOutputTime(instant, window);
    assertFalse(
        "getOutputTime must be greater than or equal to input timestamp",
        outputTimestamp.isBefore(instant));
    assertFalse(
        "getOutputTime must be less than or equal to the max timestamp",
        outputTimestamp.isAfter(window.maxTimestamp()));
  }
}
 

@Override
protected void run() {
  TimestampedValue<Event> timestampedEvent = nextInput();
  if (timestampedEvent == null) {
    // Capture all remaining bids in results.
    retireWindow(lastTimestamp);
    allDone();
    return;
  }

  Event event = timestampedEvent.getValue();
  if (event.bid == null) {
    // Ignore non-bid events.
    return;
  }
  lastTimestamp = timestampedEvent.getTimestamp();
  Instant newWindowStart =
      windowStart(
          Duration.standardSeconds(configuration.windowSizeSec),
          Duration.standardSeconds(configuration.windowSizeSec),
          lastTimestamp);
  if (!newWindowStart.equals(windowStart)) {
    // Capture highest priced bids in current window and retire it.
    retireWindow(lastTimestamp);
    windowStart = newWindowStart;
  }
  // Keep only the highest bids.
  captureBid(event.bid);
}
 

/**
 * Assigns the given {@code timestampedValue} to windows using the specified {@code windowFn}, and
 * verifies that result of {@link WindowFn#getOutputTime windowFn.getOutputTime} for later windows
 * (as defined by {@code maxTimestamp} won't prevent the watermark from passing the end of earlier
 * windows.
 *
 * <p>This verifies that overlapping windows don't interfere at all. Depending on the {@code
 * windowFn} this may be stricter than desired. This version allows passing a {@link
 * TimestampedValue} in case the value is needed to assign windows.
 */
public static <T, W extends BoundedWindow> void validateGetOutputTimestampWithValue(
    WindowFn<T, W> windowFn, TimestampedValue<T> timestampedValue) throws Exception {
  Collection<W> windows = assignedWindowsWithValue(windowFn, timestampedValue);
  List<W> sortedWindows = new ArrayList<>(windows);
  sortedWindows.sort(Comparator.comparing(BoundedWindow::maxTimestamp));

  Instant instant = timestampedValue.getTimestamp();
  Instant endOfPrevious = null;
  for (W window : sortedWindows) {
    Instant outputTimestamp = windowFn.getOutputTime(instant, window);
    if (endOfPrevious == null) {
      // If this is the first window, the output timestamp can be anything, as long as it is in
      // the valid range.
      assertFalse(
          "getOutputTime must be greater than or equal to input timestamp",
          outputTimestamp.isBefore(instant));
      assertFalse(
          "getOutputTime must be less than or equal to the max timestamp",
          outputTimestamp.isAfter(window.maxTimestamp()));
    } else {
      // If this is a later window, the output timestamp must be after the end of the previous
      // window
      assertTrue(
          "getOutputTime must be greater than the end of the previous window",
          outputTimestamp.isAfter(endOfPrevious));
      assertFalse(
          "getOutputTime must be less than or equal to the max timestamp",
          outputTimestamp.isAfter(window.maxTimestamp()));
    }
    endOfPrevious = window.maxTimestamp();
  }
}
 

@Override
public void run(SourceContext<WindowedValue<T>> ctx) throws CoderException {
  TestStream<T> testStream = testStreamDecoder.apply(payload);
  List<TestStream.Event<T>> events = testStream.getEvents();

  for (int eventId = 0; isRunning && eventId < events.size(); eventId++) {
    TestStream.Event<T> event = events.get(eventId);

    synchronized (ctx.getCheckpointLock()) {
      if (event instanceof TestStream.ElementEvent) {
        for (TimestampedValue<T> element : ((TestStream.ElementEvent<T>) event).getElements()) {
          Instant timestamp = element.getTimestamp();
          WindowedValue<T> value =
              WindowedValue.of(
                  element.getValue(), timestamp, GlobalWindow.INSTANCE, PaneInfo.NO_FIRING);
          ctx.collectWithTimestamp(value, timestamp.getMillis());
        }
      } else if (event instanceof TestStream.WatermarkEvent) {
        long millis = ((TestStream.WatermarkEvent<T>) event).getWatermark().getMillis();
        ctx.emitWatermark(new Watermark(millis));
      } else if (event instanceof TestStream.ProcessingTimeEvent) {
        // There seems to be no clean way to implement this
        throw new UnsupportedOperationException(
            "Advancing Processing time is not supported by the Flink Runner.");
      } else {
        throw new IllegalStateException("Unknown event type " + event);
      }
    }
  }
}
 

@Override
protected void run() {
  TimestampedValue<Event> timestampedEvent = nextInput();
  if (timestampedEvent == null) {
    allDone();
    return;
  }
  Event event = timestampedEvent.getValue();
  if (event.bid != null) {
    // Ignore bid events.
    return;
  }

  Instant timestamp = timestampedEvent.getTimestamp();

  if (event.newAuction != null) {
    // Only want auctions in category 10.
    if (event.newAuction.category == 10) {
      // Join new auction with existing person, if any.
      Person person = newPersons.get(event.newAuction.seller);
      if (person != null) {
        addResult(event.newAuction, person, timestamp);
      } else {
        // Remember auction for future new person event.
        newAuctions.put(event.newAuction.seller, event.newAuction);
      }
    }
  } else {
    // Only want people in OR, ID or CA.
    if ("OR".equals(event.newPerson.state)
        || "ID".equals(event.newPerson.state)
        || "CA".equals(event.newPerson.state)) {
      // Join new person with existing auctions.
      for (Auction auction : newAuctions.get(event.newPerson.id)) {
        addResult(auction, event.newPerson, timestamp);
      }
      // We'll never need these auctions again.
      newAuctions.removeAll(event.newPerson.id);
      // Remember person for future auctions.
      newPersons.put(event.newPerson.id, event.newPerson);
    }
  }
}
 

@Override
public void run() {
  TimestampedValue<Event> timestampedEvent = nextInput();
  if (timestampedEvent == null) {
    allDone();
    return;
  }

  Event event = timestampedEvent.getValue();
  if (event.bid != null) {
    // Ignore bid events.
    // Keep looking for next events.
    return;
  }
  Instant timestamp = timestampedEvent.getTimestamp();
  Instant newWindowStart =
      windowStart(
          Duration.standardSeconds(configuration.windowSizeSec),
          Duration.standardSeconds(configuration.windowSizeSec),
          timestamp);
  if (!newWindowStart.equals(windowStart)) {
    // Retire this window.
    retirePersons();
    retireAuctions();
    windowStart = newWindowStart;
  }

  if (event.newAuction != null) {
    // Join new auction with existing person, if any.
    Person person = newPersons.get(event.newAuction.seller);
    if (person != null) {
      addResult(event.newAuction, person, timestamp);
    } else {
      // Remember auction for future new people.
      newAuctions.put(event.newAuction.seller, event.newAuction);
    }
  } else { // event is not an auction, nor a bid, so it is a person
    // Join new person with existing auctions.
    for (Auction auction : newAuctions.get(event.newPerson.id)) {
      addResult(auction, event.newPerson, timestamp);
    }
    // We'll never need these auctions again.
    newAuctions.removeAll(event.newPerson.id);
    // Remember person for future auctions.
    newPersons.put(event.newPerson.id, event.newPerson);
  }
}