Java Code Examples for org.apache.beam.sdk.util.WindowedValue#timestampedValueInGlobalWindow()

private boolean tryProduceNext() {
  try {
    checkState(next == null, "unexpected non-null value for next");
    if (seekNext()) {
      next =
          WindowedValue.timestampedValueInGlobalWindow(
              reader.getCurrent(), reader.getCurrentTimestamp());
      return SUCCESSFULLY_OBTAINED_NEXT;
    } else {
      close();
      return FAILED_TO_OBTAIN_NEXT;
    }

  } catch (final Exception e) {
    throw new RuntimeException("Failed to read data.", e);
  }
}
 

/**
 * Creates a {@link Reader} for each {@link BoundedSource} and executes the {@link Reader}s read
 * loop. See {@link Reader} for further details of the read loop.
 *
 * <p>Propagates any exceptions caused during reading or processing via a consumer to the caller.
 */
public void runReadLoop(WindowedValue<InputT> value) throws Exception {
  try (Reader<OutputT> reader = value.getValue().createReader(pipelineOptions)) {
    if (!reader.start()) {
      // Reader has no data, immediately return
      return;
    }
    do {
      // TODO: Should this use the input window as the window for all the outputs?
      WindowedValue<OutputT> nextValue =
          WindowedValue.timestampedValueInGlobalWindow(
              reader.getCurrent(), reader.getCurrentTimestamp());
      for (FnDataReceiver<WindowedValue<OutputT>> consumer : consumers) {
        consumer.accept(nextValue);
      }
    } while (reader.advance());
  }
}
 

@Override
public Object readCurrent() {
  try {
    if (!isStarted) {
      isStarted = true;
      isCurrentAvailable = reader.start();
    } else {
      isCurrentAvailable = reader.advance();
    }
  } catch (final Exception e) {
    throw new RuntimeException(e);
  }

  if (isCurrentAvailable) {
    final O elem = reader.getCurrent();
    return WindowedValue.timestampedValueInGlobalWindow(elem, reader.getCurrentTimestamp());
  } else {
    throw new NoSuchElementException();
  }
}
 

@Override
public Object next() {
  if (currentReader == null) {
    return null;
  }
  try {
    Object item = currentReader.getCurrent();
    WindowedValue<Object> res =
        WindowedValue.timestampedValueInGlobalWindow(item, currentReader.getCurrentTimestamp());
    if (!currentReader.advance()) {
      nextShard();
    }
    return outputCoder == null
        ? res
        : Utils.encode(
            res, outputCoder); // todo: this is not nice, have done this only as a quick fix for
    // BoundedSourcePTest
  } catch (IOException e) {
    throw ExceptionUtil.rethrow(e);
  }
}
 

private boolean tryProduceNext() {
  try (Closeable ignored = MetricsEnvironment.scopedMetricsContainer(metricsContainer)) {
    if (closed) {
      return FAILED_TO_OBTAIN_NEXT;
    } else {
      checkState(next == null, "unexpected non-null value for next");
      if (seekNext()) {
        next =
            WindowedValue.timestampedValueInGlobalWindow(
                reader.getCurrent(), reader.getCurrentTimestamp());
        return SUCCESSFULLY_OBTAINED_NEXT;
      } else {
        close();
        return FAILED_TO_OBTAIN_NEXT;
      }
    }
  } catch (final Exception e) {
    throw new RuntimeException("Failed to read data.", e);
  }
}
 

@Override
protected WindowedValue<T> decodeMessage(Windmill.Message message) throws IOException {
  T value;
  InputStream data = message.getData().newInput();
  notifyElementRead(data.available());
  if (parseFn != null) {
    Pubsub.PubsubMessage pubsubMessage = Pubsub.PubsubMessage.parseFrom(data);
    value =
        parseFn.apply(
            new PubsubMessage(
                pubsubMessage.getData().toByteArray(),
                pubsubMessage.getAttributesMap(),
                pubsubMessage.getMessageId()));
  } else {
    value = coder.decode(data, Coder.Context.OUTER);
  }
  return WindowedValue.timestampedValueInGlobalWindow(
      value, WindmillTimeUtils.windmillToHarnessTimestamp(message.getTimestamp()));
}
 

/** The dummy stream created will only be used in Beam tests. */
private static InputDescriptor<OpMessage<String>, ?> createDummyStreamDescriptor(String id) {
  final GenericSystemDescriptor dummySystem =
      new GenericSystemDescriptor(id, InMemorySystemFactory.class.getName());
  final GenericInputDescriptor<OpMessage<String>> dummyInput =
      dummySystem.getInputDescriptor(id, new NoOpSerde<>());
  dummyInput.withOffsetDefault(SystemStreamMetadata.OffsetType.OLDEST);
  final Config config = new MapConfig(dummyInput.toConfig(), dummySystem.toConfig());
  final SystemFactory factory = new InMemorySystemFactory();
  final StreamSpec dummyStreamSpec = new StreamSpec(id, id, id, 1);
  factory.getAdmin(id, config).createStream(dummyStreamSpec);

  final SystemProducer producer = factory.getProducer(id, config, null);
  final SystemStream sysStream = new SystemStream(id, id);
  final Consumer<Object> sendFn =
      (msg) -> {
        producer.send(id, new OutgoingMessageEnvelope(sysStream, 0, null, msg));
      };
  final WindowedValue<String> windowedValue =
      WindowedValue.timestampedValueInGlobalWindow("dummy", new Instant());

  sendFn.accept(OpMessage.ofElement(windowedValue));
  sendFn.accept(new WatermarkMessage(BoundedWindow.TIMESTAMP_MAX_VALUE.getMillis()));
  sendFn.accept(new EndOfStreamMessage(null));
  return dummyInput;
}
 

@Test
public void addElementsAtEndOfTimeThrows() {
  Instant timestamp = BoundedWindow.TIMESTAMP_MAX_VALUE;
  WindowedValue<Integer> value = WindowedValue.timestampedValueInGlobalWindow(1, timestamp);

  UncommittedBundle<Integer> bundle = bundleFactory.createRootBundle();
  thrown.expect(IllegalArgumentException.class);
  thrown.expectMessage(timestamp.toString());
  bundle.add(value);
}
 

@Test
public void getElementsAfterAddShouldReturnAddedElements() {
  WindowedValue<Integer> firstValue = WindowedValue.valueInGlobalWindow(1);
  WindowedValue<Integer> secondValue =
      WindowedValue.timestampedValueInGlobalWindow(2, new Instant(1000L));

  afterCommitGetElementsShouldHaveAddedElements(ImmutableList.of(firstValue, secondValue));
}
 

@Test
public void updateWatermarkWithCompletedElementsNotPending() {
  WindowedValue<Integer> first = WindowedValue.timestampedValueInGlobalWindow(1, new Instant(22));
  CommittedBundle<Integer> createdBundle =
      bundleFactory.createBundle(createdInts).add(first).commit(clock.now());

  WindowedValue<Integer> second =
      WindowedValue.timestampedValueInGlobalWindow(2, new Instant(22));
  CommittedBundle<Integer> neverCreatedBundle =
      bundleFactory.createBundle(createdInts).add(second).commit(clock.now());

  manager.updateWatermarks(
      null,
      TimerUpdate.empty(),
      graph.getProducer(createdInts),
      null,
      Collections.<CommittedBundle<?>>singleton(createdBundle),
      BoundedWindow.TIMESTAMP_MAX_VALUE);

  manager.updateWatermarks(
      neverCreatedBundle,
      TimerUpdate.empty(),
      graph.getProducer(filtered),
      neverCreatedBundle.withElements(Collections.emptyList()),
      Collections.emptyList(),
      BoundedWindow.TIMESTAMP_MAX_VALUE);

  manager.refreshAll();
  TransformWatermarks filteredWms = manager.getWatermarks(graph.getProducer(filtered));
  assertThat(filteredWms.getInputWatermark(), equalTo(new Instant(22L)));
}
 

@Test
public void updateWatermarkWithUnprocessedElements() {
  WindowedValue<Integer> first = WindowedValue.valueInGlobalWindow(1);
  WindowedValue<Integer> second =
      WindowedValue.timestampedValueInGlobalWindow(2, new Instant(-1000L));
  WindowedValue<Integer> third =
      WindowedValue.timestampedValueInGlobalWindow(3, new Instant(1234L));
  CommittedBundle<Integer> createdBundle =
      bundleFactory
          .createBundle(createdInts)
          .add(first)
          .add(second)
          .add(third)
          .commit(clock.now());

  manager.updateWatermarks(
      null,
      TimerUpdate.empty(),
      graph.getProducer(createdInts),
      null,
      Collections.<CommittedBundle<?>>singleton(createdBundle),
      BoundedWindow.TIMESTAMP_MAX_VALUE);

  CommittedBundle<KV<String, Integer>> keyBundle =
      timestampedBundle(
          keyed, TimestampedValue.of(KV.of("MyKey", 1), BoundedWindow.TIMESTAMP_MIN_VALUE));
  manager.updateWatermarks(
      createdBundle,
      TimerUpdate.empty(),
      graph.getProducer(keyed),
      createdBundle.withElements(ImmutableList.of(second, third)),
      Collections.<CommittedBundle<?>>singleton(keyBundle),
      BoundedWindow.TIMESTAMP_MAX_VALUE);
  TransformWatermarks keyedWatermarks = manager.getWatermarks(graph.getProducer(keyed));
  // the unprocessed second and third are readded to pending
  assertThat(keyedWatermarks.getInputWatermark(), not(greaterThan(new Instant(-1000L))));
}
 

@SuppressWarnings("unchecked")
@Test
public void withElementsShouldReturnIndependentBundle() {
  WindowedValue<Integer> firstValue = WindowedValue.valueInGlobalWindow(1);
  WindowedValue<Integer> secondValue =
      WindowedValue.timestampedValueInGlobalWindow(2, new Instant(1000L));

  CommittedBundle<Integer> committed =
      afterCommitGetElementsShouldHaveAddedElements(ImmutableList.of(firstValue, secondValue));

  WindowedValue<Integer> firstReplacement =
      WindowedValue.of(
          9,
          new Instant(2048L),
          new IntervalWindow(new Instant(2044L), Instant.now()),
          PaneInfo.NO_FIRING);
  WindowedValue<Integer> secondReplacement =
      WindowedValue.timestampedValueInGlobalWindow(-1, Instant.now());
  CommittedBundle<Integer> withed =
      committed.withElements(ImmutableList.of(firstReplacement, secondReplacement));

  assertThat(withed.getElements(), containsInAnyOrder(firstReplacement, secondReplacement));
  assertThat(committed.getElements(), containsInAnyOrder(firstValue, secondValue));
  assertThat(withed.getKey(), equalTo(committed.getKey()));
  assertThat(withed.getPCollection(), equalTo(committed.getPCollection()));
  assertThat(
      withed.getSynchronizedProcessingOutputWatermark(),
      equalTo(committed.getSynchronizedProcessingOutputWatermark()));
  assertThat(withed.getMinimumTimestamp(), equalTo(new Instant(2048L)));
}
 

private void enqueueMessage(UnboundedReader reader) throws InterruptedException {
  @SuppressWarnings("unchecked")
  final T value = (T) reader.getCurrent();
  final Instant time = reader.getCurrentTimestamp();
  final SystemStreamPartition ssp = readerToSsp.get(reader);
  final WindowedValue<T> windowedValue =
      WindowedValue.timestampedValueInGlobalWindow(value, time);

  final OpMessage<T> opMessage = OpMessage.ofElement(windowedValue);
  final IncomingMessageEnvelope envelope =
      new IncomingMessageEnvelope(ssp, getOffset(reader), null, opMessage);

  queues.get(ssp).put(envelope);
}
 

private void enqueueMessage(BoundedReader<T> reader) throws InterruptedException {
  final T value = reader.getCurrent();
  final WindowedValue<T> windowedValue =
      WindowedValue.timestampedValueInGlobalWindow(value, reader.getCurrentTimestamp());
  final SystemStreamPartition ssp = readerToSsp.get(reader);
  final IncomingMessageEnvelope envelope =
      new IncomingMessageEnvelope(
          ssp, Long.toString(offset++), null, OpMessage.ofElement(windowedValue));

  available.acquire();
  queues.get(ssp).put(envelope);
}
 

@Override
public InternalRow get() {
  WindowedValue<T> windowedValue =
      WindowedValue.timestampedValueInGlobalWindow(
          reader.getCurrent(), reader.getCurrentTimestamp());
  return RowHelpers.storeWindowedValueInRow(windowedValue, source.getOutputCoder());
}
 

@Override
public InternalRow get() {
  WindowedValue<T> windowedValue =
      WindowedValue.timestampedValueInGlobalWindow(
          reader.getCurrent(), reader.getCurrentTimestamp());
  return RowHelpers.storeWindowedValueInRow(windowedValue, source.getOutputCoder());
}
 

private boolean tryProduceNext() {
  try {
    if (seekNext()) {
      next =
          WindowedValue.timestampedValueInGlobalWindow(
              reader.getCurrent(), reader.getCurrentTimestamp());
      return SUCCESSFULLY_OBTAINED_NEXT;
    } else {
      return FAILED_TO_OBTAIN_NEXT;
    }

  } catch (final Exception e) {
    throw new RuntimeException("Failed to read data.", e);
  }
}
 

@Test
public void addElementsPastEndOfTimeThrows() {
  Instant timestamp = BoundedWindow.TIMESTAMP_MAX_VALUE.plus(Duration.standardMinutes(2));
  WindowedValue<Integer> value = WindowedValue.timestampedValueInGlobalWindow(1, timestamp);

  UncommittedBundle<Integer> bundle = bundleFactory.createRootBundle();
  thrown.expect(IllegalArgumentException.class);
  thrown.expectMessage(timestamp.toString());
  bundle.add(value);
}
 

@Override
public WindowedValue<T> getCurrent() throws NoSuchElementException {
  return WindowedValue.timestampedValueInGlobalWindow(
      reader.getCurrent(), reader.getCurrentTimestamp());
}
 

/**
 * A terse alias for producing timestamped longs in the {@link GlobalWindow}, where the timestamp
 * is the epoch offset by the value of the element.
 */
private static WindowedValue<Long> tgw(Long elem) {
  return WindowedValue.timestampedValueInGlobalWindow(elem, new Instant(elem));
}