Java Code Examples for org.joda.time.Instant#minus()

public static String buildBigQueryProcessedUrlsQuery(IndexerPipelineOptions options) {
	String timeWindow = null;

	if (options.getProcessedUrlHistorySec() != null) {
		if (options.getProcessedUrlHistorySec() != Integer.MAX_VALUE) {
			Instant fromTime = Instant.now();
			fromTime = fromTime.minus(options.getProcessedUrlHistorySec() * 1000L);
			Integer fromDateId = IdConverterUtils.getDateIdFromTimestamp(fromTime.getMillis());
			timeWindow = "PublicationDateId >= " + fromDateId;
		}
	}

	if (timeWindow != null)
		timeWindow = "WHERE " + timeWindow;

	String result = "SELECT Url, MAX(ProcessingTime) AS ProcessingTime\n" + "FROM " + options.getBigQueryDataset()
			+ "." + WEBRESOURCE_TABLE + "\n" + timeWindow + "\n" + "GROUP BY Url";

	return result;
}
 

public static String buildBigQueryProcessedDocsQuery(IndexerPipelineOptions options) {
	String timeWindow = null;

	if (options.getProcessedUrlHistorySec() != null) {
		if (options.getProcessedUrlHistorySec() != Integer.MAX_VALUE) {
			Instant fromTime = Instant.now();
			fromTime = fromTime.minus(options.getProcessedUrlHistorySec() * 1000L);
			Integer fromDateId = IdConverterUtils.getDateIdFromTimestamp(fromTime.getMillis());
			timeWindow = "PublicationDateId >= " + fromDateId;
		}
	}

	if (timeWindow != null)
		timeWindow = "WHERE " + timeWindow;

	String result = "SELECT DocumentHash, MAX(ProcessingTime) AS ProcessingTime\n" + "FROM " + options.getBigQueryDataset()
			+ "." + DOCUMENT_TABLE + "\n" + timeWindow + "\n" + "GROUP BY DocumentHash";

	return result;
}
 

public static String buildBigQueryProcessedSocialCountsQuery(IndexerPipelineOptions options) {
	String timeWindow = null;

	if (options.getWrSocialCountHistoryWindowSec() != null) {
		if (options.getWrSocialCountHistoryWindowSec() != Integer.MAX_VALUE) {
			Instant fromTime = Instant.now();
			fromTime = fromTime.minus(options.getWrSocialCountHistoryWindowSec() * 1000L);
			Integer fromDateId = IdConverterUtils.getDateIdFromTimestamp(fromTime.getMillis());
			timeWindow = "WrPublicationDateId >= " + fromDateId;
		}
	}

	if (timeWindow != null)
		timeWindow = "WHERE " + timeWindow;

	String result = "SELECT WebResourceHash, MAX(CountTime) AS LastCountTime\n" + "FROM "
			+ options.getBigQueryDataset() + "." + WRSOCIALCOUNT_TABLE + "\n" + timeWindow + "\n"
			+ "GROUP BY WebResourceHash";

	return result;

}
 

@Test
public void testExplodeWindowsManyWindowsMultipleWindowedValues() {
  Instant now = Instant.now();
  BoundedWindow centerWindow = new IntervalWindow(now.minus(1000L), now.plus(1000L));
  BoundedWindow pastWindow = new IntervalWindow(now.minus(1500L), now.plus(500L));
  BoundedWindow futureWindow = new IntervalWindow(now.minus(500L), now.plus(1500L));
  BoundedWindow futureFutureWindow = new IntervalWindow(now, now.plus(2000L));
  PaneInfo pane = PaneInfo.createPane(false, false, Timing.ON_TIME, 3L, 0L);
  WindowedValue<String> value =
      WindowedValue.of(
          "foo",
          now,
          ImmutableList.of(pastWindow, centerWindow, futureWindow, futureFutureWindow),
          pane);

  assertThat(
      value.explodeWindows(),
      containsInAnyOrder(
          WindowedValue.of("foo", now, futureFutureWindow, pane),
          WindowedValue.of("foo", now, futureWindow, pane),
          WindowedValue.of("foo", now, centerWindow, pane),
          WindowedValue.of("foo", now, pastWindow, pane)));

  assertThat(value.isSingleWindowedValue(), equalTo(false));
}
 

@Override
public Instant getWatermark() {
  Instant now = Instant.now();
  Instant watermarkIdleThreshold =
      now.minus(watermarkParameters.getWatermarkIdleDurationThreshold());

  Instant newWatermark =
      watermarkParameters.getLastUpdateTime().isBefore(watermarkIdleThreshold)
          ? watermarkIdleThreshold
          : watermarkParameters.getEventTime();

  if (newWatermark.isAfter(watermarkParameters.getCurrentWatermark())) {
    watermarkParameters =
        watermarkParameters.toBuilder().setCurrentWatermark(newWatermark).build();
  }
  return watermarkParameters.getCurrentWatermark();
}
 

private <W extends BoundedWindow> void registerStateCleanup(
    WindowingStrategy<?, W> windowingStrategy, Collection<W> windowsToCleanup) {
  Coder<W> windowCoder = windowingStrategy.getWindowFn().windowCoder();

  for (W window : windowsToCleanup) {
    // The stepContext is the thing that know if it is batch or streaming, hence
    // whether state needs to be cleaned up or will simply be discarded so the
    // timer can be ignored

    Instant cleanupTime = earliestAllowableCleanupTime(window, windowingStrategy);
    // if DoFn has OnWindowExpiration then set holds for system timer.
    Instant cleanupOutputTimestamp =
        fnSignature.onWindowExpiration() == null ? cleanupTime : cleanupTime.minus(1L);
    stepContext.setStateCleanupTimer(
        CLEANUP_TIMER_ID, window, windowCoder, cleanupTime, cleanupOutputTimestamp);
  }
}
 

@VisibleForTesting
Instant getWatermark(PartitionContext ctx, Instant now) {
  if (maxEventTimestamp.isAfter(now)) {
    return now.minus(maxDelay); // (a) above.
  } else if (ctx.getMessageBacklog() == 0
      && ctx.getBacklogCheckTime().minus(maxDelay).isAfter(maxEventTimestamp) // Idle
      && maxEventTimestamp.getMillis() > 0) { // Read at least one record with positive timestamp.
    return ctx.getBacklogCheckTime().minus(maxDelay);
  } else {
    return maxEventTimestamp.minus(maxDelay);
  }
}
 

@Test
@Category({
  NeedsRunner.class,
  UsesImpulse.class,
  UsesStatefulParDo.class,
})
public void testOutputsProperElements() {
  Instant instant = Instant.now();

  Instant startTime = instant.minus(Duration.standardHours(100));
  long duration = 500;
  Duration interval = Duration.millis(250);
  long intervalMillis = interval.getMillis();
  Instant stopTime = startTime.plus(duration);

  PCollection<KV<Instant, Instant>> result =
      p.apply(
              Create.<PeriodicSequence.SequenceDefinition>of(
                  new PeriodicSequence.SequenceDefinition(startTime, stopTime, interval)))
          .apply(PeriodicSequence.create())
          .apply(ParDo.of(new ExtractTsDoFn<>())); // used to validate timestamp

  ArrayList<KV<Instant, Instant>> expectedResults =
      new ArrayList<>((int) (duration / intervalMillis + 1));
  for (long i = 0; i <= duration; i += intervalMillis) {
    Instant el = startTime.plus(i);
    expectedResults.add(KV.of(el, el));
  }

  PAssert.that(result).containsInAnyOrder(expectedResults);

  p.run().waitUntilFinish();
}
 

@Test
@Category({
  NeedsRunner.class,
  UsesImpulse.class,
  UsesStatefulParDo.class,
})
public void testOutputsProperElements() {
  Instant instant = Instant.now();

  Instant startTime = instant.minus(Duration.standardHours(100));
  long duration = 500;
  Duration interval = Duration.millis(250);
  long intervalMillis = interval.getMillis();
  Instant stopTime = startTime.plus(duration);

  PCollection<KV<Instant, Instant>> result =
      p.apply(PeriodicImpulse.create().startAt(startTime).stopAt(stopTime).withInterval(interval))
          .apply(ParDo.of(new ExtractTsDoFn<>()));

  ArrayList<KV<Instant, Instant>> expectedResults =
      new ArrayList<>((int) (duration / intervalMillis + 1));
  for (long i = 0; i <= duration; i += intervalMillis) {
    Instant el = startTime.plus(i);
    expectedResults.add(KV.of(el, el));
  }

  PAssert.that(result).containsInAnyOrder(expectedResults);

  p.run().waitUntilFinish();
}
 

@Test
public void testExplodeWindowsInOneWindowEquals() {
  Instant now = Instant.now();
  BoundedWindow window = new IntervalWindow(now.minus(1000L), now.plus(1000L));
  WindowedValue<String> value =
      WindowedValue.of("foo", now, window, PaneInfo.ON_TIME_AND_ONLY_FIRING);

  assertThat(Iterables.getOnlyElement(value.explodeWindows()), equalTo(value));
}
 

@Override
public boolean advance() {
  currentOffset++;

  processingTime = new Instant();
  eventTime = processingTime.minus(sourceOptions.nextProcessingTimeDelay(currentOffset));

  SyntheticSourceOptions.Record record =
      getCurrentSource().sourceOptions.genRecord(currentOffset);
  currentKVPair = record.kv;

  delay.delayRecord(record);

  return currentOffset < source.endOffset;
}
 

@VisibleForTesting
Instant getWatermark(PartitionContext ctx, Instant now) {
  if (maxEventTimestamp.isAfter(now)) {
    return now.minus(maxDelay); // (a) above.
  } else if (ctx.getMessageBacklog() == 0
      && ctx.getBacklogCheckTime().minus(maxDelay).isAfter(maxEventTimestamp) // Idle
      && maxEventTimestamp.getMillis() > 0) { // Read at least one record with positive timestamp.
    return ctx.getBacklogCheckTime().minus(maxDelay);
  } else {
    return maxEventTimestamp.minus(maxDelay);
  }
}
 

@Test
public void testTrivialProcessFnPropagatesOutputWindowAndTimestamp() throws Exception {
  // Tests that ProcessFn correctly propagates the window and timestamp of the element
  // inside the KeyedWorkItem.
  // The underlying DoFn is actually monolithic, so this doesn't test splitting.
  DoFn<Integer, String> fn = new ToStringFn();

  Instant base = Instant.now();

  IntervalWindow w =
      new IntervalWindow(
          base.minus(Duration.standardMinutes(1)), base.plus(Duration.standardMinutes(1)));

  ProcessFnTester<Integer, String, SomeRestriction, Void, Void> tester =
      new ProcessFnTester<>(
          base,
          fn,
          BigEndianIntegerCoder.of(),
          SerializableCoder.of(SomeRestriction.class),
          VoidCoder.of(),
          MAX_OUTPUTS_PER_BUNDLE,
          MAX_BUNDLE_DURATION);
  tester.startElement(
      WindowedValue.of(
          KV.of(42, new SomeRestriction()),
          base,
          Collections.singletonList(w),
          PaneInfo.ON_TIME_AND_ONLY_FIRING));

  assertEquals(
      Arrays.asList(
          TimestampedValue.of("42a", base),
          TimestampedValue.of("42b", base),
          TimestampedValue.of("42c", base)),
      tester.peekOutputElementsInWindow(w));
}
 

@Override
public IntervalWindow decode(InputStream inStream) throws IOException, CoderException {
  Instant end = instantCoder.decode(inStream);
  ReadableDuration duration = durationCoder.decode(inStream);
  return new IntervalWindow(end.minus(duration), end);
}
 

/**
 * Generates a {@link TestStream} of the given duration containing the values [0, numElements) and
 * the same number of random but monotonic watermark updates, with each element within
 * allowedLateness of the respective watermark update.
 *
 * <p>TODO: Consider moving this into TestStream if it's useful enough.
 */
private PCollection<Long> generateStreamWithBoundedDisorder(
    String name,
    Instant base,
    Duration totalDuration,
    int numElements,
    Duration allowedLateness) {
  TestStream.Builder<Long> stream = TestStream.create(VarLongCoder.of());

  // Generate numElements random watermark updates. After each one also generate an element within
  // allowedLateness of it.
  List<Instant> watermarks = Lists.newArrayList();
  for (int i = 0; i < numElements; ++i) {
    watermarks.add(base.plus(new Duration((long) (totalDuration.getMillis() * Math.random()))));
  }
  Collections.sort(watermarks);

  List<Event<Long>> events = Lists.newArrayList();
  for (int i = 0; i < numElements; ++i) {
    Instant processingTimestamp =
        base.plus((long) (1.0 * i * totalDuration.getMillis() / (numElements + 1)));
    Instant watermark = watermarks.get(i);
    Instant elementTimestamp =
        watermark.minus((long) (Math.random() * allowedLateness.getMillis()));
    events.add(new Event<>(processingTimestamp, watermark));
    events.add(new Event<>(processingTimestamp, TimestampedValue.of((long) i, elementTimestamp)));
  }

  Instant lastProcessingTime = base;
  for (Event<Long> event : events) {
    Duration processingTimeDelta = new Duration(lastProcessingTime, event.processingTime);
    if (processingTimeDelta.getMillis() > 0) {
      stream = stream.advanceProcessingTime(processingTimeDelta);
    }
    lastProcessingTime = event.processingTime;

    if (event.element != null) {
      stream = stream.addElements(event.element);
    } else {
      stream = stream.advanceWatermarkTo(event.watermarkUpdate);
    }
  }
  return p.apply(name, stream.advanceWatermarkToInfinity());
}
 

private Instant eventTime(long offset, Instant processingTime) {
  return processingTime.minus(options.nextProcessingTimeDelay(offset));
}