Java Code Examples for org.joda.time.Instant#now()

@Test
public void testNonPollingGrowthTrackerCheckpointEmpty() {
  Instant now = Instant.now();
  PollResult<String> claim =
      PollResult.incomplete(
              Arrays.asList(
                  TimestampedValue.of("d", now.plus(standardSeconds(4))),
                  TimestampedValue.of("c", now.plus(standardSeconds(3))),
                  TimestampedValue.of("a", now.plus(standardSeconds(1))),
                  TimestampedValue.of("b", now.plus(standardSeconds(2)))))
          .withWatermark(now.plus(standardSeconds(7)));

  GrowthTracker<String, Integer> tracker = newTracker(NonPollingGrowthState.of(claim));

  NonPollingGrowthState<String> residual =
      (NonPollingGrowthState<String>) tracker.trySplit(0).getResidual();
  GrowthState primary = tracker.currentRestriction();
  tracker.checkDone();

  // Verify primary: should contain what the current tracker claimed, and nothing else.
  assertEquals(GrowthTracker.EMPTY_STATE, primary);

  // Verify residual: should contain what the current tracker didn't claim.
  assertEquals(claim, residual.getPending());
}
 

@Test
public void testAdvanceTimestamp() throws IOException, InterruptedException {
  final Instant timestamp = Instant.now();

  final TestUnboundedSource<String> source =
      TestUnboundedSource.<String>createBuilder()
          .addElements("before")
          .setTimestamp(timestamp)
          .addElements("after")
          .build();

  final UnboundedSourceSystem.Consumer<String, TestCheckpointMark> consumer =
      createConsumer(source);

  consumer.register(DEFAULT_SSP, NULL_STRING);
  consumer.start();
  assertEquals(
      Arrays.asList(
          createElementMessage(
              DEFAULT_SSP, offset(0), "before", BoundedWindow.TIMESTAMP_MIN_VALUE),
          createElementMessage(DEFAULT_SSP, offset(1), "after", timestamp)),
      consumeUntilTimeoutOrWatermark(consumer, DEFAULT_SSP, DEFAULT_TIMEOUT_MILLIS));
  consumer.stop();
}
 

@Override
public boolean advance() throws IOException {
  try {
    LOG.trace("MQTT reader (client ID {}) waiting message ...", client.getClientId());
    Message message = connection.receive(1, TimeUnit.SECONDS);
    if (message == null) {
      return false;
    }
    current = message.getPayload();
    currentTimestamp = Instant.now();
    checkpointMark.add(message, currentTimestamp);
  } catch (Exception e) {
    throw new IOException(e);
  }
  return true;
}
 

@Test
@Category({
  NeedsRunner.class,
  UsesImpulse.class,
  UsesStatefulParDo.class,
})
public void testOutputsProperElements() {
  Instant instant = Instant.now();

  Instant startTime = instant.minus(Duration.standardHours(100));
  long duration = 500;
  Duration interval = Duration.millis(250);
  long intervalMillis = interval.getMillis();
  Instant stopTime = startTime.plus(duration);

  PCollection<KV<Instant, Instant>> result =
      p.apply(
              Create.<PeriodicSequence.SequenceDefinition>of(
                  new PeriodicSequence.SequenceDefinition(startTime, stopTime, interval)))
          .apply(PeriodicSequence.create())
          .apply(ParDo.of(new ExtractTsDoFn<>())); // used to validate timestamp

  ArrayList<KV<Instant, Instant>> expectedResults =
      new ArrayList<>((int) (duration / intervalMillis + 1));
  for (long i = 0; i <= duration; i += intervalMillis) {
    Instant el = startTime.plus(i);
    expectedResults.add(KV.of(el, el));
  }

  PAssert.that(result).containsInAnyOrder(expectedResults);

  p.run().waitUntilFinish();
}
 

@Test
public void timerUpdateBuilderWithSetThenDeleteHasOnlyDeleted() {
  TimerUpdateBuilder builder = TimerUpdate.builder(null);
  Instant now = Instant.now();
  TimerData timer = TimerData.of(StateNamespaces.global(), now, now, TimeDomain.EVENT_TIME);

  TimerUpdate built = builder.setTimer(timer).deletedTimer(timer).build();

  assertThat(built.getSetTimers(), emptyIterable());
  assertThat(built.getDeletedTimers(), contains(timer));
}
 

@Test
public void testDecodeEncodeEqual() throws Exception {
  Instant now = Instant.now();
  ValueInSingleWindow<String> value =
      ValueInSingleWindow.of(
          "foo",
          now,
          new IntervalWindow(now, now.plus(Duration.standardSeconds(10))),
          PaneInfo.NO_FIRING);

  CoderProperties.coderDecodeEncodeEqual(
      ValueInSingleWindow.Coder.of(StringUtf8Coder.of(), IntervalWindow.getCoder()), value);
}
 

public void start(
    @Nullable Object key,
    Windmill.WorkItem work,
    Instant inputDataWatermark,
    @Nullable Instant outputDataWatermark,
    @Nullable Instant synchronizedProcessingTime,
    WindmillStateReader stateReader,
    StateFetcher stateFetcher,
    Windmill.WorkItemCommitRequest.Builder outputBuilder) {
  this.key = key;
  this.work = work;
  this.stateFetcher = stateFetcher;
  this.outputBuilder = outputBuilder;
  this.sideInputCache.clear();
  clearSinkFullHint();

  Instant processingTime = Instant.now();
  // Ensure that the processing time is greater than any fired processing time
  // timers.  Otherwise a trigger could ignore the timer and orphan the window.
  for (Windmill.Timer timer : work.getTimers().getTimersList()) {
    if (timer.getType() == Windmill.Timer.Type.REALTIME) {
      Instant inferredFiringTime =
          WindmillTimeUtils.windmillToHarnessTimestamp(timer.getTimestamp())
              .plus(Duration.millis(1));
      if (inferredFiringTime.isAfter(processingTime)) {
        processingTime = inferredFiringTime;
      }
    }
  }

  for (StepContext stepContext : getAllStepContexts()) {
    stepContext.start(
        stateReader,
        inputDataWatermark,
        processingTime,
        outputDataWatermark,
        synchronizedProcessingTime);
  }
}
 

@Test
public void testTimerInternalsProcessingTimeSkew() {
  Windmill.WorkItemCommitRequest.Builder outputBuilder =
      Windmill.WorkItemCommitRequest.newBuilder();

  NameContext nameContext = NameContextsForTests.nameContextForTest();
  DataflowOperationContext operationContext =
      executionContext.createOperationContext(nameContext);
  StreamingModeExecutionContext.StepContext stepContext =
      executionContext.getStepContext(operationContext);
  Windmill.WorkItem.Builder workItemBuilder =
      Windmill.WorkItem.newBuilder().setKey(ByteString.EMPTY).setWorkToken(17L);
  Windmill.Timer.Builder timerBuilder = workItemBuilder.getTimersBuilder().addTimersBuilder();

  // Trigger a realtime timer that with clock skew but ensure that it would
  // still fire.
  Instant now = Instant.now();
  long offsetMillis = 60 * 1000;
  Instant timerTimestamp = now.plus(offsetMillis);
  timerBuilder
      .setTag(ByteString.copyFromUtf8("a"))
      .setTimestamp(timerTimestamp.getMillis() * 1000)
      .setType(Windmill.Timer.Type.REALTIME);

  executionContext.start(
      "key",
      workItemBuilder.build(),
      new Instant(1000), // input watermark
      null, // output watermark
      null, // synchronized processing time
      stateReader,
      stateFetcher,
      outputBuilder);
  TimerInternals timerInternals = stepContext.timerInternals();
  assertTrue(timerTimestamp.isBefore(timerInternals.currentProcessingTime()));
}
 

@ProcessElement
public ProcessContinuation processElement(
    @Element SequenceDefinition srcElement,
    OutputReceiver<Instant> out,
    RestrictionTracker<OffsetRange, Long> restrictionTracker) {

  OffsetRange restriction = restrictionTracker.currentRestriction();
  Long interval = srcElement.durationMilliSec;
  Long nextOutput = restriction.getFrom() + interval;

  boolean claimSuccess = true;

  while (claimSuccess && Instant.ofEpochMilli(nextOutput).isBeforeNow()) {
    claimSuccess = restrictionTracker.tryClaim(nextOutput);
    if (claimSuccess) {
      Instant output = Instant.ofEpochMilli(nextOutput);
      out.outputWithTimestamp(output, output);
      nextOutput = nextOutput + interval;
    }
  }

  ProcessContinuation continuation = ProcessContinuation.stop();
  if (claimSuccess) {
    Duration offset = new Duration(Instant.now(), Instant.ofEpochMilli(nextOutput));
    continuation = ProcessContinuation.resume().withResumeDelay(offset);
  }
  return continuation;
}
 

private void advanceWatermark(long millis) {
  this.latestWatermark = new Instant(millis);
  Instant now = Instant.now();
  for (KeyManager m : keyManagers.values()) {
    m.advanceWatermark(latestWatermark, now);
  }
}
 

@Test
public void testTerminationConditionsEitherOf() {
  Instant now = Instant.now();
  Watch.Growth.AfterTotalOf<Object> a = Growth.afterTotalOf(standardSeconds(5));
  Watch.Growth.AfterTotalOf<Object> b = Growth.afterTotalOf(standardSeconds(10));

  Watch.Growth.BinaryCombined<
          Object, KV<Instant, ReadableDuration>, KV<Instant, ReadableDuration>>
      c = eitherOf(a, b);
  KV<KV<Instant, ReadableDuration>, KV<Instant, ReadableDuration>> state =
      c.forNewInput(now, null);
  assertFalse(c.canStopPolling(now.plus(standardSeconds(3)), state));
  assertTrue(c.canStopPolling(now.plus(standardSeconds(7)), state));
  assertTrue(c.canStopPolling(now.plus(standardSeconds(12)), state));
}
 

@Test
public void testResumeSetsTimer() throws Exception {
  DoFn<Integer, String> fn = new SelfInitiatedResumeFn();
  Instant base = Instant.now();
  dateTimeProvider.setDateTimeFixed(base.getMillis());
  ProcessFnTester<Integer, String, SomeRestriction, Void, Void> tester =
      new ProcessFnTester<>(
          base,
          fn,
          BigEndianIntegerCoder.of(),
          SerializableCoder.of(SomeRestriction.class),
          VoidCoder.of(),
          MAX_OUTPUTS_PER_BUNDLE,
          MAX_BUNDLE_DURATION);

  tester.startElement(42, new SomeRestriction());
  assertThat(tester.takeOutputElements(), contains("42"));

  // Should resume after 5 seconds: advancing by 3 seconds should have no effect.
  assertFalse(tester.advanceProcessingTimeBy(Duration.standardSeconds(3)));
  assertTrue(tester.takeOutputElements().isEmpty());

  // 6 seconds should be enough  should invoke the fn again.
  assertTrue(tester.advanceProcessingTimeBy(Duration.standardSeconds(3)));
  assertThat(tester.takeOutputElements(), contains("42"));

  // Should again resume after 5 seconds: advancing by 3 seconds should again have no effect.
  assertFalse(tester.advanceProcessingTimeBy(Duration.standardSeconds(3)));
  assertTrue(tester.takeOutputElements().isEmpty());

  // 6 seconds should again be enough.
  assertTrue(tester.advanceProcessingTimeBy(Duration.standardSeconds(3)));
  assertThat(tester.takeOutputElements(), contains("42"));
}
 

@Override
public Instant currentWatermark() {
  // Beyond bounds error checking isn't important since the runner is expected to perform
  // watermark bounds checking.
  Instant now = Instant.now();
  this.watermark = now.isAfter(watermark) ? now : watermark;
  return watermark;
}
 

@Test
public void testTrivialProcessFnPropagatesOutputWindowAndTimestamp() throws Exception {
  // Tests that ProcessFn correctly propagates the window and timestamp of the element
  // inside the KeyedWorkItem.
  // The underlying DoFn is actually monolithic, so this doesn't test splitting.
  DoFn<Integer, String> fn = new ToStringFn();

  Instant base = Instant.now();

  IntervalWindow w =
      new IntervalWindow(
          base.minus(Duration.standardMinutes(1)), base.plus(Duration.standardMinutes(1)));

  ProcessFnTester<Integer, String, SomeRestriction, Void, Void> tester =
      new ProcessFnTester<>(
          base,
          fn,
          BigEndianIntegerCoder.of(),
          SerializableCoder.of(SomeRestriction.class),
          VoidCoder.of(),
          MAX_OUTPUTS_PER_BUNDLE,
          MAX_BUNDLE_DURATION);
  tester.startElement(
      WindowedValue.of(
          KV.of(42, new SomeRestriction()),
          base,
          Collections.singletonList(w),
          PaneInfo.ON_TIME_AND_ONLY_FIRING));

  assertEquals(
      Arrays.asList(
          TimestampedValue.of("42a", base),
          TimestampedValue.of("42b", base),
          TimestampedValue.of("42c", base)),
      tester.peekOutputElementsInWindow(w));
}
 

@SuppressWarnings("unused") // For AvroCoder
private Checkpoint() {
    this.lastEmittedKey = 1;
    this.lastEmittedValue = 1;
    this.startTime = Instant.now();
}
 

@Override
public GetWorkStream getWorkStream(Windmill.GetWorkRequest request, WorkItemReceiver receiver) {
  LOG.debug("getWorkStream: {}", request.toString());
  Instant startTime = Instant.now();
  final CountDownLatch done = new CountDownLatch(1);
  return new GetWorkStream() {
    @Override
    public void close() {
      done.countDown();
    }

    @Override
    public boolean awaitTermination(int time, TimeUnit unit) throws InterruptedException {
      while (done.getCount() > 0) {
        Windmill.GetWorkResponse response = workToOffer.poll();
        if (response == null) {
          try {
            sleepMillis(500);
          } catch (InterruptedException e) {
            close();
            Thread.currentThread().interrupt();
          }
          continue;
        }
        for (Windmill.ComputationWorkItems computationWork : response.getWorkList()) {
          Instant inputDataWatermark =
              WindmillTimeUtils.windmillToHarnessWatermark(
                  computationWork.getInputDataWatermark());
          for (Windmill.WorkItem workItem : computationWork.getWorkList()) {
            receiver.receiveWork(
                computationWork.getComputationId(), inputDataWatermark, Instant.now(), workItem);
          }
        }
      }
      return done.await(time, unit);
    }

    @Override
    public Instant startTime() {
      return startTime;
    }
  };
}
 

/**
 * Returns total size of all records that remain in Kinesis stream. The size is estimated taking
 * into account size of the records that were added to the stream after timestamp of the most
 * recent record returned by the reader. If no records have yet been retrieved from the reader
 * {@link UnboundedSource.UnboundedReader#BACKLOG_UNKNOWN} is returned. When currently processed
 * record is not further behind than {@link #upToDateThreshold} then this method returns 0.
 *
 * <p>The method can over-estimate size of the records for the split as it reports the backlog
 * across all shards. This can lead to unnecessary decisions to scale up the number of workers but
 * will never fail to scale up when this is necessary due to backlog size.
 *
 * @see <a href="https://issues.apache.org/jira/browse/BEAM-9439">BEAM-9439</a>
 */
@Override
public long getSplitBacklogBytes() {
  Instant latestRecordTimestamp = shardReadersPool.getLatestRecordTimestamp();

  if (latestRecordTimestamp.equals(BoundedWindow.TIMESTAMP_MIN_VALUE)) {
    LOG.debug("Split backlog bytes for stream {} unknown", source.getStreamName());
    return UnboundedSource.UnboundedReader.BACKLOG_UNKNOWN;
  }

  if (latestRecordTimestamp.plus(upToDateThreshold).isAfterNow()) {
    LOG.debug(
        "Split backlog bytes for stream {} with latest record timestamp {}: 0 (latest record timestamp is up-to-date with threshold of {})",
        source.getStreamName(),
        latestRecordTimestamp,
        upToDateThreshold);
    return 0L;
  }

  if (backlogBytesLastCheckTime.plus(backlogBytesCheckThreshold).isAfterNow()) {
    LOG.debug(
        "Split backlog bytes for {} stream with latest record timestamp {}: {} (cached value)",
        source.getStreamName(),
        latestRecordTimestamp,
        lastBacklogBytes);
    return lastBacklogBytes;
  }

  try {
    lastBacklogBytes = kinesis.getBacklogBytes(source.getStreamName(), latestRecordTimestamp);
    backlogBytesLastCheckTime = Instant.now();
  } catch (TransientKinesisException e) {
    LOG.warn(
        "Transient exception occurred during backlog estimation for stream {}.",
        source.getStreamName(),
        e);
  }
  LOG.info(
      "Split backlog bytes for {} stream with {} latest record timestamp: {}",
      source.getStreamName(),
      latestRecordTimestamp,
      lastBacklogBytes);
  return lastBacklogBytes;
}
 

@Override
public Instant apply(Long input) {
  return Instant.now();
}
 

public Duration timeSinceLastServerActivity() {
  return new Duration(lastActivityTimestamp.get(), Instant.now());
}
 

@SuppressWarnings("checkstyle:methodname")
private void _process(final List<QualifiedName> qNames, final Supplier<ListenableFuture<Void>> supplier,
                      final String requestName, final boolean delete, final int queueSize) {
    if (isElasticSearchMetacatRefreshAlreadyRunning.compareAndSet(false, true)) {
        final long start = registry.clock().wallTime();
        try {
            log.info("Start: Full refresh of metacat index in elastic search. Processing {} ...", qNames);
            final MetacatRequestContext context = MetacatRequestContext.builder()
                .userName("admin")
                .clientAppName("elasticSearchRefresher")
                .apiUri("esRefresh")
                .scheme("internal")
                .build();
            MetacatContextManager.setContext(context);
            refreshMarker = Instant.now();
            refreshMarkerText = refreshMarker.toString();
            service = MoreExecutors
                .listeningDecorator(newFixedThreadPool(10, "elasticsearch-refresher-%d", queueSize));
            esService = MoreExecutors
                .listeningDecorator(newFixedThreadPool(5, "elasticsearch-refresher-es-%d", queueSize));
            defaultService = Executors.newSingleThreadExecutor();
            supplier.get().get(24, TimeUnit.HOURS);
            log.info("End: Full refresh of metacat index in elastic search");
            if (delete) {
                deleteUnmarkedEntities(qNames, config.getElasticSearchRefreshExcludeQualifiedNames());
            }
        } catch (Exception e) {
            log.error("Full refresh of metacat index failed", e);
            registry.counter(registry.createId(Metrics.CounterElasticSearchRefresh.getMetricName())
                .withTags(Metrics.tagStatusFailureMap)).increment();
        } finally {
            try {
                shutdown(service);
                shutdown(esService);
                shutdown(defaultService);
            } finally {
                isElasticSearchMetacatRefreshAlreadyRunning.set(false);
                final long duration = registry.clock().wallTime() - start;
                this.registry.timer(Metrics.TimerElasticSearchRefresh.getMetricName()
                    + "." + requestName).record(duration, TimeUnit.MILLISECONDS);
                log.info("### Time taken to complete {} is {} ms", requestName, duration);
            }
        }

    } else {
        log.info("Full refresh of metacat index is already running.");
        registry.counter(registry.createId(Metrics.CounterElasticSearchRefreshAlreadyRunning.getMetricName()))
            .increment();
    }
}