Java Code Examples for org.joda.time.Duration#getMillis()

/**
 * Check if topics exist.
 *
 * @param project GCP project identifier.
 * @param timeoutDuration Joda duration that sets a period of time before checking times out.
 */
public void checkIfAnySubscriptionExists(String project, Duration timeoutDuration)
    throws InterruptedException, IllegalArgumentException, IOException, TimeoutException {
  if (timeoutDuration.getMillis() <= 0) {
    throw new IllegalArgumentException(String.format("timeoutDuration should be greater than 0"));
  }

  DateTime startTime = new DateTime();
  int sizeOfSubscriptionList = 0;
  while (sizeOfSubscriptionList == 0
      && Seconds.secondsBetween(new DateTime(), startTime).getSeconds()
          < timeoutDuration.toStandardSeconds().getSeconds()) {
    // Sleep 1 sec
    Thread.sleep(1000);
    sizeOfSubscriptionList =
        listSubscriptions(projectPathFromPath(String.format("projects/%s", project)), topicPath())
            .size();
  }

  if (sizeOfSubscriptionList > 0) {
    return;
  } else {
    throw new TimeoutException("Timed out when checking if topics exist for " + topicPath());
  }
}
 

/**
 * 获得两个时间点之间的时间跨度
 * 
 * @param time1
 *            开始的时间点
 * @param time2
 *            结束的时间点
 * @param timeUnit
 *            跨度的时间单位 see {@link JodaTime}
 *            （支持的时间单位有DAY,HOUR,MINUTE,SECOND,MILLI）
 */
public static long lengthBetween(DateTime time1, DateTime time2,
		DurationFieldType timeUnit) {
	Duration duration = Days.daysBetween(time1, time2).toStandardDuration();
	if (timeUnit == JodaTime.DAY) {
		return duration.getStandardDays();
	} else if (timeUnit == JodaTime.HOUR) {
		return duration.getStandardHours();
	} else if (timeUnit == JodaTime.MINUTE) {
		return duration.getStandardMinutes();
	} else if (timeUnit == JodaTime.SECOND) {
		return duration.getStandardSeconds();
	} else if (timeUnit == JodaTime.MILLI) {
		return duration.getMillis();
	} else {
		throw new RuntimeException(
				"TimeUnit not supported except DAY,HOUR,MINUTE,SECOND,MILLI");
	}
}
 

@Override
public PollResult<OutputT> apply(InputT element, Context c) throws Exception {
  Instant now = Instant.now();
  Duration elapsed = new Duration(baseTime, Instant.now());
  if (elapsed.isLongerThan(timeToFail)) {
    fail(
        String.format(
            "Poll called %s after base time, which is longer than the threshold of %s",
            elapsed, timeToFail));
  }

  double fractionElapsed = 1.0 * elapsed.getMillis() / timeToOutputEverything.getMillis();
  int numToEmit = (int) Math.min(outputs.size(), fractionElapsed * outputs.size());
  List<TimestampedValue<OutputT>> toEmit = Lists.newArrayList();
  for (int i = 0; i < numToEmit; ++i) {
    toEmit.add(TimestampedValue.of(outputs.get(i), now));
  }
  return elapsed.isLongerThan(timeToDeclareOutputFinal)
      ? PollResult.complete(toEmit)
      : PollResult.incomplete(toEmit).withWatermark(now);
}
 

private String getReconnectionTimerValue() {
  if (endSessionRequestTime.isPresent()) {
    Duration difference = new Duration(endSessionRequestTime.get(), Instant.now());
    return "<" + difference.getMillis() / 1000.0f + "s>";
  }
  return "<Timer not set>";
}
 

private KV<byte[], byte[]> outputElement(
    byte[] inputKey, byte[] inputValue, long inputValueHashcode, int index, Random random) {

  long seed = options.hashFunction().hashLong(inputValueHashcode + index).asLong();
  Duration delay = Duration.millis(options.nextDelay(seed));
  long millisecondsSpentSleeping = 0;

  while (delay.getMillis() > 0) {
    millisecondsSpentSleeping +=
        delay(delay, options.cpuUtilizationInMixedDelay, options.delayType, random);

    if (isWithinThroughputLimit()) {
      break;
    } else {
      // try an extra delay of 1 millisecond
      delay = Duration.millis(1);
    }
  }

  reportThrottlingTimeMetrics(millisecondsSpentSleeping);

  if (options.preservesInputKeyDistribution) {
    // Generate the new byte array value whose hashcode will be
    // used as seed to initialize a Random object in next stages.
    byte[] newValue = new byte[inputValue.length];
    random.nextBytes(newValue);
    return KV.of(inputKey, newValue);
  } else {
    return options.genKvPair(seed);
  }
}
 

public static <T> Stream<T> adaptStream(Stream<T> stream, Function<? super T, ? extends DateTime> timestamp_fn, Duration stepsize) {
    if (stepsize.getMillis() <= 0L) return stream;

    final Iterator<T> iter = new SkippingIterator<>(stream.iterator(), timestamp_fn, stepsize);
    final Spliterator<T> spliter = Spliterators.spliteratorUnknownSize(iter, NONNULL | IMMUTABLE | ORDERED);
    return StreamSupport.stream(spliter, false);
}
 

/**
 * Validates that a given time is valid for the total time slept by a BackOff given the number of
 * retries and an initial polling interval.
 *
 * @param pollingInterval The initial polling interval given.
 * @param retries The number of retries made
 * @param timeSleptMillis The amount of time slept by the clock. This is checked against the valid
 *     interval.
 */
private void checkValidInterval(Duration pollingInterval, int retries, long timeSleptMillis) {
  long highSum = 0;
  long lowSum = 0;
  for (int i = 0; i < retries; i++) {
    double currentInterval =
        pollingInterval.getMillis() * Math.pow(DataflowPipelineJob.DEFAULT_BACKOFF_EXPONENT, i);
    double randomOffset = 0.5 * currentInterval;
    highSum += Math.round(currentInterval + randomOffset);
    lowSum += Math.round(currentInterval - randomOffset);
  }
  assertThat(timeSleptMillis, allOf(greaterThanOrEqualTo(lowSum), lessThanOrEqualTo(highSum)));
}
 

@Test
@Category(NeedsRunner.class)
public void testUnboundedSourceRateSplits() throws Exception {
  int elementsPerPeriod = 10;
  Duration period = Duration.millis(5);

  long numElements = 1000;
  int numSplits = 10;

  UnboundedCountingSource initial =
      CountingSource.createUnboundedFrom(0).withRate(elementsPerPeriod, period);
  List<? extends UnboundedSource<Long, ?>> splits = initial.split(numSplits, p.getOptions());
  assertEquals("Expected exact splitting", numSplits, splits.size());

  long elementsPerSplit = numElements / numSplits;
  assertEquals("Expected even splits", numElements, elementsPerSplit * numSplits);
  PCollectionList<Long> pcollections = PCollectionList.empty(p);
  for (int i = 0; i < splits.size(); ++i) {
    pcollections =
        pcollections.and(
            p.apply("split" + i, Read.from(splits.get(i)).withMaxNumRecords(elementsPerSplit)));
  }
  PCollection<Long> input = pcollections.apply(Flatten.pCollections());

  addCountingAsserts(input, numElements);
  Instant startTime = Instant.now();
  p.run();
  Instant endTime = Instant.now();
  // 500 ms if the readers are all initialized in parallel; 5000 ms if they are evaluated serially
  long expectedMinimumMillis = (numElements * period.getMillis()) / elementsPerPeriod;
  assertThat(expectedMinimumMillis, lessThan(endTime.getMillis() - startTime.getMillis()));
}
 

@Test
@Category({
  NeedsRunner.class,
  UsesImpulse.class,
  UsesStatefulParDo.class,
})
public void testOutputsProperElements() {
  Instant instant = Instant.now();

  Instant startTime = instant.minus(Duration.standardHours(100));
  long duration = 500;
  Duration interval = Duration.millis(250);
  long intervalMillis = interval.getMillis();
  Instant stopTime = startTime.plus(duration);

  PCollection<KV<Instant, Instant>> result =
      p.apply(
              Create.<PeriodicSequence.SequenceDefinition>of(
                  new PeriodicSequence.SequenceDefinition(startTime, stopTime, interval)))
          .apply(PeriodicSequence.create())
          .apply(ParDo.of(new ExtractTsDoFn<>())); // used to validate timestamp

  ArrayList<KV<Instant, Instant>> expectedResults =
      new ArrayList<>((int) (duration / intervalMillis + 1));
  for (long i = 0; i <= duration; i += intervalMillis) {
    Instant el = startTime.plus(i);
    expectedResults.add(KV.of(el, el));
  }

  PAssert.that(result).containsInAnyOrder(expectedResults);

  p.run().waitUntilFinish();
}
 

@Test
@Category({
  NeedsRunner.class,
  UsesImpulse.class,
  UsesStatefulParDo.class,
})
public void testOutputsProperElements() {
  Instant instant = Instant.now();

  Instant startTime = instant.minus(Duration.standardHours(100));
  long duration = 500;
  Duration interval = Duration.millis(250);
  long intervalMillis = interval.getMillis();
  Instant stopTime = startTime.plus(duration);

  PCollection<KV<Instant, Instant>> result =
      p.apply(PeriodicImpulse.create().startAt(startTime).stopAt(stopTime).withInterval(interval))
          .apply(ParDo.of(new ExtractTsDoFn<>()));

  ArrayList<KV<Instant, Instant>> expectedResults =
      new ArrayList<>((int) (duration / intervalMillis + 1));
  for (long i = 0; i <= duration; i += intervalMillis) {
    Instant el = startTime.plus(i);
    expectedResults.add(KV.of(el, el));
  }

  PAssert.that(result).containsInAnyOrder(expectedResults);

  p.run().waitUntilFinish();
}
 

public int getPercentComplete() {
    Duration eta = eta();
    if (eta.getMillis() == 0) {
        return 0;
    }
    if (eta.isShorterThan(getElapsedTime())) {
        return 100;
    }
    return (int) ((getElapsedTime().getMillis() * 100) / eta.getMillis());
}
 

@Test
public void testMergingWatermarkHoldAndLateDataFuzz() throws Exception {
  MetricsContainerImpl container = new MetricsContainerImpl("any");
  MetricsEnvironment.setCurrentContainer(container);
  // Test handling of late data. Specifically, ensure the watermark hold is correct.
  Duration allowedLateness = Duration.standardMinutes(100);
  long seed = ThreadLocalRandom.current().nextLong();
  LOG.info("Random seed: {}", seed);
  Random r = new Random(seed);

  Duration gapDuration = Duration.millis(10 + r.nextInt(40));
  LOG.info("Gap duration {}", gapDuration);

  ReduceFnTester<Integer, Iterable<Integer>, IntervalWindow> tester =
      ReduceFnTester.nonCombining(
          WindowingStrategy.of(Sessions.withGapDuration(gapDuration))
              .withMode(AccumulationMode.DISCARDING_FIRED_PANES)
              .withTrigger(
                  Repeatedly.forever(
                      AfterWatermark.pastEndOfWindow()
                          .withLateFirings(AfterPane.elementCountAtLeast(1))))
              .withAllowedLateness(allowedLateness));
  tester.setAutoAdvanceOutputWatermark(true);

  // Input watermark -> null
  assertEquals(null, tester.getWatermarkHold());
  assertEquals(null, tester.getOutputWatermark());

  // All on time data, verify watermark hold.
  List<Integer> times = new ArrayList<>();

  int numTs = 3 + r.nextInt(100);
  int maxTs = 1 + r.nextInt(400);
  LOG.info("Num ts {}", numTs);
  LOG.info("Max ts {}", maxTs);
  for (int i = numTs; i >= 0; --i) {
    times.add(r.nextInt(maxTs));
  }
  LOG.info("Times: {}", times);

  int split = 0;
  long watermark = 0;
  while (split < times.size()) {
    int nextSplit = split + r.nextInt(times.size());
    if (nextSplit > times.size()) {
      nextSplit = times.size();
    }
    LOG.info("nextSplit {}", nextSplit);
    injectElements(tester, times.subList(split, nextSplit));
    if (r.nextInt(3) == 0) {
      int nextWatermark = r.nextInt((int) (maxTs + gapDuration.getMillis()));
      if (nextWatermark > watermark) {
        Boolean enabled = r.nextBoolean();
        LOG.info("nextWatermark {} {}", nextWatermark, enabled);
        watermark = nextWatermark;
        tester.setAutoAdvanceOutputWatermark(enabled);
        tester.advanceInputWatermark(new Instant(watermark));
      }
    }
    split = nextSplit;
    Instant hold = tester.getWatermarkHold();
    if (hold != null) {
      assertThat(hold, greaterThanOrEqualTo(new Instant(watermark)));
      assertThat(watermark, lessThan(maxTs + gapDuration.getMillis()));
    }
  }
  tester.setAutoAdvanceOutputWatermark(true);
  watermark = gapDuration.getMillis() + maxTs;
  tester.advanceInputWatermark(new Instant(watermark));
  LOG.info("Output {}", tester.extractOutput());
  if (tester.getWatermarkHold() != null) {
    assertThat(tester.getWatermarkHold(), equalTo(new Instant(watermark).plus(allowedLateness)));
  }
  // Nothing dropped.
  long droppedElements =
      container
          .getCounter(
              MetricName.named(ReduceFnRunner.class, ReduceFnRunner.DROPPED_DUE_TO_CLOSED_WINDOW))
          .getCumulative()
          .longValue();
  assertEquals(0, droppedElements);
}
 

private CommitDeadlineSettingInterceptor(Duration commitDeadline) {
  this.commitDeadlineMilliseconds = commitDeadline.getMillis();
}
 

@Override
public void setAllowedLateness(Duration allowedLateness)
{
  this.allowedLatenessMillis = allowedLateness.getMillis();
}
 

public Script(String command, Duration timeout) {
    this(command, timeout.getMillis(), s_logger);
}
 

/**
 * Determines change points based on run length vs terciles. Typically used with diff
 * series.
 *
 * @param df time series
 * @param minDuration time series
 * @return change points
 */
public static TreeSet<Long> getChangePointsTercileMethod(DataFrame df, Duration minDuration) {
  if (df.isEmpty()) {
    return new TreeSet<>();
  }

  LongSeries time = df.getLongs(COL_TIME);
  DoubleSeries value = fastBSpline(df.getDoubles(COL_VALUE), FAST_SPLINE_ITERATIONS);

  TreeSet<Long> changePoints = new TreeSet<>();

  int lastChange = 0;
  int runStart = 0;
  int runSide = 0;
  for (int i = 0; i < df.size(); i++) {
    if (!value.isNull(i) && lastChange < i) {
      DoubleSeries history = value.slice(lastChange, i);
      double upTercile = history.quantile(0.666).doubleValue();
      double downTercile = history.quantile(0.333).doubleValue();

      double val = value.getDouble(i);

      int side = 0;
      if (val > upTercile)
        side = 1;
      if (val < downTercile)
        side = -1;

      // run side changed or last run
      if (runSide != side || i >= df.size() - 1) {
        long past = time.getLong(runStart);
        long now = time.getLong(i);
        long runDuration = now - past;
        if (runSide != 0 && runDuration >= minDuration.getMillis()) {
          lastChange = runStart;
          changePoints.add(past);
        }
        runStart = i;
        runSide = side;
      }
    }
  }

  return changePoints;
}
 

@Override
public State waitUntilFinish(final Duration duration) {
  return (State) super.waitUntilJobFinish(duration.getMillis(), TimeUnit.MILLISECONDS);
}
 

public static duration_msec encodeDuration(Duration d) {
    return new duration_msec(d.getMillis());
}
 

@Override
@Nullable
public Long convertToDatabaseColumn(@Nullable Duration duration) {
  return duration == null ? null : duration.getMillis();
}
 

@Override
public HtmlComponent createComponent(Object object, Class type) {
    StringBuilder result = new StringBuilder();
    if (object != null) {
        Duration duration = (Duration) object;
        PeriodFormatterBuilder periodFormatterBuilder = new PeriodFormatterBuilder();

        PeriodType periodType = PeriodType.forFields(getDurationFieldTypes());

        if (printZeroAlways) {
            periodFormatterBuilder.printZeroAlways();
        }
        if (duration.getMillis() < 0) {
            result.append("-");
        }

        MutablePeriod period = new MutablePeriod(Math.abs(duration.getMillis()), periodType);

        if (period.isSupported(DurationFieldType.years())) {
            periodFormatterBuilder.appendYears().appendSuffix(
                    RenderUtils.getResourceString(enumerationBundle, CLASS_NAME + DurationFieldType.years().getName()
                            + SHORT));
        }
        if (period.isSupported(DurationFieldType.months())) {
            periodFormatterBuilder.appendMonths().appendSuffix(
                    RenderUtils.getResourceString(enumerationBundle, CLASS_NAME + DurationFieldType.months().getName()
                            + SHORT));
        }
        if (period.isSupported(DurationFieldType.weeks())) {
            periodFormatterBuilder.appendWeeks().appendSuffix(
                    RenderUtils.getResourceString(enumerationBundle, CLASS_NAME + DurationFieldType.weeks().getName()
                            + SHORT));
        }
        if (period.isSupported(DurationFieldType.days())) {
            periodFormatterBuilder.appendDays().appendSuffix(
                    RenderUtils.getResourceString(enumerationBundle, CLASS_NAME + DurationFieldType.days().getName()
                            + SHORT));
        }
        if (period.isSupported(DurationFieldType.hours())) {
            periodFormatterBuilder.appendHours().appendSuffix(
                    RenderUtils.getResourceString(enumerationBundle, CLASS_NAME + DurationFieldType.hours().getName()
                            + SHORT));
        }
        if (period.isSupported(DurationFieldType.minutes())) {
            periodFormatterBuilder
                    .minimumPrintedDigits(2)
                    .appendMinutes()
                    .appendSuffix(
                            RenderUtils.getResourceString(enumerationBundle, CLASS_NAME
                                    + DurationFieldType.minutes().getName() + SHORT));
        }
        if (period.isSupported(DurationFieldType.seconds())) {
            periodFormatterBuilder
                    .minimumPrintedDigits(2)
                    .appendSeconds()
                    .appendSuffix(
                            RenderUtils.getResourceString(enumerationBundle, CLASS_NAME
                                    + DurationFieldType.seconds().getName() + SHORT));
        }
        if (period.isSupported(DurationFieldType.millis())) {
            periodFormatterBuilder
                    .minimumPrintedDigits(2)
                    .appendMillis()
                    .appendSuffix(
                            RenderUtils.getResourceString(enumerationBundle, CLASS_NAME
                                    + DurationFieldType.millis().getName() + SHORT));
        }
        result.append(periodFormatterBuilder.toFormatter().print(period));
    }
    return new HtmlText(result.toString());
}