Java Code Examples for org.apache.beam.sdk.util.BackOff#nextBackOffMillis()

The following examples show how to use org.apache.beam.sdk.util.BackOff#nextBackOffMillis() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: BoundedReadFromUnboundedSource.java    From beam with Apache License 2.0 6 votes vote down vote up 
private boolean advanceWithBackoff(UnboundedReader<T> reader, Instant endTime)
    throws IOException {
  // Try reading from the source with exponential backoff
  BackOff backoff = BACKOFF_FACTORY.backoff();
  long nextSleep = backoff.nextBackOffMillis();
  while (true) {
    if (nextSleep == BackOff.STOP || (endTime != null && Instant.now().isAfter(endTime))) {
      return false;
    }
    if (reader.advance()) {
      return true;
    }
    Uninterruptibles.sleepUninterruptibly(nextSleep, TimeUnit.MILLISECONDS);
    nextSleep = backoff.nextBackOffMillis();
  }
}
Example 2
Source File: MicrobatchSource.java    From beam with Apache License 2.0 6 votes vote down vote up 
private boolean advanceWithBackoff() throws IOException {
  // Try reading from the source with exponential backoff
  final BackOff backoff = backoffFactory.backoff();
  long nextSleep = backoff.nextBackOffMillis();
  while (nextSleep != BackOff.STOP) {
    if (readEndTime != null && Instant.now().isAfter(readEndTime)) {
      finalizeCheckpoint();
      return false;
    }
    if (unboundedReader.advance()) {
      recordsRead++;
      return true;
    }
    Uninterruptibles.sleepUninterruptibly(nextSleep, TimeUnit.MILLISECONDS);
    nextSleep = backoff.nextBackOffMillis();
  }
  finalizeCheckpoint();
  return false;
}
Example 3
Source File: WorkerCustomSources.java    From beam with Apache License 2.0 6 votes vote down vote up 
@Override
public boolean advance() throws IOException {
  if (elemsRead >= maxUnboundedBundleSize
      || Instant.now().isAfter(endTime)
      || context.isSinkFullHintSet()) {
    return false;
  }

  BackOff backoff = BACKOFF_FACTORY.backoff();
  while (true) {
    try {
      if (reader.advance()) {
        elemsRead++;
        return true;
      }
    } catch (Exception e) {
      throw new IOException("Failed to advance source: " + reader.getCurrentSource(), e);
    }
    long nextBackoff = backoff.nextBackOffMillis();
    if (nextBackoff == BackOff.STOP) {
      return false;
    }
    Uninterruptibles.sleepUninterruptibly(nextBackoff, TimeUnit.MILLISECONDS);
  }
}
Example 4
Source File: BackOffAdapter.java    From beam with Apache License 2.0 5 votes vote down vote up 
/**
 * Returns an adapter to convert from {@link BackOff} to {@link
 * com.google.api.client.util.BackOff}.
 */
public static com.google.api.client.util.BackOff toGcpBackOff(final BackOff backOff) {
  return new com.google.api.client.util.BackOff() {
    @Override
    public void reset() throws IOException {
      backOff.reset();
    }

    @Override
    public long nextBackOffMillis() throws IOException {
      return backOff.nextBackOffMillis();
    }
  };
}
Example 5
Source File: LocalSpannerIO.java    From DataflowTemplates with Apache License 2.0 4 votes vote down vote up 
/** Write the Mutations to Spanner, handling DEADLINE_EXCEEDED with backoff/retries. */
private void writeMutations(Iterable<Mutation> mutations) throws SpannerException, IOException {
  BackOff backoff = bundleWriteBackoff.backoff();
  long mutationsSize = Iterables.size(mutations);

  while (true) {
    Stopwatch timer = Stopwatch.createStarted();
    // loop is broken on success, timeout backoff/retry attempts exceeded, or other failure.
    try {
      spannerWriteWithRetryIfSchemaChange(mutations);
      spannerWriteSuccess.inc();
      return;
    } catch (SpannerException exception) {
      if (exception.getErrorCode() == ErrorCode.DEADLINE_EXCEEDED) {
        spannerWriteTimeouts.inc();

        // Potentially backoff/retry after DEADLINE_EXCEEDED.
        long sleepTimeMsecs = backoff.nextBackOffMillis();
        if (sleepTimeMsecs == BackOff.STOP) {
          LOG.error(
              "DEADLINE_EXCEEDED writing batch of {} mutations to Cloud Spanner. "
                  + "Aborting after too many retries.",
              mutationsSize);
          spannerWriteFail.inc();
          throw exception;
        }
        LOG.info(
            "DEADLINE_EXCEEDED writing batch of {} mutations to Cloud Spanner, "
                + "retrying after backoff of {}ms\n"
                + "({})",
            mutationsSize,
            sleepTimeMsecs,
            exception.getMessage());
        spannerWriteRetries.inc();
        try {
          sleeper.sleep(sleepTimeMsecs);
        } catch (InterruptedException e) {
          // ignore.
        }
      } else {
        // Some other failure: pass up the stack.
        spannerWriteFail.inc();
        throw exception;
      }
    } finally {
      spannerWriteLatency.update(timer.elapsed(TimeUnit.MILLISECONDS));
    }
  }
}
Example 6
Source File: SpannerIO.java    From beam with Apache License 2.0 4 votes vote down vote up 
/** Write the Mutations to Spanner, handling DEADLINE_EXCEEDED with backoff/retries. */
private void writeMutations(Iterable<Mutation> mutations) throws SpannerException, IOException {
  BackOff backoff = bundleWriteBackoff.backoff();
  long mutationsSize = Iterables.size(mutations);

  while (true) {
    Stopwatch timer = Stopwatch.createStarted();
    // loop is broken on success, timeout backoff/retry attempts exceeded, or other failure.
    try {
      spannerWriteWithRetryIfSchemaChange(mutations);
      spannerWriteSuccess.inc();
      return;
    } catch (SpannerException exception) {
      if (exception.getErrorCode() == ErrorCode.DEADLINE_EXCEEDED) {
        spannerWriteTimeouts.inc();

        // Potentially backoff/retry after DEADLINE_EXCEEDED.
        long sleepTimeMsecs = backoff.nextBackOffMillis();
        if (sleepTimeMsecs == BackOff.STOP) {
          LOG.error(
              "DEADLINE_EXCEEDED writing batch of {} mutations to Cloud Spanner. "
                  + "Aborting after too many retries.",
              mutationsSize);
          spannerWriteFail.inc();
          throw exception;
        }
        LOG.info(
            "DEADLINE_EXCEEDED writing batch of {} mutations to Cloud Spanner, "
                + "retrying after backoff of {}ms\n"
                + "({})",
            mutationsSize,
            sleepTimeMsecs,
            exception.getMessage());
        spannerWriteRetries.inc();
        try {
          sleeper.sleep(sleepTimeMsecs);
        } catch (InterruptedException e) {
          // ignore.
        }
      } else {
        // Some other failure: pass up the stack.
        spannerWriteFail.inc();
        throw exception;
      }
    } finally {
      spannerWriteLatency.update(timer.elapsed(TimeUnit.MILLISECONDS));
    }
  }
}