com.google.api.client.util.BackOff Java Examples

@Test
public void testGetBucket() throws IOException {
  GcsOptions pipelineOptions = gcsOptionsWithTestCredential();
  GcsUtil gcsUtil = pipelineOptions.getGcsUtil();

  Storage mockStorage = Mockito.mock(Storage.class);
  gcsUtil.setStorageClient(mockStorage);

  Storage.Buckets mockStorageObjects = Mockito.mock(Storage.Buckets.class);
  Storage.Buckets.Get mockStorageGet = Mockito.mock(Storage.Buckets.Get.class);

  BackOff mockBackOff = BackOffAdapter.toGcpBackOff(FluentBackoff.DEFAULT.backoff());

  when(mockStorage.buckets()).thenReturn(mockStorageObjects);
  when(mockStorageObjects.get("testbucket")).thenReturn(mockStorageGet);
  when(mockStorageGet.execute())
      .thenThrow(new SocketTimeoutException("SocketException"))
      .thenReturn(new Bucket());

  assertNotNull(
      gcsUtil.getBucket(
          GcsPath.fromComponents("testbucket", "testobject"),
          mockBackOff,
          new FastNanoClockAndSleeper()));
}
 

private boolean backOff(List<AsyncRequest<?>> pendingTasks) {
  long backOffMillis = getBackOffTime();
  if (backOffMillis == BackOff.STOP) {
    pendingTasks.forEach(
        t ->
            onFailure(
                t, new ExecutionException(new Exception("Maximum back-off cycle reached."))));
    return false;
  }
  try {
    batchRequestHelper.sleep(backOffMillis);
  } catch (InterruptedException e) {
    logger.log(Level.WARNING, "Task interrupted while processing batch requests, stopping.", e);
    pendingTasks.forEach(t -> onFailure(t, e));
    Thread.currentThread().interrupt();
    return false;
  }
  return true;
}
 

@Test
public void resetsCorrectly() throws Exception {
  BackOff backoff = new RetryBoundedBackOff(5, new BackOffTester());
  for (int i = 0; i < 5; i++) {
    assertThat(i + 1).isEqualTo(backoff.nextBackOffMillis());
  }
  assertThat(backoff.nextBackOffMillis()).isEqualTo(BackOff.STOP);
  assertThat(backoff.nextBackOffMillis()).isEqualTo(BackOff.STOP);
  backoff.reset();
  for (int i = 0; i < 3; i++) {
    assertThat(i + 1).isEqualTo(backoff.nextBackOffMillis());
  }
  backoff.reset();
  for (int i = 0; i < 5; i++) {
    assertThat(i + 1).isEqualTo(backoff.nextBackOffMillis());
  }
  assertThat(backoff.nextBackOffMillis()).isEqualTo(BackOff.STOP);
  assertThat(backoff.nextBackOffMillis()).isEqualTo(BackOff.STOP);
}
 

@Test
public void testCallFailsOnBadException() throws Exception {
  MockSleeper sleeper = new MockSleeper();
  ArrayList<Exception> exceptions = new ArrayList<>();
  exceptions.add(new IllegalArgumentException("FakeException"));
  CallableTester<Exception> callTester = new CallableTester<>(exceptions);
  BackOff backoff = new RetryBoundedBackOff(3, new BackOffTester());

  IllegalArgumentException thrown =
      assertThrows(
          IllegalArgumentException.class,
          () ->
              ResilientOperation.retry(
                  callTester, backoff, RetryDeterminer.DEFAULT, Exception.class, sleeper));
  assertThat(thrown).hasMessageThat().contains("FakeException");

  assertThat(callTester.timesCalled()).isEqualTo(1);
  verifySleeper(sleeper, 0);
}
 

private List<TableRow> getTableRowsFromQuery(String query, int maxRetry) throws Exception {
  FluentBackoff backoffFactory =
      FluentBackoff.DEFAULT
          .withMaxRetries(maxRetry)
          .withInitialBackoff(Duration.standardSeconds(1L));
  Sleeper sleeper = Sleeper.DEFAULT;
  BackOff backoff = BackOffAdapter.toGcpBackOff(backoffFactory.backoff());
  do {
    LOG.info("Starting querying {}", query);
    QueryResponse response = BQ_CLIENT.queryWithRetries(query, project);
    if (response.getRows() != null) {
      LOG.info("Got table content with query {}", query);
      return response.getRows();
    }
  } while (BackOffUtils.next(sleeper, backoff));
  LOG.info("Got empty table for query {} with retry {}", query, maxRetry);
  return Collections.emptyList();
}
 

@Test
public void testCallRetriesAndSucceeds() throws Exception {
  MockSleeper sleeper = new MockSleeper();
  ArrayList<Exception> exceptions = new ArrayList<>();
  exceptions.add(new SocketTimeoutException("socket"));
  exceptions.add(new SocketTimeoutException("socket2"));
  exceptions.add(new SocketTimeoutException("socket3"));
  CallableTester<Exception> callTester = new CallableTester<>(exceptions);
  BackOff backoff = new RetryBoundedBackOff(3, new BackOffTester());
  assertThat(
          ResilientOperation.retry(
              callTester, backoff, RetryDeterminer.DEFAULT, Exception.class, sleeper))
      .isEqualTo(3);
  assertThat(callTester.timesCalled()).isEqualTo(4);
  verifySleeper(sleeper, 3);
}
 

protected <T extends GenericJson> T retryFailures(
    AbstractGoogleClientRequest<T> request, BackOff backOff) throws IOException {
  while (true) {
    try {
      return request.execute();
    } catch (GoogleJsonResponseException e) {
      try {
        long nextPause = backOff.nextBackOffMillis();
        if (nextPause == BackOff.STOP) {
          throw e;
        }
        System.out.printf("Operation failed, retrying in %f seconds.%n", nextPause / 1000.0);
        BackOffUtils.next(Sleeper.DEFAULT, backOff);
      } catch (InterruptedException ie) {
        // Just go straight into retry if interrupted.
      }
    }
  }
}
 

@Test
public void testExponentialBackOff() throws IOException {
  RetryConfig config = RetryConfig.builder()
      .setMaxIntervalMillis(12000)
      .build();

  BackOff backOff = config.newBackOff();

  assertEquals(500, backOff.nextBackOffMillis());
  assertEquals(1000, backOff.nextBackOffMillis());
  assertEquals(2000, backOff.nextBackOffMillis());
  assertEquals(4000, backOff.nextBackOffMillis());
  assertEquals(8000, backOff.nextBackOffMillis());
  assertEquals(12000, backOff.nextBackOffMillis());
  assertEquals(12000, backOff.nextBackOffMillis());
}
 

@Test
public void testCallRetriesAndFailsWithSocketErrors() throws Exception {
  MockSleeper sleeper = new MockSleeper();
  ArrayList<IOException> exceptions = new ArrayList<>();
  exceptions.add(new SocketTimeoutException("socket"));
  exceptions.add(new SocketTimeoutException("socket"));
  exceptions.add(new IOException("FakeException"));
  CallableTester<IOException> callTester = new CallableTester<>(exceptions);
  BackOff backoff = new RetryBoundedBackOff(5, new BackOffTester());

  IOException thrown =
      assertThrows(
          IOException.class,
          () ->
              ResilientOperation.retry(
                  callTester,
                  backoff,
                  RetryDeterminer.SOCKET_ERRORS,
                  IOException.class,
                  sleeper));
  assertThat(thrown).hasMessageThat().contains("FakeException");

  assertThat(callTester.timesCalled()).isEqualTo(3);
  verifySleeper(sleeper, 2);
}
 

/**
 * Attempts to get the underlying {@link Job}. Uses exponential backoff on failure up to the
 * maximum number of passed in attempts.
 *
 * @param backoff the {@link BackOff} used to control retries.
 * @param sleeper Object used to do the sleeps between attempts.
 * @return The underlying {@link Job} object.
 * @throws IOException When the maximum number of retries is exhausted, the last exception is
 *     thrown.
 */
private Job getJobWithRetries(BackOff backoff, Sleeper sleeper) throws IOException {
  // Retry loop ends in return or throw
  while (true) {
    try {
      Job job = dataflowClient.getJob(getJobId());
      State currentState = MonitoringUtil.toState(job.getCurrentState());
      if (currentState.isTerminal()) {
        terminalState = currentState;
        replacedByJob =
            new DataflowPipelineJob(
                dataflowClient, job.getReplacedByJobId(), dataflowOptions, transformStepNames);
      }
      return job;
    } catch (IOException exn) {
      LOG.warn("There were problems getting current job status: {}.", exn.getMessage());
      LOG.debug("Exception information:", exn);

      if (!nextBackOff(sleeper, backoff)) {
        throw exn;
      }
    }
  }
}
 

/**
 * Reset backoff. If duration is limited, calculate time remaining, otherwise just reset retry
 * count.
 *
 * <p>If a total duration for all backoff has been set, update the new cumulative sleep time to be
 * the remaining total backoff duration, stopping if we have already exceeded the allotted time.
 */
private static BackOff resetBackoff(Duration duration, NanoClock nanoClock, long startNanos) {
  BackOff backoff;
  if (duration.isLongerThan(Duration.ZERO)) {
    long nanosConsumed = nanoClock.nanoTime() - startNanos;
    Duration consumed = Duration.millis((nanosConsumed + 999999) / 1000000);
    Duration remaining = duration.minus(consumed);
    if (remaining.isLongerThan(Duration.ZERO)) {
      backoff = getMessagesBackoff(remaining);
    } else {
      backoff = BackOff.STOP_BACKOFF;
    }
  } else {
    backoff = getMessagesBackoff(duration);
  }
  return backoff;
}
 

@Test
public void testCreateBucket() throws IOException {
  GcsOptions pipelineOptions = gcsOptionsWithTestCredential();
  GcsUtil gcsUtil = pipelineOptions.getGcsUtil();

  Storage.Buckets mockStorageObjects = Mockito.mock(Storage.Buckets.class);
  Storage mockStorage = Mockito.mock(Storage.class);
  gcsUtil.setStorageClient(mockStorage);

  Storage.Buckets.Insert mockStorageInsert = Mockito.mock(Storage.Buckets.Insert.class);

  BackOff mockBackOff = BackOffAdapter.toGcpBackOff(FluentBackoff.DEFAULT.backoff());

  when(mockStorage.buckets()).thenReturn(mockStorageObjects);
  when(mockStorageObjects.insert(any(String.class), any(Bucket.class)))
      .thenReturn(mockStorageInsert);
  when(mockStorageInsert.execute())
      .thenThrow(new SocketTimeoutException("SocketException"))
      .thenReturn(new Bucket());

  gcsUtil.createBucket("a", new Bucket(), mockBackOff, new FastNanoClockAndSleeper());
}
 

@Test
public void testBucketAccessible() throws IOException {
  GcsOptions pipelineOptions = gcsOptionsWithTestCredential();
  GcsUtil gcsUtil = pipelineOptions.getGcsUtil();

  Storage mockStorage = Mockito.mock(Storage.class);
  gcsUtil.setStorageClient(mockStorage);

  Storage.Buckets mockStorageObjects = Mockito.mock(Storage.Buckets.class);
  Storage.Buckets.Get mockStorageGet = Mockito.mock(Storage.Buckets.Get.class);

  BackOff mockBackOff = BackOffAdapter.toGcpBackOff(FluentBackoff.DEFAULT.backoff());

  when(mockStorage.buckets()).thenReturn(mockStorageObjects);
  when(mockStorageObjects.get("testbucket")).thenReturn(mockStorageGet);
  when(mockStorageGet.execute())
      .thenThrow(new SocketTimeoutException("SocketException"))
      .thenReturn(new Bucket());

  assertTrue(
      gcsUtil.bucketAccessible(
          GcsPath.fromComponents("testbucket", "testobject"),
          mockBackOff,
          new FastNanoClockAndSleeper()));
}
 

@Test
public void testExecuteWithRetries() throws IOException, InterruptedException {
  Table testTable = new Table();

  when(response.getContentType()).thenReturn(Json.MEDIA_TYPE);
  when(response.getStatusCode()).thenReturn(200);
  when(response.getContent()).thenReturn(toStream(testTable));

  Table table =
      BigQueryServicesImpl.executeWithRetries(
          bigquery.tables().get("projectId", "datasetId", "tableId"),
          "Failed to get table.",
          Sleeper.DEFAULT,
          BackOff.STOP_BACKOFF,
          BigQueryServicesImpl.ALWAYS_RETRY);

  assertEquals(testTable, table);
  verify(response, times(1)).getStatusCode();
  verify(response, times(1)).getContent();
  verify(response, times(1)).getContentType();
}
 

/**
 * Determines the amount to sleep for and sleeps if needed.
 *
 * @param backoff BackOff to determine how long to sleep for
 * @param sleeper Used to sleep
 * @param currentException exception that caused the retry and sleep. For logging.
 * @throws InterruptedException if sleep is interrupted
 */
private static boolean nextSleep(BackOff backoff, Sleeper sleeper, Exception currentException)
    throws InterruptedException {
  long backOffTime;
  try {
    backOffTime = backoff.nextBackOffMillis();
  } catch (IOException e) {
    throw new RuntimeException("Failed to to get next back off time", e);
  }
  if (backOffTime == BackOff.STOP) {
    return false;
  }
  logger.atInfo().withCause(currentException).log(
      "Transient exception caught. Sleeping for %d, then retrying.", backOffTime);
  sleeper.sleep(backOffTime);
  return true;
}
 

@Test
public void testCreateTableSucceeds() throws IOException {
  TableReference ref =
      new TableReference().setProjectId("project").setDatasetId("dataset").setTableId("table");
  Table testTable = new Table().setTableReference(ref);
  when(response.getContentType()).thenReturn(Json.MEDIA_TYPE);
  when(response.getStatusCode()).thenReturn(200);
  when(response.getContent()).thenReturn(toStream(testTable));

  BigQueryServicesImpl.DatasetServiceImpl services =
      new BigQueryServicesImpl.DatasetServiceImpl(bigquery, PipelineOptionsFactory.create());
  Table ret =
      services.tryCreateTable(
          testTable, new RetryBoundedBackOff(0, BackOff.ZERO_BACKOFF), Sleeper.DEFAULT);
  assertEquals(testTable, ret);
  verify(response, times(1)).getStatusCode();
  verify(response, times(1)).getContent();
  verify(response, times(1)).getContentType();
}
 

@Test
public void testIsTableEmptyThrows() throws Exception {
  when(response.getContentType()).thenReturn(Json.MEDIA_TYPE);
  when(response.getStatusCode()).thenReturn(401);

  TableReference tableRef =
      new TableReference()
          .setProjectId("projectId")
          .setDatasetId("datasetId")
          .setTableId("tableId");

  BigQueryServicesImpl.DatasetServiceImpl datasetService =
      new BigQueryServicesImpl.DatasetServiceImpl(bigquery, PipelineOptionsFactory.create());

  thrown.expect(IOException.class);
  thrown.expectMessage(String.format("Unable to list table data: %s", tableRef.getTableId()));

  datasetService.isTableEmpty(tableRef, BackOff.STOP_BACKOFF, Sleeper.DEFAULT);
}
 

@Test
public void testIsTableEmptyNoRetryForNotFound() throws IOException, InterruptedException {
  when(response.getContentType()).thenReturn(Json.MEDIA_TYPE);
  when(response.getStatusCode()).thenReturn(404);

  BigQueryServicesImpl.DatasetServiceImpl datasetService =
      new BigQueryServicesImpl.DatasetServiceImpl(bigquery, PipelineOptionsFactory.create());

  TableReference tableRef =
      new TableReference()
          .setProjectId("projectId")
          .setDatasetId("datasetId")
          .setTableId("tableId");

  thrown.expect(IOException.class);
  thrown.expectMessage(String.format("Unable to list table data: %s", tableRef.getTableId()));

  try {
    datasetService.isTableEmpty(tableRef, BackOff.ZERO_BACKOFF, Sleeper.DEFAULT);
  } finally {
    verify(response, times(1)).getStatusCode();
    verify(response, times(1)).getContent();
    verify(response, times(1)).getContentType();
  }
}
 

@Test
public void testIsTableEmptySucceeds() throws Exception {
  TableReference tableRef =
      new TableReference()
          .setProjectId("projectId")
          .setDatasetId("datasetId")
          .setTableId("tableId");

  TableDataList testDataList = new TableDataList().setRows(ImmutableList.of(new TableRow()));

  // First response is 403 rate limited, second response has valid payload.
  when(response.getContentType()).thenReturn(Json.MEDIA_TYPE);
  when(response.getStatusCode()).thenReturn(403).thenReturn(200);
  when(response.getContent())
      .thenReturn(toStream(errorWithReasonAndStatus("rateLimitExceeded", 403)))
      .thenReturn(toStream(testDataList));

  BigQueryServicesImpl.DatasetServiceImpl datasetService =
      new BigQueryServicesImpl.DatasetServiceImpl(bigquery, PipelineOptionsFactory.create());

  assertFalse(datasetService.isTableEmpty(tableRef, BackOff.ZERO_BACKOFF, Sleeper.DEFAULT));

  verify(response, times(2)).getStatusCode();
  verify(response, times(2)).getContent();
  verify(response, times(2)).getContentType();
}
 

@Test
public void testGetTableThrows() throws Exception {
  when(response.getContentType()).thenReturn(Json.MEDIA_TYPE);
  when(response.getStatusCode()).thenReturn(401);

  TableReference tableRef =
      new TableReference()
          .setProjectId("projectId")
          .setDatasetId("datasetId")
          .setTableId("tableId");

  thrown.expect(IOException.class);
  thrown.expectMessage(String.format("Unable to get table: %s", tableRef.getTableId()));

  BigQueryServicesImpl.DatasetServiceImpl datasetService =
      new BigQueryServicesImpl.DatasetServiceImpl(bigquery, PipelineOptionsFactory.create());
  datasetService.getTable(tableRef, null, BackOff.STOP_BACKOFF, Sleeper.DEFAULT);
}
 

@Test
public void testGetTableNotFound() throws IOException, InterruptedException {
  when(response.getContentType()).thenReturn(Json.MEDIA_TYPE);
  when(response.getStatusCode()).thenReturn(404);

  BigQueryServicesImpl.DatasetServiceImpl datasetService =
      new BigQueryServicesImpl.DatasetServiceImpl(bigquery, PipelineOptionsFactory.create());

  TableReference tableRef =
      new TableReference()
          .setProjectId("projectId")
          .setDatasetId("datasetId")
          .setTableId("tableId");
  Table table = datasetService.getTable(tableRef, null, BackOff.ZERO_BACKOFF, Sleeper.DEFAULT);

  assertNull(table);
  verify(response, times(1)).getStatusCode();
  verify(response, times(1)).getContent();
  verify(response, times(1)).getContentType();
}
 

@VisibleForTesting
@Nullable
Table getTable(
    TableReference ref, @Nullable List<String> selectedFields, BackOff backoff, Sleeper sleeper)
    throws IOException, InterruptedException {
  Tables.Get get =
      client.tables().get(ref.getProjectId(), ref.getDatasetId(), ref.getTableId());
  if (selectedFields != null && !selectedFields.isEmpty()) {
    get.setSelectedFields(String.join(",", selectedFields));
  }
  try {
    return executeWithRetries(
        get,
        String.format(
            "Unable to get table: %s, aborting after %d retries.",
            ref.getTableId(), MAX_RPC_RETRIES),
        sleeper,
        backoff,
        DONT_RETRY_NOT_FOUND);
  } catch (IOException e) {
    if (errorExtractor.itemNotFound(e)) {
      return null;
    }
    throw e;
  }
}
 

@ProcessElement
public void processElement(DoFn<String, Integer>.ProcessContext context) throws Exception {
  String variantId = context.element();
  // Call the deletion operation via exponential backoff so that "Rate Limit Exceeded"
  // quota issues do not cause the pipeline to fail.
  ExponentialBackOff backoff = new ExponentialBackOff.Builder().build();
  while (true) {
    try {
      genomics.variants().delete(variantId).execute();
      Metrics.counter(DeleteVariantFn.class, "Number of variants deleted").inc();
      context.output(1);
      return;
    } catch (Exception e) {
      if (e.getMessage().startsWith("429 Too Many Requests")) {
        LOG.warn("Backing-off per: ", e);
        long backOffMillis = backoff.nextBackOffMillis();
        if (backOffMillis == BackOff.STOP) {
          throw e;
        }
        Thread.sleep(backOffMillis);
      } else {
        throw e;
      }
    }
  }
}
 

@Test
public void testGetJobSucceeds() throws Exception {
  Job testJob = new Job();
  testJob.setStatus(new JobStatus());

  when(response.getContentType()).thenReturn(Json.MEDIA_TYPE);
  when(response.getStatusCode()).thenReturn(200);
  when(response.getContent()).thenReturn(toStream(testJob));

  BigQueryServicesImpl.JobServiceImpl jobService =
      new BigQueryServicesImpl.JobServiceImpl(bigquery);
  JobReference jobRef = new JobReference().setProjectId("projectId").setJobId("jobId");
  Job job = jobService.getJob(jobRef, Sleeper.DEFAULT, BackOff.ZERO_BACKOFF);

  assertEquals(testJob, job);
  verify(response, times(1)).getStatusCode();
  verify(response, times(1)).getContent();
  verify(response, times(1)).getContentType();
}
 

@VisibleForTesting
boolean isTableEmpty(TableReference tableRef, BackOff backoff, Sleeper sleeper)
    throws IOException, InterruptedException {
  TableDataList dataList =
      executeWithRetries(
          client
              .tabledata()
              .list(tableRef.getProjectId(), tableRef.getDatasetId(), tableRef.getTableId()),
          String.format(
              "Unable to list table data: %s, aborting after %d retries.",
              tableRef.getTableId(), MAX_RPC_RETRIES),
          sleeper,
          backoff,
          DONT_RETRY_NOT_FOUND);
  return dataList.getRows() == null || dataList.getRows().isEmpty();
}
 

@VisibleForTesting
static <T> T executeWithRetries(
    AbstractGoogleClientRequest<T> request,
    String errorMessage,
    Sleeper sleeper,
    BackOff backoff,
    SerializableFunction<IOException, Boolean> shouldRetry)
    throws IOException, InterruptedException {
  Exception lastException = null;
  do {
    try {
      return request.execute();
    } catch (IOException e) {
      lastException = e;
      if (!shouldRetry.apply(e)) {
        break;
      }
      LOG.info("Ignore the error and retry the request.", e);
    }
  } while (nextBackOff(sleeper, backoff));
  throw new IOException(errorMessage, lastException);
}
 

/** Tests that {@link BigQueryServicesImpl.JobServiceImpl#pollJob} returns UNKNOWN. */
@Test
public void testPollJobUnknown() throws IOException, InterruptedException {
  Job testJob = new Job();
  testJob.setStatus(new JobStatus());

  when(response.getContentType()).thenReturn(Json.MEDIA_TYPE);
  when(response.getStatusCode()).thenReturn(200);
  when(response.getContent()).thenReturn(toStream(testJob));

  BigQueryServicesImpl.JobServiceImpl jobService =
      new BigQueryServicesImpl.JobServiceImpl(bigquery);
  JobReference jobRef = new JobReference().setProjectId("projectId").setJobId("jobId");
  Job job = jobService.pollJob(jobRef, Sleeper.DEFAULT, BackOff.STOP_BACKOFF);

  assertEquals(null, job);
  verify(response, times(1)).getStatusCode();
  verify(response, times(1)).getContent();
  verify(response, times(1)).getContentType();
}
 

/** Tests that {@link BigQueryServicesImpl.JobServiceImpl#pollJob} fails. */
@Test
public void testPollJobFailed() throws IOException, InterruptedException {
  Job testJob = new Job();
  testJob.setStatus(new JobStatus().setState("DONE").setErrorResult(new ErrorProto()));

  when(response.getContentType()).thenReturn(Json.MEDIA_TYPE);
  when(response.getStatusCode()).thenReturn(200);
  when(response.getContent()).thenReturn(toStream(testJob));

  BigQueryServicesImpl.JobServiceImpl jobService =
      new BigQueryServicesImpl.JobServiceImpl(bigquery);
  JobReference jobRef = new JobReference().setProjectId("projectId").setJobId("jobId");
  Job job = jobService.pollJob(jobRef, Sleeper.DEFAULT, BackOff.ZERO_BACKOFF);

  assertEquals(testJob, job);
  verify(response, times(1)).getStatusCode();
  verify(response, times(1)).getContent();
  verify(response, times(1)).getContentType();
}
 

@Override
public Job pollJob(JobReference jobRef, int maxAttempts) throws InterruptedException {
  BackOff backoff =
      BackOffAdapter.toGcpBackOff(
          FluentBackoff.DEFAULT
              .withMaxRetries(maxAttempts)
              .withInitialBackoff(Duration.millis(10))
              .withMaxBackoff(Duration.standardSeconds(1))
              .backoff());
  Sleeper sleeper = Sleeper.DEFAULT;
  try {
    do {
      Job job = getJob(jobRef);
      if (job != null) {
        JobStatus status = job.getStatus();
        if (status != null
            && ("DONE".equals(status.getState()) || "FAILED".equals(status.getState()))) {
          return job;
        }
      }
    } while (BackOffUtils.next(sleeper, backoff));
  } catch (IOException e) {
    return null;
  }
  return null;
}
 

/** Tests that {@link BigQueryServicesImpl.JobServiceImpl#pollJob} succeeds. */
@Test
public void testPollJobSucceeds() throws IOException, InterruptedException {
  Job testJob = new Job();
  testJob.setStatus(new JobStatus().setState("DONE"));

  when(response.getContentType()).thenReturn(Json.MEDIA_TYPE);
  when(response.getStatusCode()).thenReturn(200);
  when(response.getContent()).thenReturn(toStream(testJob));

  BigQueryServicesImpl.JobServiceImpl jobService =
      new BigQueryServicesImpl.JobServiceImpl(bigquery);
  JobReference jobRef = new JobReference().setProjectId("projectId").setJobId("jobId");
  Job job = jobService.pollJob(jobRef, Sleeper.DEFAULT, BackOff.ZERO_BACKOFF);

  assertEquals(testJob, job);
  verify(response, times(1)).getStatusCode();
  verify(response, times(1)).getContent();
  verify(response, times(1)).getContentType();
}