Java Code Examples for org.apache.hadoop.io.retry.RetryPolicies#RETRY_FOREVER

The following examples show how to use org.apache.hadoop.io.retry.RetryPolicies#RETRY_FOREVER . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: TestRetriableTask.java    From hadoop-ozone with Apache License 2.0 5 votes vote down vote up 
@Test
public void returnsSuccessfulResult() throws Exception {
  String result = "bilbo";
  RetriableTask<String> task = new RetriableTask<>(
      RetryPolicies.RETRY_FOREVER, "test", () -> result);
  assertEquals(result, task.call());
}
Example 2
Source File: TestRetriableTask.java    From hadoop-ozone with Apache License 2.0 5 votes vote down vote up 
@Test
public void returnsSuccessfulResultAfterFailures() throws Exception {
  String result = "gandalf";
  AtomicInteger attempts = new AtomicInteger();
  RetriableTask<String> task = new RetriableTask<>(
      RetryPolicies.RETRY_FOREVER, "test",
      () -> {
        if (attempts.incrementAndGet() <= 2) {
          throw new Exception("testing");
        }
        return result;
      });
  assertEquals(result, task.call());
}
Example 3
Source File: ServerProxy.java    From hadoop with Apache License 2.0 5 votes vote down vote up 
protected static RetryPolicy createRetryPolicy(Configuration conf,
    String maxWaitTimeStr, long defMaxWaitTime,
    String connectRetryIntervalStr, long defRetryInterval) {
  long maxWaitTime = conf.getLong(maxWaitTimeStr, defMaxWaitTime);
  long retryIntervalMS =
      conf.getLong(connectRetryIntervalStr, defRetryInterval);
  if (maxWaitTime == -1) {
    // wait forever.
    return RetryPolicies.RETRY_FOREVER;
  }

  Preconditions.checkArgument(maxWaitTime > 0, "Invalid Configuration. "
      + maxWaitTimeStr + " should be a positive value.");
  Preconditions.checkArgument(retryIntervalMS > 0, "Invalid Configuration. "
      + connectRetryIntervalStr + "should be a positive value.");

  RetryPolicy retryPolicy =
      RetryPolicies.retryUpToMaximumTimeWithFixedSleep(maxWaitTime,
        retryIntervalMS, TimeUnit.MILLISECONDS);

  Map<Class<? extends Exception>, RetryPolicy> exceptionToPolicyMap =
      new HashMap<Class<? extends Exception>, RetryPolicy>();
  exceptionToPolicyMap.put(EOFException.class, retryPolicy);
  exceptionToPolicyMap.put(ConnectException.class, retryPolicy);
  exceptionToPolicyMap.put(NoRouteToHostException.class, retryPolicy);
  exceptionToPolicyMap.put(UnknownHostException.class, retryPolicy);
  exceptionToPolicyMap.put(RetriableException.class, retryPolicy);
  exceptionToPolicyMap.put(SocketException.class, retryPolicy);
  exceptionToPolicyMap.put(NMNotYetReadyException.class, retryPolicy);

  return RetryPolicies.retryByException(RetryPolicies.TRY_ONCE_THEN_FAIL,
    exceptionToPolicyMap);
}
Example 4
Source File: ServerProxy.java    From big-c with Apache License 2.0 5 votes vote down vote up 
protected static RetryPolicy createRetryPolicy(Configuration conf,
    String maxWaitTimeStr, long defMaxWaitTime,
    String connectRetryIntervalStr, long defRetryInterval) {
  long maxWaitTime = conf.getLong(maxWaitTimeStr, defMaxWaitTime);
  long retryIntervalMS =
      conf.getLong(connectRetryIntervalStr, defRetryInterval);
  if (maxWaitTime == -1) {
    // wait forever.
    return RetryPolicies.RETRY_FOREVER;
  }

  Preconditions.checkArgument(maxWaitTime > 0, "Invalid Configuration. "
      + maxWaitTimeStr + " should be a positive value.");
  Preconditions.checkArgument(retryIntervalMS > 0, "Invalid Configuration. "
      + connectRetryIntervalStr + "should be a positive value.");

  RetryPolicy retryPolicy =
      RetryPolicies.retryUpToMaximumTimeWithFixedSleep(maxWaitTime,
        retryIntervalMS, TimeUnit.MILLISECONDS);

  Map<Class<? extends Exception>, RetryPolicy> exceptionToPolicyMap =
      new HashMap<Class<? extends Exception>, RetryPolicy>();
  exceptionToPolicyMap.put(EOFException.class, retryPolicy);
  exceptionToPolicyMap.put(ConnectException.class, retryPolicy);
  exceptionToPolicyMap.put(NoRouteToHostException.class, retryPolicy);
  exceptionToPolicyMap.put(UnknownHostException.class, retryPolicy);
  exceptionToPolicyMap.put(RetriableException.class, retryPolicy);
  exceptionToPolicyMap.put(SocketException.class, retryPolicy);
  exceptionToPolicyMap.put(NMNotYetReadyException.class, retryPolicy);

  return RetryPolicies.retryByException(RetryPolicies.TRY_ONCE_THEN_FAIL,
    exceptionToPolicyMap);
}
Example 5
Source File: RMProxy.java    From hadoop with Apache License 2.0 4 votes vote down vote up 
/**
 * Fetch retry policy from Configuration
 */
@Private
@VisibleForTesting
public static RetryPolicy createRetryPolicy(Configuration conf) {
  long rmConnectWaitMS =
      conf.getLong(
          YarnConfiguration.RESOURCEMANAGER_CONNECT_MAX_WAIT_MS,
          YarnConfiguration.DEFAULT_RESOURCEMANAGER_CONNECT_MAX_WAIT_MS);
  long rmConnectionRetryIntervalMS =
      conf.getLong(
          YarnConfiguration.RESOURCEMANAGER_CONNECT_RETRY_INTERVAL_MS,
          YarnConfiguration
              .DEFAULT_RESOURCEMANAGER_CONNECT_RETRY_INTERVAL_MS);

  boolean waitForEver = (rmConnectWaitMS == -1);
  if (!waitForEver) {
    if (rmConnectWaitMS < 0) {
      throw new YarnRuntimeException("Invalid Configuration. "
          + YarnConfiguration.RESOURCEMANAGER_CONNECT_MAX_WAIT_MS
          + " can be -1, but can not be other negative numbers");
    }

    // try connect once
    if (rmConnectWaitMS < rmConnectionRetryIntervalMS) {
      LOG.warn(YarnConfiguration.RESOURCEMANAGER_CONNECT_MAX_WAIT_MS
          + " is smaller than "
          + YarnConfiguration.RESOURCEMANAGER_CONNECT_RETRY_INTERVAL_MS
          + ". Only try connect once.");
      rmConnectWaitMS = 0;
    }
  }

  // Handle HA case first
  if (HAUtil.isHAEnabled(conf)) {
    final long failoverSleepBaseMs = conf.getLong(
        YarnConfiguration.CLIENT_FAILOVER_SLEEPTIME_BASE_MS,
        rmConnectionRetryIntervalMS);

    final long failoverSleepMaxMs = conf.getLong(
        YarnConfiguration.CLIENT_FAILOVER_SLEEPTIME_MAX_MS,
        rmConnectionRetryIntervalMS);

    int maxFailoverAttempts = conf.getInt(
        YarnConfiguration.CLIENT_FAILOVER_MAX_ATTEMPTS, -1);

    if (maxFailoverAttempts == -1) {
      if (waitForEver) {
        maxFailoverAttempts = Integer.MAX_VALUE;
      } else {
        maxFailoverAttempts = (int) (rmConnectWaitMS / failoverSleepBaseMs);
      }
    }

    return RetryPolicies.failoverOnNetworkException(
        RetryPolicies.TRY_ONCE_THEN_FAIL, maxFailoverAttempts,
        failoverSleepBaseMs, failoverSleepMaxMs);
  }

  if (rmConnectionRetryIntervalMS < 0) {
    throw new YarnRuntimeException("Invalid Configuration. " +
        YarnConfiguration.RESOURCEMANAGER_CONNECT_RETRY_INTERVAL_MS +
        " should not be negative.");
  }

  RetryPolicy retryPolicy = null;
  if (waitForEver) {
    retryPolicy = RetryPolicies.RETRY_FOREVER;
  } else {
    retryPolicy =
        RetryPolicies.retryUpToMaximumTimeWithFixedSleep(rmConnectWaitMS,
            rmConnectionRetryIntervalMS, TimeUnit.MILLISECONDS);
  }

  Map<Class<? extends Exception>, RetryPolicy> exceptionToPolicyMap =
      new HashMap<Class<? extends Exception>, RetryPolicy>();

  exceptionToPolicyMap.put(EOFException.class, retryPolicy);
  exceptionToPolicyMap.put(ConnectException.class, retryPolicy);
  exceptionToPolicyMap.put(NoRouteToHostException.class, retryPolicy);
  exceptionToPolicyMap.put(UnknownHostException.class, retryPolicy);
  exceptionToPolicyMap.put(ConnectTimeoutException.class, retryPolicy);
  exceptionToPolicyMap.put(RetriableException.class, retryPolicy);
  exceptionToPolicyMap.put(SocketException.class, retryPolicy);

  return RetryPolicies.retryByException(
      RetryPolicies.TRY_ONCE_THEN_FAIL, exceptionToPolicyMap);
}
Example 6
Source File: RMProxy.java    From big-c with Apache License 2.0 4 votes vote down vote up 
/**
 * Fetch retry policy from Configuration
 */
@Private
@VisibleForTesting
public static RetryPolicy createRetryPolicy(Configuration conf) {
  long rmConnectWaitMS =
      conf.getLong(
          YarnConfiguration.RESOURCEMANAGER_CONNECT_MAX_WAIT_MS,
          YarnConfiguration.DEFAULT_RESOURCEMANAGER_CONNECT_MAX_WAIT_MS);
  long rmConnectionRetryIntervalMS =
      conf.getLong(
          YarnConfiguration.RESOURCEMANAGER_CONNECT_RETRY_INTERVAL_MS,
          YarnConfiguration
              .DEFAULT_RESOURCEMANAGER_CONNECT_RETRY_INTERVAL_MS);

  boolean waitForEver = (rmConnectWaitMS == -1);
  if (!waitForEver) {
    if (rmConnectWaitMS < 0) {
      throw new YarnRuntimeException("Invalid Configuration. "
          + YarnConfiguration.RESOURCEMANAGER_CONNECT_MAX_WAIT_MS
          + " can be -1, but can not be other negative numbers");
    }

    // try connect once
    if (rmConnectWaitMS < rmConnectionRetryIntervalMS) {
      LOG.warn(YarnConfiguration.RESOURCEMANAGER_CONNECT_MAX_WAIT_MS
          + " is smaller than "
          + YarnConfiguration.RESOURCEMANAGER_CONNECT_RETRY_INTERVAL_MS
          + ". Only try connect once.");
      rmConnectWaitMS = 0;
    }
  }

  // Handle HA case first
  if (HAUtil.isHAEnabled(conf)) {
    final long failoverSleepBaseMs = conf.getLong(
        YarnConfiguration.CLIENT_FAILOVER_SLEEPTIME_BASE_MS,
        rmConnectionRetryIntervalMS);

    final long failoverSleepMaxMs = conf.getLong(
        YarnConfiguration.CLIENT_FAILOVER_SLEEPTIME_MAX_MS,
        rmConnectionRetryIntervalMS);

    int maxFailoverAttempts = conf.getInt(
        YarnConfiguration.CLIENT_FAILOVER_MAX_ATTEMPTS, -1);

    if (maxFailoverAttempts == -1) {
      if (waitForEver) {
        maxFailoverAttempts = Integer.MAX_VALUE;
      } else {
        maxFailoverAttempts = (int) (rmConnectWaitMS / failoverSleepBaseMs);
      }
    }

    return RetryPolicies.failoverOnNetworkException(
        RetryPolicies.TRY_ONCE_THEN_FAIL, maxFailoverAttempts,
        failoverSleepBaseMs, failoverSleepMaxMs);
  }

  if (rmConnectionRetryIntervalMS < 0) {
    throw new YarnRuntimeException("Invalid Configuration. " +
        YarnConfiguration.RESOURCEMANAGER_CONNECT_RETRY_INTERVAL_MS +
        " should not be negative.");
  }

  RetryPolicy retryPolicy = null;
  if (waitForEver) {
    retryPolicy = RetryPolicies.RETRY_FOREVER;
  } else {
    retryPolicy =
        RetryPolicies.retryUpToMaximumTimeWithFixedSleep(rmConnectWaitMS,
            rmConnectionRetryIntervalMS, TimeUnit.MILLISECONDS);
  }

  Map<Class<? extends Exception>, RetryPolicy> exceptionToPolicyMap =
      new HashMap<Class<? extends Exception>, RetryPolicy>();

  exceptionToPolicyMap.put(EOFException.class, retryPolicy);
  exceptionToPolicyMap.put(ConnectException.class, retryPolicy);
  exceptionToPolicyMap.put(NoRouteToHostException.class, retryPolicy);
  exceptionToPolicyMap.put(UnknownHostException.class, retryPolicy);
  exceptionToPolicyMap.put(ConnectTimeoutException.class, retryPolicy);
  exceptionToPolicyMap.put(RetriableException.class, retryPolicy);
  exceptionToPolicyMap.put(SocketException.class, retryPolicy);

  return RetryPolicies.retryByException(
      RetryPolicies.TRY_ONCE_THEN_FAIL, exceptionToPolicyMap);
}