Java Code Examples for org.apache.hadoop.io.retry.RetryPolicies#retryByException()

The following examples show how to use org.apache.hadoop.io.retry.RetryPolicies#retryByException() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: AvatarZKShell.java    From RDFS with Apache License 2.0 6 votes vote down vote up 
private static AvatarProtocol createAvatarnode(AvatarProtocol rpcAvatarnode)
    throws IOException {
  RetryPolicy createPolicy = RetryPolicies
      .retryUpToMaximumCountWithFixedSleep(5, 5000, TimeUnit.MILLISECONDS);

  Map<Class<? extends Exception>, RetryPolicy> remoteExceptionToPolicyMap = new HashMap<Class<? extends Exception>, RetryPolicy>();

  Map<Class<? extends Exception>, RetryPolicy> exceptionToPolicyMap = new HashMap<Class<? extends Exception>, RetryPolicy>();
  exceptionToPolicyMap.put(RemoteException.class, RetryPolicies
      .retryByRemoteException(RetryPolicies.TRY_ONCE_THEN_FAIL,
          remoteExceptionToPolicyMap));
  RetryPolicy methodPolicy = RetryPolicies.retryByException(
      RetryPolicies.TRY_ONCE_THEN_FAIL, exceptionToPolicyMap);
  Map<String, RetryPolicy> methodNameToPolicyMap = new HashMap<String, RetryPolicy>();

  methodNameToPolicyMap.put("create", methodPolicy);

  return (AvatarProtocol) RetryProxy.create(AvatarProtocol.class,
      rpcAvatarnode, methodNameToPolicyMap);
}
Example 2
Source File: DFSClient.java    From hadoop-gpu with Apache License 2.0 6 votes vote down vote up 
private static ClientProtocol createNamenode(ClientProtocol rpcNamenode)
  throws IOException {
  RetryPolicy createPolicy = RetryPolicies.retryUpToMaximumCountWithFixedSleep(
      5, LEASE_SOFTLIMIT_PERIOD, TimeUnit.MILLISECONDS);
  
  Map<Class<? extends Exception>,RetryPolicy> remoteExceptionToPolicyMap =
    new HashMap<Class<? extends Exception>, RetryPolicy>();
  remoteExceptionToPolicyMap.put(AlreadyBeingCreatedException.class, createPolicy);

  Map<Class<? extends Exception>,RetryPolicy> exceptionToPolicyMap =
    new HashMap<Class<? extends Exception>, RetryPolicy>();
  exceptionToPolicyMap.put(RemoteException.class, 
      RetryPolicies.retryByRemoteException(
          RetryPolicies.TRY_ONCE_THEN_FAIL, remoteExceptionToPolicyMap));
  RetryPolicy methodPolicy = RetryPolicies.retryByException(
      RetryPolicies.TRY_ONCE_THEN_FAIL, exceptionToPolicyMap);
  Map<String,RetryPolicy> methodNameToPolicyMap = new HashMap<String,RetryPolicy>();
  
  methodNameToPolicyMap.put("create", methodPolicy);

  return (ClientProtocol) RetryProxy.create(ClientProtocol.class,
      rpcNamenode, methodNameToPolicyMap);
}
Example 3
Source File: S3FileSystem.java    From hadoop-gpu with Apache License 2.0 6 votes vote down vote up 
private static FileSystemStore createDefaultStore(Configuration conf) {
  FileSystemStore store = new Jets3tFileSystemStore();
  
  RetryPolicy basePolicy = RetryPolicies.retryUpToMaximumCountWithFixedSleep(
                                                                             conf.getInt("fs.s3.maxRetries", 4),
                                                                             conf.getLong("fs.s3.sleepTimeSeconds", 10), TimeUnit.SECONDS);
  Map<Class<? extends Exception>,RetryPolicy> exceptionToPolicyMap =
    new HashMap<Class<? extends Exception>, RetryPolicy>();
  exceptionToPolicyMap.put(IOException.class, basePolicy);
  exceptionToPolicyMap.put(S3Exception.class, basePolicy);
  
  RetryPolicy methodPolicy = RetryPolicies.retryByException(
                                                            RetryPolicies.TRY_ONCE_THEN_FAIL, exceptionToPolicyMap);
  Map<String,RetryPolicy> methodNameToPolicyMap = new HashMap<String,RetryPolicy>();
  methodNameToPolicyMap.put("storeBlock", methodPolicy);
  methodNameToPolicyMap.put("retrieveBlock", methodPolicy);
  
  return (FileSystemStore) RetryProxy.create(FileSystemStore.class,
                                             store, methodNameToPolicyMap);
}
Example 4
Source File: NativeS3FileSystem.java    From hadoop-gpu with Apache License 2.0 6 votes vote down vote up 
private static NativeFileSystemStore createDefaultStore(Configuration conf) {
  NativeFileSystemStore store = new Jets3tNativeFileSystemStore();
  
  RetryPolicy basePolicy = RetryPolicies.retryUpToMaximumCountWithFixedSleep(
      conf.getInt("fs.s3.maxRetries", 4),
      conf.getLong("fs.s3.sleepTimeSeconds", 10), TimeUnit.SECONDS);
  Map<Class<? extends Exception>, RetryPolicy> exceptionToPolicyMap =
    new HashMap<Class<? extends Exception>, RetryPolicy>();
  exceptionToPolicyMap.put(IOException.class, basePolicy);
  exceptionToPolicyMap.put(S3Exception.class, basePolicy);
  
  RetryPolicy methodPolicy = RetryPolicies.retryByException(
      RetryPolicies.TRY_ONCE_THEN_FAIL, exceptionToPolicyMap);
  Map<String, RetryPolicy> methodNameToPolicyMap =
    new HashMap<String, RetryPolicy>();
  methodNameToPolicyMap.put("storeFile", methodPolicy);
  
  return (NativeFileSystemStore)
    RetryProxy.create(NativeFileSystemStore.class, store,
        methodNameToPolicyMap);
}
Example 5
Source File: HighTideShell.java    From RDFS with Apache License 2.0 6 votes vote down vote up 
private static HighTideProtocol createHighTidenode(HighTideProtocol rpcHighTidenode)
  throws IOException {
  RetryPolicy createPolicy = RetryPolicies.retryUpToMaximumCountWithFixedSleep(
      5, 5000, TimeUnit.MILLISECONDS);

  Map<Class<? extends Exception>,RetryPolicy> remoteExceptionToPolicyMap =
    new HashMap<Class<? extends Exception>, RetryPolicy>();

  Map<Class<? extends Exception>,RetryPolicy> exceptionToPolicyMap =
    new HashMap<Class<? extends Exception>, RetryPolicy>();
  exceptionToPolicyMap.put(RemoteException.class,
      RetryPolicies.retryByRemoteException(
          RetryPolicies.TRY_ONCE_THEN_FAIL, remoteExceptionToPolicyMap));
  RetryPolicy methodPolicy = RetryPolicies.retryByException(
      RetryPolicies.TRY_ONCE_THEN_FAIL, exceptionToPolicyMap);
  Map<String,RetryPolicy> methodNameToPolicyMap = new HashMap<String,RetryPolicy>();

  methodNameToPolicyMap.put("create", methodPolicy);

  return (HighTideProtocol) RetryProxy.create(HighTideProtocol.class,
      rpcHighTidenode, methodNameToPolicyMap);
}
Example 6
Source File: DFSClient.java    From RDFS with Apache License 2.0 6 votes vote down vote up 
private static ClientProtocol createNamenode(ClientProtocol rpcNamenode,
    Configuration conf)
  throws IOException {
  long sleepTime = conf.getLong("dfs.client.rpc.retry.sleep",
      LEASE_SOFTLIMIT_PERIOD);
  RetryPolicy createPolicy = RetryPolicies.retryUpToMaximumCountWithFixedSleep(
      5, sleepTime, TimeUnit.MILLISECONDS);

  Map<Class<? extends Exception>,RetryPolicy> remoteExceptionToPolicyMap =
    new HashMap<Class<? extends Exception>, RetryPolicy>();
  remoteExceptionToPolicyMap.put(AlreadyBeingCreatedException.class, createPolicy);

  Map<Class<? extends Exception>,RetryPolicy> exceptionToPolicyMap =
    new HashMap<Class<? extends Exception>, RetryPolicy>();
  exceptionToPolicyMap.put(RemoteException.class,
      RetryPolicies.retryByRemoteException(
          RetryPolicies.TRY_ONCE_THEN_FAIL, remoteExceptionToPolicyMap));
  RetryPolicy methodPolicy = RetryPolicies.retryByException(
      RetryPolicies.TRY_ONCE_THEN_FAIL, exceptionToPolicyMap);
  Map<String,RetryPolicy> methodNameToPolicyMap = new HashMap<String,RetryPolicy>();

  methodNameToPolicyMap.put("create", methodPolicy);

  return (ClientProtocol) RetryProxy.create(ClientProtocol.class,
      rpcNamenode, methodNameToPolicyMap);
}
Example 7
Source File: S3FileSystem.java    From RDFS with Apache License 2.0 6 votes vote down vote up 
private static FileSystemStore createDefaultStore(Configuration conf) {
  FileSystemStore store = new Jets3tFileSystemStore();
  
  RetryPolicy basePolicy = RetryPolicies.retryUpToMaximumCountWithFixedSleep(
                                                                             conf.getInt("fs.s3.maxRetries", 4),
                                                                             conf.getLong("fs.s3.sleepTimeSeconds", 10), TimeUnit.SECONDS);
  Map<Class<? extends Exception>,RetryPolicy> exceptionToPolicyMap =
    new HashMap<Class<? extends Exception>, RetryPolicy>();
  exceptionToPolicyMap.put(IOException.class, basePolicy);
  exceptionToPolicyMap.put(S3Exception.class, basePolicy);
  
  RetryPolicy methodPolicy = RetryPolicies.retryByException(
                                                            RetryPolicies.TRY_ONCE_THEN_FAIL, exceptionToPolicyMap);
  Map<String,RetryPolicy> methodNameToPolicyMap = new HashMap<String,RetryPolicy>();
  methodNameToPolicyMap.put("storeBlock", methodPolicy);
  methodNameToPolicyMap.put("retrieveBlock", methodPolicy);
  
  return (FileSystemStore) RetryProxy.create(FileSystemStore.class,
                                             store, methodNameToPolicyMap);
}
Example 8
Source File: NativeS3FileSystem.java    From RDFS with Apache License 2.0 6 votes vote down vote up 
private static NativeFileSystemStore createDefaultStore(Configuration conf) {
  NativeFileSystemStore store = new Jets3tNativeFileSystemStore();
  
  RetryPolicy basePolicy = RetryPolicies.retryUpToMaximumCountWithFixedSleep(
      conf.getInt("fs.s3.maxRetries", 4),
      conf.getLong("fs.s3.sleepTimeSeconds", 10), TimeUnit.SECONDS);
  Map<Class<? extends Exception>, RetryPolicy> exceptionToPolicyMap =
    new HashMap<Class<? extends Exception>, RetryPolicy>();
  exceptionToPolicyMap.put(IOException.class, basePolicy);
  exceptionToPolicyMap.put(S3Exception.class, basePolicy);
  
  RetryPolicy methodPolicy = RetryPolicies.retryByException(
      RetryPolicies.TRY_ONCE_THEN_FAIL, exceptionToPolicyMap);
  Map<String, RetryPolicy> methodNameToPolicyMap =
    new HashMap<String, RetryPolicy>();
  methodNameToPolicyMap.put("storeFile", methodPolicy);
  
  return (NativeFileSystemStore)
    RetryProxy.create(NativeFileSystemStore.class, store,
        methodNameToPolicyMap);
}
Example 9
Source File: AvatarShell.java    From RDFS with Apache License 2.0 6 votes vote down vote up 
private static AvatarProtocol createAvatarnode(AvatarProtocol rpcAvatarnode)
    throws IOException {
  RetryPolicy createPolicy = RetryPolicies
      .retryUpToMaximumCountWithFixedSleep(5, 5000, TimeUnit.MILLISECONDS);

  Map<Class<? extends Exception>, RetryPolicy> remoteExceptionToPolicyMap = new HashMap<Class<? extends Exception>, RetryPolicy>();

  Map<Class<? extends Exception>, RetryPolicy> exceptionToPolicyMap = new HashMap<Class<? extends Exception>, RetryPolicy>();
  exceptionToPolicyMap.put(RemoteException.class, RetryPolicies
      .retryByRemoteException(RetryPolicies.TRY_ONCE_THEN_FAIL,
          remoteExceptionToPolicyMap));
  RetryPolicy methodPolicy = RetryPolicies.retryByException(
      RetryPolicies.TRY_ONCE_THEN_FAIL, exceptionToPolicyMap);
  Map<String, RetryPolicy> methodNameToPolicyMap = new HashMap<String, RetryPolicy>();

  methodNameToPolicyMap.put("create", methodPolicy);

  return (AvatarProtocol) RetryProxy.create(AvatarProtocol.class,
      rpcAvatarnode, methodNameToPolicyMap);
}
Example 10
Source File: SnapshotShell.java    From RDFS with Apache License 2.0 6 votes vote down vote up 
private static SnapshotProtocol createSnapshotNode(SnapshotProtocol rpcSnapshotNode) 
throws IOException {
  RetryPolicy createPolicy = RetryPolicies.retryUpToMaximumCountWithFixedSleep(
                                                 5, 5000, TimeUnit.MILLISECONDS);

  Map<Class<? extends Exception>,RetryPolicy> remoteExceptionToPolicyMap =
       new HashMap<Class<? extends Exception>, RetryPolicy>();
  Map<Class<? extends Exception>,RetryPolicy> exceptionToPolicyMap =
      new HashMap<Class<? extends Exception>, RetryPolicy>();

  exceptionToPolicyMap.put(RemoteException.class,
RetryPolicies.retryByRemoteException(
    RetryPolicies.TRY_ONCE_THEN_FAIL, remoteExceptionToPolicyMap));
  RetryPolicy methodPolicy = RetryPolicies.retryByException(
    RetryPolicies.TRY_ONCE_THEN_FAIL, exceptionToPolicyMap);

  Map<String,RetryPolicy> methodNameToPolicyMap = new HashMap<String,RetryPolicy>();

  methodNameToPolicyMap.put("create", methodPolicy);

  return (SnapshotProtocol) RetryProxy.create(SnapshotProtocol.class, 
                               rpcSnapshotNode, methodNameToPolicyMap);
}
Example 11
Source File: RaidShell.java    From RDFS with Apache License 2.0 6 votes vote down vote up 
private static RaidProtocol createRaidnode(RaidProtocol rpcRaidnode)
  throws IOException {
  RetryPolicy createPolicy = RetryPolicies.retryUpToMaximumCountWithFixedSleep(
      5, 5000, TimeUnit.MILLISECONDS);

  Map<Class<? extends Exception>,RetryPolicy> remoteExceptionToPolicyMap =
    new HashMap<Class<? extends Exception>, RetryPolicy>();

  Map<Class<? extends Exception>,RetryPolicy> exceptionToPolicyMap =
    new HashMap<Class<? extends Exception>, RetryPolicy>();
  exceptionToPolicyMap.put(RemoteException.class,
      RetryPolicies.retryByRemoteException(
          RetryPolicies.TRY_ONCE_THEN_FAIL, remoteExceptionToPolicyMap));
  RetryPolicy methodPolicy = RetryPolicies.retryByException(
      RetryPolicies.TRY_ONCE_THEN_FAIL, exceptionToPolicyMap);
  Map<String,RetryPolicy> methodNameToPolicyMap = new HashMap<String,RetryPolicy>();

  methodNameToPolicyMap.put("create", methodPolicy);

  return (RaidProtocol) RetryProxy.create(RaidProtocol.class,
      rpcRaidnode, methodNameToPolicyMap);
}
Example 12
Source File: S3FileSystem.java    From big-c with Apache License 2.0 6 votes vote down vote up 
private static FileSystemStore createDefaultStore(Configuration conf) {
  FileSystemStore store = new Jets3tFileSystemStore();
  
  RetryPolicy basePolicy = RetryPolicies.retryUpToMaximumCountWithFixedSleep(
                                                                             conf.getInt("fs.s3.maxRetries", 4),
                                                                             conf.getLong("fs.s3.sleepTimeSeconds", 10), TimeUnit.SECONDS);
  Map<Class<? extends Exception>,RetryPolicy> exceptionToPolicyMap =
    new HashMap<Class<? extends Exception>, RetryPolicy>();
  exceptionToPolicyMap.put(IOException.class, basePolicy);
  exceptionToPolicyMap.put(S3Exception.class, basePolicy);
  
  RetryPolicy methodPolicy = RetryPolicies.retryByException(
                                                            RetryPolicies.TRY_ONCE_THEN_FAIL, exceptionToPolicyMap);
  Map<String,RetryPolicy> methodNameToPolicyMap = new HashMap<String,RetryPolicy>();
  methodNameToPolicyMap.put("storeBlock", methodPolicy);
  methodNameToPolicyMap.put("retrieveBlock", methodPolicy);
  
  return (FileSystemStore) RetryProxy.create(FileSystemStore.class,
                                             store, methodNameToPolicyMap);
}
Example 13
Source File: S3FileSystem.java    From hadoop with Apache License 2.0 6 votes vote down vote up 
private static FileSystemStore createDefaultStore(Configuration conf) {
  FileSystemStore store = new Jets3tFileSystemStore();
  
  RetryPolicy basePolicy = RetryPolicies.retryUpToMaximumCountWithFixedSleep(
                                                                             conf.getInt("fs.s3.maxRetries", 4),
                                                                             conf.getLong("fs.s3.sleepTimeSeconds", 10), TimeUnit.SECONDS);
  Map<Class<? extends Exception>,RetryPolicy> exceptionToPolicyMap =
    new HashMap<Class<? extends Exception>, RetryPolicy>();
  exceptionToPolicyMap.put(IOException.class, basePolicy);
  exceptionToPolicyMap.put(S3Exception.class, basePolicy);
  
  RetryPolicy methodPolicy = RetryPolicies.retryByException(
                                                            RetryPolicies.TRY_ONCE_THEN_FAIL, exceptionToPolicyMap);
  Map<String,RetryPolicy> methodNameToPolicyMap = new HashMap<String,RetryPolicy>();
  methodNameToPolicyMap.put("storeBlock", methodPolicy);
  methodNameToPolicyMap.put("retrieveBlock", methodPolicy);
  
  return (FileSystemStore) RetryProxy.create(FileSystemStore.class,
                                             store, methodNameToPolicyMap);
}
Example 14
Source File: NativeS3FileSystem.java    From hadoop with Apache License 2.0 6 votes vote down vote up 
private static NativeFileSystemStore createDefaultStore(Configuration conf) {
  NativeFileSystemStore store = new Jets3tNativeFileSystemStore();
  
  RetryPolicy basePolicy = RetryPolicies.retryUpToMaximumCountWithFixedSleep(
      conf.getInt("fs.s3.maxRetries", 4),
      conf.getLong("fs.s3.sleepTimeSeconds", 10), TimeUnit.SECONDS);
  Map<Class<? extends Exception>, RetryPolicy> exceptionToPolicyMap =
    new HashMap<Class<? extends Exception>, RetryPolicy>();
  exceptionToPolicyMap.put(IOException.class, basePolicy);
  exceptionToPolicyMap.put(S3Exception.class, basePolicy);
  
  RetryPolicy methodPolicy = RetryPolicies.retryByException(
      RetryPolicies.TRY_ONCE_THEN_FAIL, exceptionToPolicyMap);
  Map<String, RetryPolicy> methodNameToPolicyMap =
    new HashMap<String, RetryPolicy>();
  methodNameToPolicyMap.put("storeFile", methodPolicy);
  methodNameToPolicyMap.put("rename", methodPolicy);
  
  return (NativeFileSystemStore)
    RetryProxy.create(NativeFileSystemStore.class, store,
        methodNameToPolicyMap);
}
Example 15
Source File: ServerProxy.java    From hadoop with Apache License 2.0 5 votes vote down vote up 
protected static RetryPolicy createRetryPolicy(Configuration conf,
    String maxWaitTimeStr, long defMaxWaitTime,
    String connectRetryIntervalStr, long defRetryInterval) {
  long maxWaitTime = conf.getLong(maxWaitTimeStr, defMaxWaitTime);
  long retryIntervalMS =
      conf.getLong(connectRetryIntervalStr, defRetryInterval);
  if (maxWaitTime == -1) {
    // wait forever.
    return RetryPolicies.RETRY_FOREVER;
  }

  Preconditions.checkArgument(maxWaitTime > 0, "Invalid Configuration. "
      + maxWaitTimeStr + " should be a positive value.");
  Preconditions.checkArgument(retryIntervalMS > 0, "Invalid Configuration. "
      + connectRetryIntervalStr + "should be a positive value.");

  RetryPolicy retryPolicy =
      RetryPolicies.retryUpToMaximumTimeWithFixedSleep(maxWaitTime,
        retryIntervalMS, TimeUnit.MILLISECONDS);

  Map<Class<? extends Exception>, RetryPolicy> exceptionToPolicyMap =
      new HashMap<Class<? extends Exception>, RetryPolicy>();
  exceptionToPolicyMap.put(EOFException.class, retryPolicy);
  exceptionToPolicyMap.put(ConnectException.class, retryPolicy);
  exceptionToPolicyMap.put(NoRouteToHostException.class, retryPolicy);
  exceptionToPolicyMap.put(UnknownHostException.class, retryPolicy);
  exceptionToPolicyMap.put(RetriableException.class, retryPolicy);
  exceptionToPolicyMap.put(SocketException.class, retryPolicy);
  exceptionToPolicyMap.put(NMNotYetReadyException.class, retryPolicy);

  return RetryPolicies.retryByException(RetryPolicies.TRY_ONCE_THEN_FAIL,
    exceptionToPolicyMap);
}
Example 16
Source File: Balancer.java    From hadoop-gpu with Apache License 2.0 5 votes vote down vote up 
private static NamenodeProtocol createNamenode(Configuration conf)
  throws IOException {
  InetSocketAddress nameNodeAddr = NameNode.getAddress(conf);
  RetryPolicy timeoutPolicy = RetryPolicies.exponentialBackoffRetry(
      5, 200, TimeUnit.MILLISECONDS);
  Map<Class<? extends Exception>,RetryPolicy> exceptionToPolicyMap =
    new HashMap<Class<? extends Exception>, RetryPolicy>();
  RetryPolicy methodPolicy = RetryPolicies.retryByException(
      timeoutPolicy, exceptionToPolicyMap);
  Map<String,RetryPolicy> methodNameToPolicyMap =
      new HashMap<String, RetryPolicy>();
  methodNameToPolicyMap.put("getBlocks", methodPolicy);

  UserGroupInformation ugi;
  try {
    ugi = UnixUserGroupInformation.login(conf);
  } catch (javax.security.auth.login.LoginException e) {
    throw new IOException(StringUtils.stringifyException(e));
  }

  return (NamenodeProtocol) RetryProxy.create(
      NamenodeProtocol.class,
      RPC.getProxy(NamenodeProtocol.class,
          NamenodeProtocol.versionID,
          nameNodeAddr,
          ugi,
          conf,
          NetUtils.getDefaultSocketFactory(conf)),
      methodNameToPolicyMap);
}
Example 17
Source File: Balancer.java    From RDFS with Apache License 2.0 5 votes vote down vote up 
private static NamenodeProtocol createNamenode(InetSocketAddress nameNodeAddr, Configuration conf)
  throws IOException {
  RetryPolicy timeoutPolicy = RetryPolicies.exponentialBackoffRetry(
      5, 200, TimeUnit.MILLISECONDS);
  Map<Class<? extends Exception>,RetryPolicy> exceptionToPolicyMap =
    new HashMap<Class<? extends Exception>, RetryPolicy>();
  RetryPolicy methodPolicy = RetryPolicies.retryByException(
      timeoutPolicy, exceptionToPolicyMap);
  Map<String,RetryPolicy> methodNameToPolicyMap =
      new HashMap<String, RetryPolicy>();
  methodNameToPolicyMap.put("getBlocks", methodPolicy);

  UserGroupInformation ugi;
  try {
    ugi = UnixUserGroupInformation.login(conf);
  } catch (javax.security.auth.login.LoginException e) {
    throw new IOException(StringUtils.stringifyException(e));
  }

  return (NamenodeProtocol) RetryProxy.create(
      NamenodeProtocol.class,
      RPC.getProxy(NamenodeProtocol.class,
          NamenodeProtocol.versionID,
          nameNodeAddr,
          ugi,
          conf,
          NetUtils.getDefaultSocketFactory(conf)),
      methodNameToPolicyMap);
}
Example 18
Source File: ServerProxy.java    From big-c with Apache License 2.0 5 votes vote down vote up 
protected static RetryPolicy createRetryPolicy(Configuration conf,
    String maxWaitTimeStr, long defMaxWaitTime,
    String connectRetryIntervalStr, long defRetryInterval) {
  long maxWaitTime = conf.getLong(maxWaitTimeStr, defMaxWaitTime);
  long retryIntervalMS =
      conf.getLong(connectRetryIntervalStr, defRetryInterval);
  if (maxWaitTime == -1) {
    // wait forever.
    return RetryPolicies.RETRY_FOREVER;
  }

  Preconditions.checkArgument(maxWaitTime > 0, "Invalid Configuration. "
      + maxWaitTimeStr + " should be a positive value.");
  Preconditions.checkArgument(retryIntervalMS > 0, "Invalid Configuration. "
      + connectRetryIntervalStr + "should be a positive value.");

  RetryPolicy retryPolicy =
      RetryPolicies.retryUpToMaximumTimeWithFixedSleep(maxWaitTime,
        retryIntervalMS, TimeUnit.MILLISECONDS);

  Map<Class<? extends Exception>, RetryPolicy> exceptionToPolicyMap =
      new HashMap<Class<? extends Exception>, RetryPolicy>();
  exceptionToPolicyMap.put(EOFException.class, retryPolicy);
  exceptionToPolicyMap.put(ConnectException.class, retryPolicy);
  exceptionToPolicyMap.put(NoRouteToHostException.class, retryPolicy);
  exceptionToPolicyMap.put(UnknownHostException.class, retryPolicy);
  exceptionToPolicyMap.put(RetriableException.class, retryPolicy);
  exceptionToPolicyMap.put(SocketException.class, retryPolicy);
  exceptionToPolicyMap.put(NMNotYetReadyException.class, retryPolicy);

  return RetryPolicies.retryByException(RetryPolicies.TRY_ONCE_THEN_FAIL,
    exceptionToPolicyMap);
}
Example 19
Source File: RMProxy.java    From big-c with Apache License 2.0 4 votes vote down vote up 
/**
 * Fetch retry policy from Configuration
 */
@Private
@VisibleForTesting
public static RetryPolicy createRetryPolicy(Configuration conf) {
  long rmConnectWaitMS =
      conf.getLong(
          YarnConfiguration.RESOURCEMANAGER_CONNECT_MAX_WAIT_MS,
          YarnConfiguration.DEFAULT_RESOURCEMANAGER_CONNECT_MAX_WAIT_MS);
  long rmConnectionRetryIntervalMS =
      conf.getLong(
          YarnConfiguration.RESOURCEMANAGER_CONNECT_RETRY_INTERVAL_MS,
          YarnConfiguration
              .DEFAULT_RESOURCEMANAGER_CONNECT_RETRY_INTERVAL_MS);

  boolean waitForEver = (rmConnectWaitMS == -1);
  if (!waitForEver) {
    if (rmConnectWaitMS < 0) {
      throw new YarnRuntimeException("Invalid Configuration. "
          + YarnConfiguration.RESOURCEMANAGER_CONNECT_MAX_WAIT_MS
          + " can be -1, but can not be other negative numbers");
    }

    // try connect once
    if (rmConnectWaitMS < rmConnectionRetryIntervalMS) {
      LOG.warn(YarnConfiguration.RESOURCEMANAGER_CONNECT_MAX_WAIT_MS
          + " is smaller than "
          + YarnConfiguration.RESOURCEMANAGER_CONNECT_RETRY_INTERVAL_MS
          + ". Only try connect once.");
      rmConnectWaitMS = 0;
    }
  }

  // Handle HA case first
  if (HAUtil.isHAEnabled(conf)) {
    final long failoverSleepBaseMs = conf.getLong(
        YarnConfiguration.CLIENT_FAILOVER_SLEEPTIME_BASE_MS,
        rmConnectionRetryIntervalMS);

    final long failoverSleepMaxMs = conf.getLong(
        YarnConfiguration.CLIENT_FAILOVER_SLEEPTIME_MAX_MS,
        rmConnectionRetryIntervalMS);

    int maxFailoverAttempts = conf.getInt(
        YarnConfiguration.CLIENT_FAILOVER_MAX_ATTEMPTS, -1);

    if (maxFailoverAttempts == -1) {
      if (waitForEver) {
        maxFailoverAttempts = Integer.MAX_VALUE;
      } else {
        maxFailoverAttempts = (int) (rmConnectWaitMS / failoverSleepBaseMs);
      }
    }

    return RetryPolicies.failoverOnNetworkException(
        RetryPolicies.TRY_ONCE_THEN_FAIL, maxFailoverAttempts,
        failoverSleepBaseMs, failoverSleepMaxMs);
  }

  if (rmConnectionRetryIntervalMS < 0) {
    throw new YarnRuntimeException("Invalid Configuration. " +
        YarnConfiguration.RESOURCEMANAGER_CONNECT_RETRY_INTERVAL_MS +
        " should not be negative.");
  }

  RetryPolicy retryPolicy = null;
  if (waitForEver) {
    retryPolicy = RetryPolicies.RETRY_FOREVER;
  } else {
    retryPolicy =
        RetryPolicies.retryUpToMaximumTimeWithFixedSleep(rmConnectWaitMS,
            rmConnectionRetryIntervalMS, TimeUnit.MILLISECONDS);
  }

  Map<Class<? extends Exception>, RetryPolicy> exceptionToPolicyMap =
      new HashMap<Class<? extends Exception>, RetryPolicy>();

  exceptionToPolicyMap.put(EOFException.class, retryPolicy);
  exceptionToPolicyMap.put(ConnectException.class, retryPolicy);
  exceptionToPolicyMap.put(NoRouteToHostException.class, retryPolicy);
  exceptionToPolicyMap.put(UnknownHostException.class, retryPolicy);
  exceptionToPolicyMap.put(ConnectTimeoutException.class, retryPolicy);
  exceptionToPolicyMap.put(RetriableException.class, retryPolicy);
  exceptionToPolicyMap.put(SocketException.class, retryPolicy);

  return RetryPolicies.retryByException(
      RetryPolicies.TRY_ONCE_THEN_FAIL, exceptionToPolicyMap);
}
Example 20
Source File: RMProxy.java    From hadoop with Apache License 2.0 4 votes vote down vote up 
/**
 * Fetch retry policy from Configuration
 */
@Private
@VisibleForTesting
public static RetryPolicy createRetryPolicy(Configuration conf) {
  long rmConnectWaitMS =
      conf.getLong(
          YarnConfiguration.RESOURCEMANAGER_CONNECT_MAX_WAIT_MS,
          YarnConfiguration.DEFAULT_RESOURCEMANAGER_CONNECT_MAX_WAIT_MS);
  long rmConnectionRetryIntervalMS =
      conf.getLong(
          YarnConfiguration.RESOURCEMANAGER_CONNECT_RETRY_INTERVAL_MS,
          YarnConfiguration
              .DEFAULT_RESOURCEMANAGER_CONNECT_RETRY_INTERVAL_MS);

  boolean waitForEver = (rmConnectWaitMS == -1);
  if (!waitForEver) {
    if (rmConnectWaitMS < 0) {
      throw new YarnRuntimeException("Invalid Configuration. "
          + YarnConfiguration.RESOURCEMANAGER_CONNECT_MAX_WAIT_MS
          + " can be -1, but can not be other negative numbers");
    }

    // try connect once
    if (rmConnectWaitMS < rmConnectionRetryIntervalMS) {
      LOG.warn(YarnConfiguration.RESOURCEMANAGER_CONNECT_MAX_WAIT_MS
          + " is smaller than "
          + YarnConfiguration.RESOURCEMANAGER_CONNECT_RETRY_INTERVAL_MS
          + ". Only try connect once.");
      rmConnectWaitMS = 0;
    }
  }

  // Handle HA case first
  if (HAUtil.isHAEnabled(conf)) {
    final long failoverSleepBaseMs = conf.getLong(
        YarnConfiguration.CLIENT_FAILOVER_SLEEPTIME_BASE_MS,
        rmConnectionRetryIntervalMS);

    final long failoverSleepMaxMs = conf.getLong(
        YarnConfiguration.CLIENT_FAILOVER_SLEEPTIME_MAX_MS,
        rmConnectionRetryIntervalMS);

    int maxFailoverAttempts = conf.getInt(
        YarnConfiguration.CLIENT_FAILOVER_MAX_ATTEMPTS, -1);

    if (maxFailoverAttempts == -1) {
      if (waitForEver) {
        maxFailoverAttempts = Integer.MAX_VALUE;
      } else {
        maxFailoverAttempts = (int) (rmConnectWaitMS / failoverSleepBaseMs);
      }
    }

    return RetryPolicies.failoverOnNetworkException(
        RetryPolicies.TRY_ONCE_THEN_FAIL, maxFailoverAttempts,
        failoverSleepBaseMs, failoverSleepMaxMs);
  }

  if (rmConnectionRetryIntervalMS < 0) {
    throw new YarnRuntimeException("Invalid Configuration. " +
        YarnConfiguration.RESOURCEMANAGER_CONNECT_RETRY_INTERVAL_MS +
        " should not be negative.");
  }

  RetryPolicy retryPolicy = null;
  if (waitForEver) {
    retryPolicy = RetryPolicies.RETRY_FOREVER;
  } else {
    retryPolicy =
        RetryPolicies.retryUpToMaximumTimeWithFixedSleep(rmConnectWaitMS,
            rmConnectionRetryIntervalMS, TimeUnit.MILLISECONDS);
  }

  Map<Class<? extends Exception>, RetryPolicy> exceptionToPolicyMap =
      new HashMap<Class<? extends Exception>, RetryPolicy>();

  exceptionToPolicyMap.put(EOFException.class, retryPolicy);
  exceptionToPolicyMap.put(ConnectException.class, retryPolicy);
  exceptionToPolicyMap.put(NoRouteToHostException.class, retryPolicy);
  exceptionToPolicyMap.put(UnknownHostException.class, retryPolicy);
  exceptionToPolicyMap.put(ConnectTimeoutException.class, retryPolicy);
  exceptionToPolicyMap.put(RetriableException.class, retryPolicy);
  exceptionToPolicyMap.put(SocketException.class, retryPolicy);

  return RetryPolicies.retryByException(
      RetryPolicies.TRY_ONCE_THEN_FAIL, exceptionToPolicyMap);
}