Java Code Examples for com.amazonaws.ClientConfiguration#setMaxErrorRetry()

protected ClientConfiguration createConfiguration() {
    ClientConfiguration configuration = new ClientConfiguration();

    // Proxy設定
    if (proxyHost != null && proxyPort != null) {
        configuration.setProxyHost(proxyHost);
        configuration.setProxyPort(proxyPort);

        if (proxyUser != null && proxyPassword != null) {
            configuration.setProxyUsername(proxyUser);
            configuration.setProxyPassword(proxyPassword);
        }
    }

    // リトライしない
    configuration.setMaxErrorRetry(0);

    return configuration;
}
 

static ClientConfiguration buildConfiguration(S3ClientSettings clientSettings) {
    final ClientConfiguration clientConfiguration = new ClientConfiguration();
    // the response metadata cache is only there for diagnostics purposes,
    // but can force objects from every response to the old generation.
    clientConfiguration.setResponseMetadataCacheSize(0);
    clientConfiguration.setProtocol(clientSettings.protocol);

    if (Strings.hasText(clientSettings.proxyHost)) {
        // TODO: remove this leniency, these settings should exist together and be validated
        clientConfiguration.setProxyHost(clientSettings.proxyHost);
        clientConfiguration.setProxyPort(clientSettings.proxyPort);
        clientConfiguration.setProxyUsername(clientSettings.proxyUsername);
        clientConfiguration.setProxyPassword(clientSettings.proxyPassword);
    }

    clientConfiguration.setMaxErrorRetry(clientSettings.maxRetries);
    clientConfiguration.setUseThrottleRetries(clientSettings.throttleRetries);
    clientConfiguration.setSocketTimeout(clientSettings.readTimeoutMillis);

    return clientConfiguration;
}
 

protected ClientConfiguration createConfiguration(final ProcessContext context) {
    final ClientConfiguration config = new ClientConfiguration();
    config.setMaxConnections(context.getMaxConcurrentTasks());
    config.setMaxErrorRetry(0);
    config.setUserAgent(DEFAULT_USER_AGENT);
    // If this is changed to be a property, ensure other uses are also changed
    config.setProtocol(DEFAULT_PROTOCOL);
    final int commsTimeout = context.getProperty(TIMEOUT).asTimePeriod(TimeUnit.MILLISECONDS).intValue();
    config.setConnectionTimeout(commsTimeout);
    config.setSocketTimeout(commsTimeout);

    final SSLContextService sslContextService = context.getProperty(SSL_CONTEXT_SERVICE).asControllerService(SSLContextService.class);
    if (sslContextService != null) {
        final SSLContext sslContext = sslContextService.createSSLContext(SSLContextService.ClientAuth.NONE);
        SdkTLSSocketFactory sdkTLSSocketFactory = new SdkTLSSocketFactory(sslContext, null);
        config.getApacheHttpClientConfig().setSslSocketFactory(sdkTLSSocketFactory);
    }

    if (context.getProperty(PROXY_HOST).isSet()) {
        String proxyHost = context.getProperty(PROXY_HOST).getValue();
        config.setProxyHost(proxyHost);
        Integer proxyPort = context.getProperty(PROXY_HOST_PORT).asInteger();
        config.setProxyPort(proxyPort);
    }

    return config;
}
 

public void run(String target_host, String target_path, String rest_endpoint, int times) {
  // create spark context and spark sql instance
  SparkSession sc = SparkSession.builder().appName("Uber").getOrCreate();
  sc.conf().set("fs.s3a.attempts.maximum", "30");
  sc.conf().set("fs.s3.impl", "org.apache.hadoop.fs.s3a.S3AFileSystem");
  // conf.set("mapreduce.fileoutputcommitter.algorithm.version", "2")

  // set the S3 client configuration and create a client connected to S3.
  ClientConfiguration configuration = new ClientConfiguration();
  configuration.setMaxErrorRetry(10);
  configuration.setConnectionTimeout(501000);
  configuration.setSocketTimeout(501000);
  //configuration.setUseTcpKeepAlive(true);
  AmazonS3Client s3client = new AmazonS3Client(configuration);

  if (!awsAccessKey.trim().equals("") && !awsSecretKey.trim().equals("")) {
    logInfo("Read S3 credentials", true);
    s3client = new AmazonS3Client(new BasicAWSCredentials(awsAccessKey, awsSecretKey), configuration);
    sc.conf().set("fs.s3a.access.key", awsAccessKey);
    sc.conf().set("fs.s3a.secret.key", awsSecretKey);
  }

  logInfo("Start process", true);
  for (String schema_folder : schema_folders) {
    logInfo("Start to load schema folder " + schema_folder, true);
    post2Tg(s3client, sc, schema_folder, target_host, target_path, rest_endpoint, times);
    logInfo("Loding schema folder " + schema_folder + " successfully", true);
  }
  logInfo("End process", true);
}
 

public AmazonS3 newInstance(Configuration conf) {
  int maxErrorRetry = conf.getInt(ConfigurationVariable.UPLOAD_RETRY_COUNT.getName(),
      ConfigurationVariable.UPLOAD_RETRY_COUNT.defaultIntValue());
  long errorRetryDelay = conf.getLong(ConfigurationVariable.UPLOAD_RETRY_DELAY_MS.getName(),
      ConfigurationVariable.UPLOAD_RETRY_DELAY_MS.defaultLongValue());

  LOG.info("Creating AWS S3 client with a retry policy of {} retries and {} ms of exponential backoff delay",
      maxErrorRetry, errorRetryDelay);

  RetryPolicy retryPolicy = new RetryPolicy(new CounterBasedRetryCondition(maxErrorRetry),
      new ExponentialBackoffStrategy(errorRetryDelay), maxErrorRetry, true);
  ClientConfiguration clientConfiguration = new ClientConfiguration();
  clientConfiguration.setRetryPolicy(retryPolicy);
  clientConfiguration.setMaxErrorRetry(maxErrorRetry);

  AmazonS3ClientBuilder builder = AmazonS3ClientBuilder
      .standard()
      .withCredentials(new HadoopAWSCredentialProviderChain(conf))
      .withClientConfiguration(clientConfiguration);

  EndpointConfiguration endpointConfiguration = getEndpointConfiguration(conf);
  if (endpointConfiguration != null) {
    builder.withEndpointConfiguration(endpointConfiguration);
  } else {
    builder.withRegion(getRegion(conf));
  }

  return builder.build();
}
 

@Override protected AmazonDynamoDB newClient(int i) {
  ClientConfiguration clientConfiguration = new ClientConfiguration();
  clientConfiguration.setMaxErrorRetry(0);
  clientConfiguration.setRequestTimeout(1000);

  return AmazonDynamoDBClientBuilder.standard()
      .withCredentials(new AWSStaticCredentialsProvider(new BasicAWSCredentials("x", "y")))
      .withEndpointConfiguration(
          new AwsClientBuilder.EndpointConfiguration("http://127.0.0.1:" + i, "us-east-1"))
      .withRequestHandlers(new TracingRequestHandler(httpTracing))
      .withClientConfiguration(clientConfiguration)
      .build();
}
 

/**
 * Creates a SnowflakeS3ClientObject which encapsulates
 * the Amazon S3 client
 *
 * @param stageCredentials Map of stage credential properties
 * @param parallel         degree of parallelism
 * @param encMat           encryption material for the client
 * @param stageRegion      the region where the stage is located
 * @param stageEndPoint    the FIPS endpoint for the stage, if needed
 * @return the SnowflakeS3Client  instance created
 * @throws SnowflakeSQLException failure to create the S3 client
 */
private SnowflakeS3Client createS3Client(Map<?, ?> stageCredentials,
                                         int parallel,
                                         RemoteStoreFileEncryptionMaterial encMat,
                                         String stageRegion,
                                         String stageEndPoint)
throws SnowflakeSQLException
{
  final int S3_TRANSFER_MAX_RETRIES = 3;

  logger.debug("createS3Client encryption={}", (encMat == null ? "no" : "yes"));

  SnowflakeS3Client s3Client;

  ClientConfiguration clientConfig = new ClientConfiguration();
  clientConfig.setMaxConnections(parallel + 1);
  clientConfig.setMaxErrorRetry(S3_TRANSFER_MAX_RETRIES);
  clientConfig.setDisableSocketProxy(HttpUtil.isSocksProxyDisabled());

  logger.debug("s3 client configuration: maxConnection={}, connectionTimeout={}, " +
               "socketTimeout={}, maxErrorRetry={}",
               clientConfig.getMaxConnections(),
               clientConfig.getConnectionTimeout(),
               clientConfig.getSocketTimeout(),
               clientConfig.getMaxErrorRetry());

  try
  {
    s3Client = new SnowflakeS3Client(stageCredentials, clientConfig, encMat, stageRegion, stageEndPoint);
  }
  catch (Exception ex)
  {
    logger.debug("Exception creating s3 client", ex);
    throw ex;
  }
  logger.debug("s3 client created");

  return s3Client;
}
 

/**
 * Initializes connection management
 *
 * @param conf Hadoop configuration
 * @param clientConf client SDK configuration
 */
private void initConnectionSettings(Configuration conf,
    ClientConfiguration clientConf) throws IOException {
  clientConf.setMaxConnections(Utils.getInt(conf, FS_COS, FS_ALT_KEYS,
      MAXIMUM_CONNECTIONS, DEFAULT_MAXIMUM_CONNECTIONS));
  clientConf.setClientExecutionTimeout(Utils.getInt(conf, FS_COS, FS_ALT_KEYS,
      CLIENT_EXEC_TIMEOUT, DEFAULT_CLIENT_EXEC_TIMEOUT));
  clientConf.setMaxErrorRetry(Utils.getInt(conf, FS_COS, FS_ALT_KEYS,
      MAX_ERROR_RETRIES, DEFAULT_MAX_ERROR_RETRIES));
  clientConf.setConnectionTimeout(Utils.getInt(conf, FS_COS, FS_ALT_KEYS,
      ESTABLISH_TIMEOUT, DEFAULT_ESTABLISH_TIMEOUT));
  clientConf.setSocketTimeout(Utils.getInt(conf, FS_COS, FS_ALT_KEYS,
      SOCKET_TIMEOUT, DEFAULT_SOCKET_TIMEOUT));
  clientConf.setRequestTimeout(Utils.getInt(conf, FS_COS, FS_ALT_KEYS,
      REQUEST_TIMEOUT, DEFAULT_REQUEST_TIMEOUT));
  int sockSendBuffer = Utils.getInt(conf, FS_COS, FS_ALT_KEYS,
      SOCKET_SEND_BUFFER, DEFAULT_SOCKET_SEND_BUFFER);
  int sockRecvBuffer = Utils.getInt(conf, FS_COS, FS_ALT_KEYS,
      SOCKET_RECV_BUFFER, DEFAULT_SOCKET_RECV_BUFFER);
  clientConf.setSocketBufferSizeHints(sockSendBuffer, sockRecvBuffer);
  String signerOverride = Utils.getTrimmed(conf, FS_COS, FS_ALT_KEYS,
      SIGNING_ALGORITHM, "");
  if (!signerOverride.isEmpty()) {
    LOG.debug("Signer override = {}", signerOverride);
    clientConf.setSignerOverride(signerOverride);
  }

  String userAgentPrefix = Utils.getTrimmed(conf, FS_COS, FS_ALT_KEYS,
      USER_AGENT_PREFIX, DEFAULT_USER_AGENT_PREFIX);
  String userAgentName = singletoneInitTimeData.getUserAgentName();
  if (!userAgentPrefix.equals(DEFAULT_USER_AGENT_PREFIX)) {
    userAgentName = userAgentPrefix + " " + userAgentName;
  }
  clientConf.setUserAgentPrefix(userAgentName);
}
 

public ECSService(String credentialsId, String regionName) {
    this.clientSupplier = () -> {
        ProxyConfiguration proxy = Jenkins.get().proxy;
        ClientConfiguration clientConfiguration = new ClientConfiguration();

        if (proxy != null) {
            clientConfiguration.setProxyHost(proxy.name);
            clientConfiguration.setProxyPort(proxy.port);
            clientConfiguration.setProxyUsername(proxy.getUserName());
            clientConfiguration.setProxyPassword(proxy.getPassword());
        }

        // Default is 3. 10 helps us actually utilize the SDK's backoff strategy
        // The strategy will wait up to 20 seconds per request (after multiple failures)
        clientConfiguration.setMaxErrorRetry(10);

        AmazonECSClientBuilder builder = AmazonECSClientBuilder
                .standard()
                .withClientConfiguration(clientConfiguration)
                .withRegion(regionName);

        AmazonWebServicesCredentials credentials = getCredentials(credentialsId);
        if (credentials != null) {
            if (LOGGER.isLoggable(Level.FINE)) {
                String awsAccessKeyId = credentials.getCredentials().getAWSAccessKeyId();
                String obfuscatedAccessKeyId = StringUtils.left(awsAccessKeyId, 4) + StringUtils.repeat("*", awsAccessKeyId.length() - (2 * 4)) + StringUtils.right(awsAccessKeyId, 4);
                LOGGER.log(Level.FINE, "Connect to Amazon ECS with IAM Access Key {1}", new Object[]{obfuscatedAccessKeyId});
            }
            builder
                    .withCredentials(credentials);
        }
        LOGGER.log(Level.FINE, "Selected Region: {0}", regionName);

        return builder.build();
    };
}
 

private void createConnection(
    Stage.Context context,
    String configPrefix,
    ProxyConfig proxyConfig,
    List<Stage.ConfigIssue> issues,
    int maxErrorRetries
) throws StageException {
  AWSCredentialsProvider credentials = AWSUtil.getCredentialsProvider(awsConfig);
  ClientConfiguration clientConfig = AWSUtil.getClientConfiguration(proxyConfig);

  if (maxErrorRetries >= 0) {
    clientConfig.setMaxErrorRetry(maxErrorRetries);
  }

  AmazonS3ClientBuilder builder = AmazonS3ClientBuilder
      .standard()
      .withCredentials(credentials)
      .withClientConfiguration(clientConfig)
      .withChunkedEncodingDisabled(awsConfig.disableChunkedEncoding)
      .withPathStyleAccessEnabled(usePathAddressModel);

  if (region == AwsRegion.OTHER) {
    if (endpoint == null || endpoint.isEmpty()) {
      issues.add(context.createConfigIssue(Groups.S3.name(), configPrefix + "endpoint", Errors.S3_SPOOLDIR_10));
      return;
    }
    builder.withEndpointConfiguration(new AwsClientBuilder.EndpointConfiguration(endpoint, null));
  } else {
    builder.withRegion(region.getId());
  }
  s3Client = builder.build();
}
 

/**
  * Configures this appender instance and makes it ready for use by the
  * consumers. It validates mandatory parameters and confirms if the configured
  * stream is ready for publishing data yet.
  * 
  * Error details are made available through the fallback handler for this
  * appender
  * 
  * @throws IllegalStateException
  *           if we encounter issues configuring this appender instance
  */
 @Override
 public void activateOptions() {
   if (streamName == null) {
     initializationFailed = true;
     error("Invalid configuration - streamName cannot be null for appender: " + name);
   }

   if (layout == null) {
     initializationFailed = true;
     error("Invalid configuration - No layout for appender: " + name);
   }

   ClientConfiguration clientConfiguration = new ClientConfiguration();
   clientConfiguration = setProxySettingsFromSystemProperties(clientConfiguration);

   clientConfiguration.setMaxErrorRetry(maxRetries);
   clientConfiguration.setRetryPolicy(new RetryPolicy(PredefinedRetryPolicies.DEFAULT_RETRY_CONDITION,
       PredefinedRetryPolicies.DEFAULT_BACKOFF_STRATEGY, maxRetries, true));
   clientConfiguration.setUserAgent(AppenderConstants.USER_AGENT_STRING);

   BlockingQueue<Runnable> taskBuffer = new LinkedBlockingDeque<Runnable>(bufferSize);
   ThreadPoolExecutor threadPoolExecutor = new ThreadPoolExecutor(threadCount, threadCount,
       AppenderConstants.DEFAULT_THREAD_KEEP_ALIVE_SEC, TimeUnit.SECONDS, taskBuffer, new BlockFastProducerPolicy());
   threadPoolExecutor.prestartAllCoreThreads();
   kinesisClient = new AmazonKinesisAsyncClient(new CustomCredentialsProviderChain(), clientConfiguration,
       threadPoolExecutor);

   boolean regionProvided = !Validator.isBlank(region);
   if (!regionProvided) {
     region = AppenderConstants.DEFAULT_REGION;
   }
   if (!Validator.isBlank(endpoint)) {
     if (regionProvided) {
LOGGER
    .warn("Received configuration for both region as well as Amazon Kinesis endpoint. ("
	+ endpoint
	+ ") will be used as endpoint instead of default endpoint for region ("
	+ region + ")");
     }
     kinesisClient.setEndpoint(endpoint,
  AppenderConstants.DEFAULT_SERVICE_NAME, region);
   } else {
     kinesisClient.setRegion(Region.getRegion(Regions.fromName(region)));
   }

   DescribeStreamResult describeResult = null;
   try {
     describeResult = kinesisClient.describeStream(streamName);
     String streamStatus = describeResult.getStreamDescription().getStreamStatus();
     if (!StreamStatus.ACTIVE.name().equals(streamStatus) && !StreamStatus.UPDATING.name().equals(streamStatus)) {
       initializationFailed = true;
       error("Stream " + streamName + " is not ready (in active/updating status) for appender: " + name);
     }
   } catch (ResourceNotFoundException rnfe) {
     initializationFailed = true;
     error("Stream " + streamName + " doesn't exist for appender: " + name, rnfe);
   }

   asyncCallHander = new AsyncPutCallStatsReporter(name);
 }
 

protected ClientConfiguration createConfiguration(final ProcessContext context) {
    final ClientConfiguration config = new ClientConfiguration();
    config.setMaxConnections(context.getMaxConcurrentTasks());
    config.setMaxErrorRetry(0);
    config.setUserAgent(DEFAULT_USER_AGENT);
    // If this is changed to be a property, ensure other uses are also changed
    config.setProtocol(DEFAULT_PROTOCOL);
    final int commsTimeout = context.getProperty(TIMEOUT).asTimePeriod(TimeUnit.MILLISECONDS).intValue();
    config.setConnectionTimeout(commsTimeout);
    config.setSocketTimeout(commsTimeout);

    if(this.getSupportedPropertyDescriptors().contains(SSL_CONTEXT_SERVICE)) {
        final SSLContextService sslContextService = context.getProperty(SSL_CONTEXT_SERVICE).asControllerService(SSLContextService.class);
        if (sslContextService != null) {
            final SSLContext sslContext = sslContextService.createSSLContext(SslContextFactory.ClientAuth.NONE);
            // NIFI-3788: Changed hostnameVerifier from null to DHV (BrowserCompatibleHostnameVerifier is deprecated)
            SdkTLSSocketFactory sdkTLSSocketFactory = new SdkTLSSocketFactory(sslContext, new DefaultHostnameVerifier());
            config.getApacheHttpClientConfig().setSslSocketFactory(sdkTLSSocketFactory);
        }
    }

    final ProxyConfiguration proxyConfig = ProxyConfiguration.getConfiguration(context, () -> {
        if (context.getProperty(PROXY_HOST).isSet()) {
            final ProxyConfiguration componentProxyConfig = new ProxyConfiguration();
            String proxyHost = context.getProperty(PROXY_HOST).evaluateAttributeExpressions().getValue();
            Integer proxyPort = context.getProperty(PROXY_HOST_PORT).evaluateAttributeExpressions().asInteger();
            String proxyUsername = context.getProperty(PROXY_USERNAME).evaluateAttributeExpressions().getValue();
            String proxyPassword = context.getProperty(PROXY_PASSWORD).evaluateAttributeExpressions().getValue();
            componentProxyConfig.setProxyType(Proxy.Type.HTTP);
            componentProxyConfig.setProxyServerHost(proxyHost);
            componentProxyConfig.setProxyServerPort(proxyPort);
            componentProxyConfig.setProxyUserName(proxyUsername);
            componentProxyConfig.setProxyUserPassword(proxyPassword);
            return componentProxyConfig;
        }
        return ProxyConfiguration.DIRECT_CONFIGURATION;
    });

    if (Proxy.Type.HTTP.equals(proxyConfig.getProxyType())) {
        config.setProxyHost(proxyConfig.getProxyServerHost());
        config.setProxyPort(proxyConfig.getProxyServerPort());

        if (proxyConfig.hasCredential()) {
            config.setProxyUsername(proxyConfig.getProxyUserName());
            config.setProxyPassword(proxyConfig.getProxyUserPassword());
        }
    }

    return config;
}
 

public static TransferManager getTransferManager(final ClientOptions clientOptions) {

        if(TRANSFERMANAGER_ACCESSKEY_MAP.containsKey(clientOptions.getAccessKey())) {
            return TRANSFERMANAGER_ACCESSKEY_MAP.get(clientOptions.getAccessKey());
        }

        final AWSCredentials basicAWSCredentials = new BasicAWSCredentials(clientOptions.getAccessKey(), clientOptions.getSecretKey());

        final ClientConfiguration configuration = new ClientConfiguration();

        if (clientOptions.isHttps() != null) {
            configuration.setProtocol(clientOptions.isHttps() ? HTTPS : HTTP);
        }

        if (clientOptions.getConnectionTimeout() != null) {
            configuration.setConnectionTimeout(clientOptions.getConnectionTimeout());
        }

        if (clientOptions.getMaxErrorRetry() != null) {
            configuration.setMaxErrorRetry(clientOptions.getMaxErrorRetry());
        }

        if (clientOptions.getSocketTimeout() != null) {
            configuration.setSocketTimeout(clientOptions.getSocketTimeout());
        }

        if (clientOptions.getUseTCPKeepAlive() != null) {
            configuration.setUseTcpKeepAlive(clientOptions.getUseTCPKeepAlive());
        }

        if (clientOptions.getConnectionTtl() != null) {
            configuration.setConnectionTTL(clientOptions.getConnectionTtl());
        }

        if (clientOptions.getSigner() != null) {

            configuration.setSignerOverride(clientOptions.getSigner());
        }

        LOGGER.debug(format("Creating S3 client with configuration: [protocol: %1$s, signer: %2$s, connectionTimeOut: %3$s, maxErrorRetry: %4$s, socketTimeout: %5$s, useTCPKeepAlive: %6$s, connectionTtl: %7$s]",
                configuration.getProtocol(), configuration.getSignerOverride(), configuration.getConnectionTimeout(), configuration.getMaxErrorRetry(), configuration.getSocketTimeout(),
                clientOptions.getUseTCPKeepAlive(), clientOptions.getConnectionTtl()));

        final AmazonS3Client client = new AmazonS3Client(basicAWSCredentials, configuration);

        if (isNotBlank(clientOptions.getEndPoint())) {
            LOGGER.debug(format("Setting the end point for S3 client with access key %1$s to %2$s.", clientOptions.getAccessKey(), clientOptions.getEndPoint()));

            client.setEndpoint(clientOptions.getEndPoint());
        }

        TRANSFERMANAGER_ACCESSKEY_MAP.put(clientOptions.getAccessKey(), new TransferManager(client));

        return TRANSFERMANAGER_ACCESSKEY_MAP.get(clientOptions.getAccessKey());
    }