Java Code Examples for com.amazonaws.ClientConfiguration#setConnectionTimeout()

public static ClientConfiguration getClientConfiguration(ProxyConfig config) throws StageException {
  ClientConfiguration clientConfig = new ClientConfiguration();

  clientConfig.setConnectionTimeout(config.connectionTimeout * MILLIS);
  clientConfig.setSocketTimeout(config.socketTimeout * MILLIS);
  clientConfig.withMaxErrorRetry(config.retryCount);

  // Optional proxy settings
  if (config.useProxy) {
    if (config.proxyHost != null && !config.proxyHost.isEmpty()) {
      clientConfig.setProxyHost(config.proxyHost);
      clientConfig.setProxyPort(config.proxyPort);

      if (config.proxyUser != null && !config.proxyUser.get().isEmpty()) {
        clientConfig.setProxyUsername(config.proxyUser.get());
      }

      if (config.proxyPassword != null) {
        clientConfig.setProxyPassword(config.proxyPassword.get());
      }
    }
  }
  return clientConfig;
}
 

ClientConfiguration createClientConfiguration() throws StageException {
  ClientConfiguration clientConfig = new ClientConfiguration();

  clientConfig.setConnectionTimeout(connectionConfigs.getConnectionTimeoutMillis());
  clientConfig.setSocketTimeout(connectionConfigs.getSocketTimeoutMillis());
  clientConfig.withMaxErrorRetry(connectionConfigs.getMaxErrorRetry());

  if (connectionConfigs.isProxyEnabled()) {
    clientConfig.setProxyHost(connectionConfigs.getProxyHost());
    clientConfig.setProxyPort(connectionConfigs.getProxyPort());
    if (connectionConfigs.isProxyAuthenticationEnabled()) {
      clientConfig.setProxyUsername(connectionConfigs.getProxyUser().get());
      clientConfig.setProxyPassword(connectionConfigs.getProxyPassword().get());
    }
  }
  return clientConfig;
}
 

protected ClientConfiguration createConfiguration(final ProcessContext context) {
    final ClientConfiguration config = new ClientConfiguration();
    config.setMaxConnections(context.getMaxConcurrentTasks());
    config.setMaxErrorRetry(0);
    config.setUserAgent(DEFAULT_USER_AGENT);
    // If this is changed to be a property, ensure other uses are also changed
    config.setProtocol(DEFAULT_PROTOCOL);
    final int commsTimeout = context.getProperty(TIMEOUT).asTimePeriod(TimeUnit.MILLISECONDS).intValue();
    config.setConnectionTimeout(commsTimeout);
    config.setSocketTimeout(commsTimeout);

    final SSLContextService sslContextService = context.getProperty(SSL_CONTEXT_SERVICE).asControllerService(SSLContextService.class);
    if (sslContextService != null) {
        final SSLContext sslContext = sslContextService.createSSLContext(SSLContextService.ClientAuth.NONE);
        SdkTLSSocketFactory sdkTLSSocketFactory = new SdkTLSSocketFactory(sslContext, null);
        config.getApacheHttpClientConfig().setSslSocketFactory(sdkTLSSocketFactory);
    }

    if (context.getProperty(PROXY_HOST).isSet()) {
        String proxyHost = context.getProperty(PROXY_HOST).getValue();
        config.setProxyHost(proxyHost);
        Integer proxyPort = context.getProperty(PROXY_HOST_PORT).asInteger();
        config.setProxyPort(proxyPort);
    }

    return config;
}
 

public void run(String target_host, String target_path, String rest_endpoint, int times) {
  // create spark context and spark sql instance
  SparkSession sc = SparkSession.builder().appName("Uber").getOrCreate();
  sc.conf().set("fs.s3a.attempts.maximum", "30");
  sc.conf().set("fs.s3.impl", "org.apache.hadoop.fs.s3a.S3AFileSystem");
  // conf.set("mapreduce.fileoutputcommitter.algorithm.version", "2")

  // set the S3 client configuration and create a client connected to S3.
  ClientConfiguration configuration = new ClientConfiguration();
  configuration.setMaxErrorRetry(10);
  configuration.setConnectionTimeout(501000);
  configuration.setSocketTimeout(501000);
  //configuration.setUseTcpKeepAlive(true);
  AmazonS3Client s3client = new AmazonS3Client(configuration);

  if (!awsAccessKey.trim().equals("") && !awsSecretKey.trim().equals("")) {
    logInfo("Read S3 credentials", true);
    s3client = new AmazonS3Client(new BasicAWSCredentials(awsAccessKey, awsSecretKey), configuration);
    sc.conf().set("fs.s3a.access.key", awsAccessKey);
    sc.conf().set("fs.s3a.secret.key", awsSecretKey);
  }

  logInfo("Start process", true);
  for (String schema_folder : schema_folders) {
    logInfo("Start to load schema folder " + schema_folder, true);
    post2Tg(s3client, sc, schema_folder, target_host, target_path, rest_endpoint, times);
    logInfo("Loding schema folder " + schema_folder + " successfully", true);
  }
  logInfo("End process", true);
}
 

/**
 * One-time initialization of resources for this Lambda function.
 */
public LambdaAggregatingForwarder()
{
    this.aggregator = new RecordAggregator();
    
    /*
     * If the Kinesis stream you're forwarding to is in the same account as this AWS Lambda function, you can just give the IAM Role executing
     * this function permissions to publish to the stream and DefaultAWSCredentialsProviderChain() will take care of it.  
     * 
     * If you're publishing to a Kinesis stream in another AWS account, it's trickier.  Kinesis doesn't currently provide cross-account publishing
     * permissions. You must create an IAM role in the AWS account with the DESTINATION stream that has permissions to publish to that stream
     * (fill that IAM role's ARN in for "<RoleToAssumeARN>" below) and then the IAM role executing this Lambda function must be given permission
     * to assume the role "<RoleToAssumeARN>" from the other AWS account.  
     */
    AWSCredentialsProvider provider = new DefaultAWSCredentialsProviderChain();
    //AWSCredentialsProvider provider = new STSAssumeRoleSessionCredentialsProvider(new DefaultAWSCredentialsProviderChain(), "<RoleToAssumeARN>", "KinesisForwarder");
    
    //Set max conns to 1 since we use this client serially
    ClientConfiguration kinesisConfig = new ClientConfiguration();
    kinesisConfig.setMaxConnections(1);
    kinesisConfig.setProtocol(Protocol.HTTPS);
    kinesisConfig.setConnectionTimeout(DESTINATION_CONNECTION_TIMEOUT);
    kinesisConfig.setSocketTimeout(DESTINATION_SOCKET_TIMEOUT);
    
    this.kinesisForwarder = new AmazonKinesisClient(provider, kinesisConfig);
    this.kinesisForwarder.setRegion(Region.getRegion(DESTINATION_STREAM_REGION));
}
 

/**
 * Create a new Kinesis producer for publishing to Kinesis.
 * 
 * @param region The region of the Kinesis stream to publish to.
 * 
 * @return An Amazon Kinesis producer for publishing to a Kinesis stream.
 */
public static AmazonKinesis getKinesisProducer(String region)
{
    ClientConfiguration config = new ClientConfiguration();
    config.setMaxConnections(25);
    config.setConnectionTimeout(60000);
    config.setSocketTimeout(60000);

    AmazonKinesis producer = new AmazonKinesisClient(new DefaultAWSCredentialsProviderChain(), config);
    producer.setRegion(Region.getRegion(Regions.fromName(region)));

    return producer;
}
 

@Test
public void testAwsHttpClientConfigurationSerializationDeserialization() throws Exception {
  ClientConfiguration clientConfiguration = new ClientConfiguration();
  clientConfiguration.setConnectionTimeout(100);
  clientConfiguration.setConnectionMaxIdleMillis(1000);
  clientConfiguration.setSocketTimeout(300);

  final String valueAsJson = objectMapper.writeValueAsString(clientConfiguration);
  final ClientConfiguration clientConfigurationDeserialized =
      objectMapper.readValue(valueAsJson, ClientConfiguration.class);
  assertEquals(100, clientConfigurationDeserialized.getConnectionTimeout());
  assertEquals(1000, clientConfigurationDeserialized.getConnectionMaxIdleMillis());
  assertEquals(300, clientConfigurationDeserialized.getSocketTimeout());
}
 

/**
 * Initializes connection management
 *
 * @param conf Hadoop configuration
 * @param clientConf client SDK configuration
 */
private void initConnectionSettings(Configuration conf,
    ClientConfiguration clientConf) throws IOException {
  clientConf.setMaxConnections(Utils.getInt(conf, FS_COS, FS_ALT_KEYS,
      MAXIMUM_CONNECTIONS, DEFAULT_MAXIMUM_CONNECTIONS));
  clientConf.setClientExecutionTimeout(Utils.getInt(conf, FS_COS, FS_ALT_KEYS,
      CLIENT_EXEC_TIMEOUT, DEFAULT_CLIENT_EXEC_TIMEOUT));
  clientConf.setMaxErrorRetry(Utils.getInt(conf, FS_COS, FS_ALT_KEYS,
      MAX_ERROR_RETRIES, DEFAULT_MAX_ERROR_RETRIES));
  clientConf.setConnectionTimeout(Utils.getInt(conf, FS_COS, FS_ALT_KEYS,
      ESTABLISH_TIMEOUT, DEFAULT_ESTABLISH_TIMEOUT));
  clientConf.setSocketTimeout(Utils.getInt(conf, FS_COS, FS_ALT_KEYS,
      SOCKET_TIMEOUT, DEFAULT_SOCKET_TIMEOUT));
  clientConf.setRequestTimeout(Utils.getInt(conf, FS_COS, FS_ALT_KEYS,
      REQUEST_TIMEOUT, DEFAULT_REQUEST_TIMEOUT));
  int sockSendBuffer = Utils.getInt(conf, FS_COS, FS_ALT_KEYS,
      SOCKET_SEND_BUFFER, DEFAULT_SOCKET_SEND_BUFFER);
  int sockRecvBuffer = Utils.getInt(conf, FS_COS, FS_ALT_KEYS,
      SOCKET_RECV_BUFFER, DEFAULT_SOCKET_RECV_BUFFER);
  clientConf.setSocketBufferSizeHints(sockSendBuffer, sockRecvBuffer);
  String signerOverride = Utils.getTrimmed(conf, FS_COS, FS_ALT_KEYS,
      SIGNING_ALGORITHM, "");
  if (!signerOverride.isEmpty()) {
    LOG.debug("Signer override = {}", signerOverride);
    clientConf.setSignerOverride(signerOverride);
  }

  String userAgentPrefix = Utils.getTrimmed(conf, FS_COS, FS_ALT_KEYS,
      USER_AGENT_PREFIX, DEFAULT_USER_AGENT_PREFIX);
  String userAgentName = singletoneInitTimeData.getUserAgentName();
  if (!userAgentPrefix.equals(DEFAULT_USER_AGENT_PREFIX)) {
    userAgentName = userAgentPrefix + " " + userAgentName;
  }
  clientConf.setUserAgentPrefix(userAgentName);
}
 

public static ClientConfiguration getClientConfiguration(ProxyConfig config) throws StageException {
  ClientConfiguration clientConfig = new ClientConfiguration();

  clientConfig.setConnectionTimeout(config.connectionTimeout * MILLIS);
  clientConfig.setSocketTimeout(config.socketTimeout * MILLIS);
  clientConfig.withMaxErrorRetry(config.retryCount);

  // Optional proxy settings
  if (config.useProxy) {
    if (config.proxyHost != null && !config.proxyHost.isEmpty()) {
      clientConfig.setProxyHost(config.proxyHost);
      clientConfig.setProxyPort(config.proxyPort);

      if (config.proxyUser != null && !config.proxyUser.get().isEmpty()) {
        clientConfig.setProxyUsername(config.proxyUser.get());
      }

      if (config.proxyPassword != null && !config.proxyPassword.get().isEmpty()) {
        clientConfig.setProxyPassword(config.proxyPassword.get());
      }

      if (config.proxyDomain != null && !config.proxyDomain.isEmpty()) {
        clientConfig.setProxyDomain(config.proxyDomain);
      }

      if (config.proxyWorkstation != null && !config.proxyWorkstation.isEmpty()) {
        clientConfig.setProxyWorkstation(config.proxyWorkstation);
      }
    }
  }
  return clientConfig;
}
 

public static AWSServiceCatalog getServiceCatalogClientBuilder(
        String accessKeyId,
        String secretAccessKey,
        String proxyHost,
        Integer proxyPort,
        String proxyUsername,
        String proxyPassword,
        Integer connectTimeoutMs,
        Integer executionTimeoutMs,
        String region,
        boolean async) {

    ClientConfiguration clientConfiguration = new ClientConfiguration();
    clientConfiguration.setConnectionTimeout(connectTimeoutMs);
    clientConfiguration.setClientExecutionTimeout(executionTimeoutMs);

    if (!StringUtils.isEmpty(proxyHost)) {
        clientConfiguration.setProxyHost(proxyHost);
        clientConfiguration.setProxyPort(proxyPort);
        if (!StringUtils.isEmpty(proxyUsername)) {
            clientConfiguration.setProxyUsername(proxyUsername);
            clientConfiguration.setProxyPassword(proxyPassword);
        }
    }
    if (!async) {
        return AWSServiceCatalogClientBuilder.standard()
                .withRegion(region)
                .withClientConfiguration(clientConfiguration)
                .withCredentials(new AWSStaticCredentialsProvider(new BasicAWSCredentials(accessKeyId, secretAccessKey)))
                .build();
    }
    return AWSServiceCatalogAsyncClientBuilder.standard()
            .withRegion(region)
            .withClientConfiguration(clientConfiguration)
            .withCredentials(new AWSStaticCredentialsProvider(new BasicAWSCredentials(accessKeyId, secretAccessKey)))
            .build();


}
 

private void buildS3Client() {
    // override default timeout for S3Client
    ClientConfiguration clientConfiguration = new ClientConfiguration();
    clientConfiguration.setConnectionTimeout(SDK_TIME_OUT);
    clientConfiguration.setSocketTimeout(SDK_TIME_OUT);

    if (s3Client == null) {
        s3Client = AmazonS3ClientBuilder.standard()
                .withCredentials(config.getAwsCredentialsProvider())
                .withClientConfiguration(clientConfiguration)
                .withRegion(config.getS3Region())
                .build();
    }
}
 

@Override
public ClientConfiguration deserialize(JsonParser jsonParser, DeserializationContext context)
    throws IOException {
  Map<String, Object> map = jsonParser.readValueAs(new TypeReference<Map<String, Object>>() {});

  ClientConfiguration clientConfiguration = new ClientConfiguration();

  if (map.containsKey(PROXY_HOST)) {
    clientConfiguration.setProxyHost((String) map.get(PROXY_HOST));
  }
  if (map.containsKey(PROXY_PORT)) {
    clientConfiguration.setProxyPort((Integer) map.get(PROXY_PORT));
  }
  if (map.containsKey(PROXY_USERNAME)) {
    clientConfiguration.setProxyUsername((String) map.get(PROXY_USERNAME));
  }
  if (map.containsKey(PROXY_PASSWORD)) {
    clientConfiguration.setProxyPassword((String) map.get(PROXY_PASSWORD));
  }
  if (map.containsKey(CLIENT_EXECUTION_TIMEOUT)) {
    clientConfiguration.setClientExecutionTimeout((Integer) map.get(CLIENT_EXECUTION_TIMEOUT));
  }
  if (map.containsKey(CONNECTION_MAX_IDLE_TIME)) {
    clientConfiguration.setConnectionMaxIdleMillis(
        ((Number) map.get(CONNECTION_MAX_IDLE_TIME)).longValue());
  }
  if (map.containsKey(CONNECTION_TIMEOUT)) {
    clientConfiguration.setConnectionTimeout((Integer) map.get(CONNECTION_TIMEOUT));
  }
  if (map.containsKey(CONNECTION_TIME_TO_LIVE)) {
    clientConfiguration.setConnectionTTL(
        ((Number) map.get(CONNECTION_TIME_TO_LIVE)).longValue());
  }
  if (map.containsKey(MAX_CONNECTIONS)) {
    clientConfiguration.setMaxConnections((Integer) map.get(MAX_CONNECTIONS));
  }
  if (map.containsKey(REQUEST_TIMEOUT)) {
    clientConfiguration.setRequestTimeout((Integer) map.get(REQUEST_TIMEOUT));
  }
  if (map.containsKey(SOCKET_TIMEOUT)) {
    clientConfiguration.setSocketTimeout((Integer) map.get(SOCKET_TIMEOUT));
  }
  return clientConfiguration;
}
 

protected ClientConfiguration createConfiguration(final ProcessContext context) {
    final ClientConfiguration config = new ClientConfiguration();
    config.setMaxConnections(context.getMaxConcurrentTasks());
    config.setMaxErrorRetry(0);
    config.setUserAgent(DEFAULT_USER_AGENT);
    // If this is changed to be a property, ensure other uses are also changed
    config.setProtocol(DEFAULT_PROTOCOL);
    final int commsTimeout = context.getProperty(TIMEOUT).asTimePeriod(TimeUnit.MILLISECONDS).intValue();
    config.setConnectionTimeout(commsTimeout);
    config.setSocketTimeout(commsTimeout);

    if(this.getSupportedPropertyDescriptors().contains(SSL_CONTEXT_SERVICE)) {
        final SSLContextService sslContextService = context.getProperty(SSL_CONTEXT_SERVICE).asControllerService(SSLContextService.class);
        if (sslContextService != null) {
            final SSLContext sslContext = sslContextService.createSSLContext(SslContextFactory.ClientAuth.NONE);
            // NIFI-3788: Changed hostnameVerifier from null to DHV (BrowserCompatibleHostnameVerifier is deprecated)
            SdkTLSSocketFactory sdkTLSSocketFactory = new SdkTLSSocketFactory(sslContext, new DefaultHostnameVerifier());
            config.getApacheHttpClientConfig().setSslSocketFactory(sdkTLSSocketFactory);
        }
    }

    final ProxyConfiguration proxyConfig = ProxyConfiguration.getConfiguration(context, () -> {
        if (context.getProperty(PROXY_HOST).isSet()) {
            final ProxyConfiguration componentProxyConfig = new ProxyConfiguration();
            String proxyHost = context.getProperty(PROXY_HOST).evaluateAttributeExpressions().getValue();
            Integer proxyPort = context.getProperty(PROXY_HOST_PORT).evaluateAttributeExpressions().asInteger();
            String proxyUsername = context.getProperty(PROXY_USERNAME).evaluateAttributeExpressions().getValue();
            String proxyPassword = context.getProperty(PROXY_PASSWORD).evaluateAttributeExpressions().getValue();
            componentProxyConfig.setProxyType(Proxy.Type.HTTP);
            componentProxyConfig.setProxyServerHost(proxyHost);
            componentProxyConfig.setProxyServerPort(proxyPort);
            componentProxyConfig.setProxyUserName(proxyUsername);
            componentProxyConfig.setProxyUserPassword(proxyPassword);
            return componentProxyConfig;
        }
        return ProxyConfiguration.DIRECT_CONFIGURATION;
    });

    if (Proxy.Type.HTTP.equals(proxyConfig.getProxyType())) {
        config.setProxyHost(proxyConfig.getProxyServerHost());
        config.setProxyPort(proxyConfig.getProxyServerPort());

        if (proxyConfig.hasCredential()) {
            config.setProxyUsername(proxyConfig.getProxyUserName());
            config.setProxyPassword(proxyConfig.getProxyUserPassword());
        }
    }

    return config;
}
 

public static TransferManager getTransferManager(final ClientOptions clientOptions) {

        if(TRANSFERMANAGER_ACCESSKEY_MAP.containsKey(clientOptions.getAccessKey())) {
            return TRANSFERMANAGER_ACCESSKEY_MAP.get(clientOptions.getAccessKey());
        }

        final AWSCredentials basicAWSCredentials = new BasicAWSCredentials(clientOptions.getAccessKey(), clientOptions.getSecretKey());

        final ClientConfiguration configuration = new ClientConfiguration();

        if (clientOptions.isHttps() != null) {
            configuration.setProtocol(clientOptions.isHttps() ? HTTPS : HTTP);
        }

        if (clientOptions.getConnectionTimeout() != null) {
            configuration.setConnectionTimeout(clientOptions.getConnectionTimeout());
        }

        if (clientOptions.getMaxErrorRetry() != null) {
            configuration.setMaxErrorRetry(clientOptions.getMaxErrorRetry());
        }

        if (clientOptions.getSocketTimeout() != null) {
            configuration.setSocketTimeout(clientOptions.getSocketTimeout());
        }

        if (clientOptions.getUseTCPKeepAlive() != null) {
            configuration.setUseTcpKeepAlive(clientOptions.getUseTCPKeepAlive());
        }

        if (clientOptions.getConnectionTtl() != null) {
            configuration.setConnectionTTL(clientOptions.getConnectionTtl());
        }

        if (clientOptions.getSigner() != null) {

            configuration.setSignerOverride(clientOptions.getSigner());
        }

        LOGGER.debug(format("Creating S3 client with configuration: [protocol: %1$s, signer: %2$s, connectionTimeOut: %3$s, maxErrorRetry: %4$s, socketTimeout: %5$s, useTCPKeepAlive: %6$s, connectionTtl: %7$s]",
                configuration.getProtocol(), configuration.getSignerOverride(), configuration.getConnectionTimeout(), configuration.getMaxErrorRetry(), configuration.getSocketTimeout(),
                clientOptions.getUseTCPKeepAlive(), clientOptions.getConnectionTtl()));

        final AmazonS3Client client = new AmazonS3Client(basicAWSCredentials, configuration);

        if (isNotBlank(clientOptions.getEndPoint())) {
            LOGGER.debug(format("Setting the end point for S3 client with access key %1$s to %2$s.", clientOptions.getAccessKey(), clientOptions.getEndPoint()));

            client.setEndpoint(clientOptions.getEndPoint());
        }

        TRANSFERMANAGER_ACCESSKEY_MAP.put(clientOptions.getAccessKey(), new TransferManager(client));

        return TRANSFERMANAGER_ACCESSKEY_MAP.get(clientOptions.getAccessKey());
    }