Java Code Examples for com.amazonaws.ClientConfiguration#setSignerOverride()

private void initAmazonS3() {
    BasicAWSCredentials credentials = new BasicAWSCredentials(
            fileClientConfig.getAccessKey(), fileClientConfig.getSecretKey());
    ClientConfiguration clientConfig = new ClientConfiguration();
    clientConfig.setSignerOverride("S3SignerType");
    String region = fileClientConfig.getRegion() == null ? FileClientConfiguration.US_EAST_1 : fileClientConfig.getRegion();
    this.amazonS3 = AmazonS3ClientBuilder.standard()
            .withCredentials(new AWSStaticCredentialsProvider(credentials))
            .withClientConfiguration(clientConfig)
            .withEndpointConfiguration(
                    new AwsClientBuilder.EndpointConfiguration(
                            fileClientConfig.getEndpoint(),
                            region))
            .withPathStyleAccessEnabled(fileClientConfig.getWithPath())
            .build();

}
 

private ClientConfiguration buildClientConfig() {
  final String userAgent = System.getProperty(PROP_S3_HANDLER_USER_AGENT, null);
  final String protocol = System.getProperty(PROP_S3_HANDLER_PROTOCOL, "https");
  final String signerOverride = System.getProperty(PROP_S3_HANDLER_SIGNER_OVERRIDE, null);

  final ClientConfiguration clientConfig =
      new ClientConfiguration().withProtocol(
          "https".equalsIgnoreCase(protocol) ? Protocol.HTTPS : Protocol.HTTP);

  if (userAgent != null) {
    clientConfig.setUserAgent(userAgent);
  }
  if (signerOverride != null) {
    clientConfig.setSignerOverride(signerOverride);
  }

  return clientConfig;
}
 

public RealS3Client(String endpoint, String accessKey, String secretKey) {
    ClientConfiguration clientConfiguration = new ClientConfiguration();
    clientConfiguration.setSignerOverride("AWSS3V4SignerType");
    AmazonS3 newClient = AmazonS3ClientBuilder.standard()
            .withEndpointConfiguration(new AwsClientBuilder.EndpointConfiguration(endpoint, Regions.US_EAST_1.name()))
            .withPathStyleAccessEnabled(true)
            .withClientConfiguration(clientConfiguration)
            .withCredentials(new AWSStaticCredentialsProvider(new BasicAWSCredentials(accessKey, secretKey)))
            .build();
    this.s3Client = newClient;
    this.endpoint = endpoint;
}
 

private void initializeSignerOverride(final ProcessContext context, final ClientConfiguration config) {
    String signer = context.getProperty(SIGNER_OVERRIDE).getValue();

    if (signer != null && !signer.equals(SIGNER_OVERRIDE.getDefaultValue())) {
        config.setSignerOverride(signer);
    }
}
 

/**
 * Initializes connection management
 *
 * @param conf Hadoop configuration
 * @param clientConf client SDK configuration
 */
private void initConnectionSettings(Configuration conf,
    ClientConfiguration clientConf) throws IOException {
  clientConf.setMaxConnections(Utils.getInt(conf, FS_COS, FS_ALT_KEYS,
      MAXIMUM_CONNECTIONS, DEFAULT_MAXIMUM_CONNECTIONS));
  clientConf.setClientExecutionTimeout(Utils.getInt(conf, FS_COS, FS_ALT_KEYS,
      CLIENT_EXEC_TIMEOUT, DEFAULT_CLIENT_EXEC_TIMEOUT));
  clientConf.setMaxErrorRetry(Utils.getInt(conf, FS_COS, FS_ALT_KEYS,
      MAX_ERROR_RETRIES, DEFAULT_MAX_ERROR_RETRIES));
  clientConf.setConnectionTimeout(Utils.getInt(conf, FS_COS, FS_ALT_KEYS,
      ESTABLISH_TIMEOUT, DEFAULT_ESTABLISH_TIMEOUT));
  clientConf.setSocketTimeout(Utils.getInt(conf, FS_COS, FS_ALT_KEYS,
      SOCKET_TIMEOUT, DEFAULT_SOCKET_TIMEOUT));
  clientConf.setRequestTimeout(Utils.getInt(conf, FS_COS, FS_ALT_KEYS,
      REQUEST_TIMEOUT, DEFAULT_REQUEST_TIMEOUT));
  int sockSendBuffer = Utils.getInt(conf, FS_COS, FS_ALT_KEYS,
      SOCKET_SEND_BUFFER, DEFAULT_SOCKET_SEND_BUFFER);
  int sockRecvBuffer = Utils.getInt(conf, FS_COS, FS_ALT_KEYS,
      SOCKET_RECV_BUFFER, DEFAULT_SOCKET_RECV_BUFFER);
  clientConf.setSocketBufferSizeHints(sockSendBuffer, sockRecvBuffer);
  String signerOverride = Utils.getTrimmed(conf, FS_COS, FS_ALT_KEYS,
      SIGNING_ALGORITHM, "");
  if (!signerOverride.isEmpty()) {
    LOG.debug("Signer override = {}", signerOverride);
    clientConf.setSignerOverride(signerOverride);
  }

  String userAgentPrefix = Utils.getTrimmed(conf, FS_COS, FS_ALT_KEYS,
      USER_AGENT_PREFIX, DEFAULT_USER_AGENT_PREFIX);
  String userAgentName = singletoneInitTimeData.getUserAgentName();
  if (!userAgentPrefix.equals(DEFAULT_USER_AGENT_PREFIX)) {
    userAgentName = userAgentPrefix + " " + userAgentName;
  }
  clientConf.setUserAgentPrefix(userAgentName);
}
 

/**
 * Create AWS client configuration and return it.
 * @return AWS client configuration
 */
private ClientConfiguration createClientConfiguration() {
  ClientConfigurationFactory configFactory = new ClientConfigurationFactory();
  ClientConfiguration config = configFactory.getConfig();

  String s3SignerOverride = conf.getS3SignerOverride();
  if (StringUtils.isNotBlank(s3SignerOverride)) {
    config.setSignerOverride(s3SignerOverride);
  }

  return config;
}
 

/**
 * Create AWS client configuration and return it.
 * @return AWS client configuration
 */
private ClientConfiguration createClientConfiguration() {
  ClientConfigurationFactory configFactory = new ClientConfigurationFactory();
  ClientConfiguration config = configFactory.getConfig();

  String s3SignerOverride = conf.getS3SignerOverride();
  if (StringUtils.isNotBlank(s3SignerOverride)) {
    config.setSignerOverride(s3SignerOverride);
  }

  return config;
}
 

private void initializeSignerOverride(final ProcessContext context, final ClientConfiguration config) {
    String signer = context.getProperty(SIGNER_OVERRIDE).getValue();

    if (signer != null && !signer.equals(SIGNER_OVERRIDE.getDefaultValue())) {
        config.setSignerOverride(signer);
    }
}
 

private AmazonS3 createAmazonS3Client(Configuration hadoopConfig, ClientConfiguration clientConfig)
{
    Optional<EncryptionMaterialsProvider> encryptionMaterialsProvider = createEncryptionMaterialsProvider(hadoopConfig);
    AmazonS3Builder<? extends AmazonS3Builder<?, ?>, ? extends AmazonS3> clientBuilder;

    String signerType = hadoopConfig.get(S3_SIGNER_TYPE);
    if (signerType != null) {
        clientConfig.withSignerOverride(signerType);
    }

    String signerClass = hadoopConfig.get(S3_SIGNER_CLASS);
    if (signerClass != null) {
        Class<? extends Signer> klass;
        try {
            klass = Class.forName(signerClass).asSubclass(Signer.class);
        }
        catch (ClassNotFoundException e) {
            throw new RuntimeException("Signer class not found: " + signerClass, e);
        }
        SignerFactory.registerSigner(S3_CUSTOM_SIGNER, klass);
        clientConfig.setSignerOverride(S3_CUSTOM_SIGNER);
    }

    if (encryptionMaterialsProvider.isPresent()) {
        clientBuilder = AmazonS3EncryptionClient.encryptionBuilder()
                .withCredentials(credentialsProvider)
                .withEncryptionMaterials(encryptionMaterialsProvider.get())
                .withClientConfiguration(clientConfig)
                .withMetricsCollector(METRIC_COLLECTOR);
    }
    else {
        clientBuilder = AmazonS3Client.builder()
                .withCredentials(credentialsProvider)
                .withClientConfiguration(clientConfig)
                .withMetricsCollector(METRIC_COLLECTOR);
    }

    boolean regionOrEndpointSet = false;

    // use local region when running inside of EC2
    if (pinS3ClientToCurrentRegion) {
        clientBuilder.setRegion(getCurrentRegionFromEC2Metadata().getName());
        regionOrEndpointSet = true;
    }

    String endpoint = hadoopConfig.get(S3_ENDPOINT);
    if (endpoint != null) {
        clientBuilder.setEndpointConfiguration(new EndpointConfiguration(endpoint, null));
        regionOrEndpointSet = true;
    }

    if (isPathStyleAccess) {
        clientBuilder.enablePathStyleAccess();
    }

    if (!regionOrEndpointSet) {
        clientBuilder.withRegion(US_EAST_1);
        clientBuilder.setForceGlobalBucketAccessEnabled(true);
    }

    return clientBuilder.build();
}
 

public AmazonS3 create(final BlobStoreConfiguration blobStoreConfiguration) {
  NexusS3ClientBuilder builder = NexusS3ClientBuilder.standard();

  String accessKeyId = blobStoreConfiguration.attributes(CONFIG_KEY).get(ACCESS_KEY_ID_KEY, String.class);
  String secretAccessKey = blobStoreConfiguration.attributes(CONFIG_KEY).get(SECRET_ACCESS_KEY_KEY, String.class);
  String region = blobStoreConfiguration.attributes(CONFIG_KEY).get(REGION_KEY, String.class);
  String signerType = blobStoreConfiguration.attributes(CONFIG_KEY).get(SIGNERTYPE_KEY, String.class);
  String forcePathStyle = blobStoreConfiguration.attributes(CONFIG_KEY).get(FORCE_PATH_STYLE_KEY, String.class);

  if (!isNullOrEmpty(accessKeyId) && !isNullOrEmpty(secretAccessKey)) {
    String sessionToken = blobStoreConfiguration.attributes(CONFIG_KEY).get(SESSION_TOKEN_KEY, String.class);
    AWSCredentials credentials = buildCredentials(accessKeyId, secretAccessKey, sessionToken);

    String assumeRole = blobStoreConfiguration.attributes(CONFIG_KEY).get(ASSUME_ROLE_KEY, String.class);
    AWSCredentialsProvider credentialsProvider = buildCredentialsProvider(credentials, region, assumeRole);

    builder = builder.withCredentials(credentialsProvider);
  }

  if (!isNullOrEmptyOrDefault(region)) {
    String endpoint = blobStoreConfiguration.attributes(CONFIG_KEY).get(ENDPOINT_KEY, String.class);
    if (!isNullOrEmpty(endpoint)) {
      builder = builder.withEndpointConfiguration(new AmazonS3ClientBuilder.EndpointConfiguration(endpoint, region));
    } else {
      builder = builder.withRegion(region);
    }
  }

  ClientConfiguration clientConfiguration = PredefinedClientConfigurations.defaultConfig();
  if (connectionPoolSize > 0) {
    clientConfiguration.setMaxConnections(connectionPoolSize);
  }
  if (!isNullOrEmptyOrDefault(signerType)) {
    clientConfiguration.setSignerOverride(signerType);
  }
  builder = builder.withClientConfiguration(clientConfiguration);

  builder = builder.withPathStyleAccessEnabled(Boolean.parseBoolean(forcePathStyle));

  builder.withBlobStoreConfig(blobStoreConfiguration);

  return builder.build();
}
 

@Override
public void configure(SyncStorage source, Iterator<SyncFilter> filters, SyncStorage target) {
    super.configure(source, filters, target);

    Assert.hasText(config.getAccessKey(), "accessKey is required");
    Assert.hasText(config.getSecretKey(), "secretKey is required");
    Assert.hasText(config.getBucketName(), "bucketName is required");
    Assert.isTrue(config.getBucketName().matches("[A-Za-z0-9._-]+"), config.getBucketName() + " is not a valid bucket name");

    AWSCredentials creds = new BasicAWSCredentials(config.getAccessKey(), config.getSecretKey());
    ClientConfiguration cc = new ClientConfiguration();

    if (config.getProtocol() != null)
        cc.setProtocol(Protocol.valueOf(config.getProtocol().toString().toUpperCase()));

    if (config.isLegacySignatures()) cc.setSignerOverride("S3SignerType");

    if (config.getSocketTimeoutMs() >= 0) cc.setSocketTimeout(config.getSocketTimeoutMs());

    AmazonS3ClientBuilder builder = AmazonS3ClientBuilder.standard()
            .withCredentials(new AWSStaticCredentialsProvider(creds))
            .withClientConfiguration(cc);

    if (config.getHost() != null) {
        String endpoint = "";
        if (config.getProtocol() != null) endpoint += config.getProtocol() + "://";
        endpoint += config.getHost();
        if (config.getPort() > 0) endpoint += ":" + config.getPort();
        builder.setEndpointConfiguration(new AwsClientBuilder.EndpointConfiguration(endpoint, config.getRegion()));
    } else if (config.getRegion() != null) {
        builder.withRegion(config.getRegion());
    }

    if (config.isDisableVHosts()) {
        log.info("The use of virtual hosted buckets has been DISABLED.  Path style buckets will be used.");
        builder.withPathStyleAccessEnabled(true);
    }

    s3 = builder.build();

    boolean bucketExists = s3.doesBucketExistV2(config.getBucketName());

    boolean bucketHasVersions = false;
    if (bucketExists && config.isIncludeVersions()) {
        // check if versioning has ever been enabled on the bucket (versions will not be collected unless required)
        BucketVersioningConfiguration versioningConfig = s3.getBucketVersioningConfiguration(config.getBucketName());
        List<String> versionedStates = Arrays.asList(BucketVersioningConfiguration.ENABLED, BucketVersioningConfiguration.SUSPENDED);
        bucketHasVersions = versionedStates.contains(versioningConfig.getStatus());
    }

    if (config.getKeyPrefix() == null) config.setKeyPrefix(""); // make sure keyPrefix isn't null

    if (source == this) {
        if (config.getExcludedKeys() != null) {
            excludedKeyPatterns = new ArrayList<>();
            for (String pattern : config.getExcludedKeys()) {
                excludedKeyPatterns.add(Pattern.compile(pattern));
            }
        }
    }

    if (target == this) {
        // create bucket if it doesn't exist
        if (!bucketExists && config.isCreateBucket()) {
            s3.createBucket(config.getBucketName());
            bucketExists = true;
            if (config.isIncludeVersions()) {
                s3.setBucketVersioningConfiguration(new SetBucketVersioningConfigurationRequest(config.getBucketName(),
                        new BucketVersioningConfiguration(BucketVersioningConfiguration.ENABLED)));
                bucketHasVersions = true;
            }
        }

        // make sure MPU settings are valid
        if (config.getMpuThresholdMb() > MAX_PUT_SIZE_MB) {
            log.warn("{}MB is above the maximum PUT size of {}MB. the maximum will be used instead",
                    config.getMpuThresholdMb(), MAX_PUT_SIZE_MB);
            config.setMpuThresholdMb(MAX_PUT_SIZE_MB);
        }
        if (config.getMpuPartSizeMb() < MIN_PART_SIZE_MB) {
            log.warn("{}MB is below the minimum MPU part size of {}MB. the minimum will be used instead",
                    config.getMpuPartSizeMb(), MIN_PART_SIZE_MB);
            config.setMpuPartSizeMb(MIN_PART_SIZE_MB);
        }

        if (source != null) sourceReadWindow = source.getReadWindow();
    }

    // make sure bucket exists
    if (!bucketExists)
        throw new ConfigurationException("The bucket " + config.getBucketName() + " does not exist.");

    // if syncing versions, make sure plugins support it and bucket has versioning enabled
    if (config.isIncludeVersions()) {
        if (!(source instanceof AbstractS3Storage && target instanceof AbstractS3Storage))
            throw new ConfigurationException("Version migration is only supported between two S3 plugins");

        if (!bucketHasVersions)
            throw new ConfigurationException("The specified bucket does not have versioning enabled.");
    }
}
 

public static TransferManager getTransferManager(final ClientOptions clientOptions) {

        if(TRANSFERMANAGER_ACCESSKEY_MAP.containsKey(clientOptions.getAccessKey())) {
            return TRANSFERMANAGER_ACCESSKEY_MAP.get(clientOptions.getAccessKey());
        }

        final AWSCredentials basicAWSCredentials = new BasicAWSCredentials(clientOptions.getAccessKey(), clientOptions.getSecretKey());

        final ClientConfiguration configuration = new ClientConfiguration();

        if (clientOptions.isHttps() != null) {
            configuration.setProtocol(clientOptions.isHttps() ? HTTPS : HTTP);
        }

        if (clientOptions.getConnectionTimeout() != null) {
            configuration.setConnectionTimeout(clientOptions.getConnectionTimeout());
        }

        if (clientOptions.getMaxErrorRetry() != null) {
            configuration.setMaxErrorRetry(clientOptions.getMaxErrorRetry());
        }

        if (clientOptions.getSocketTimeout() != null) {
            configuration.setSocketTimeout(clientOptions.getSocketTimeout());
        }

        if (clientOptions.getUseTCPKeepAlive() != null) {
            configuration.setUseTcpKeepAlive(clientOptions.getUseTCPKeepAlive());
        }

        if (clientOptions.getConnectionTtl() != null) {
            configuration.setConnectionTTL(clientOptions.getConnectionTtl());
        }

        if (clientOptions.getSigner() != null) {

            configuration.setSignerOverride(clientOptions.getSigner());
        }

        LOGGER.debug(format("Creating S3 client with configuration: [protocol: %1$s, signer: %2$s, connectionTimeOut: %3$s, maxErrorRetry: %4$s, socketTimeout: %5$s, useTCPKeepAlive: %6$s, connectionTtl: %7$s]",
                configuration.getProtocol(), configuration.getSignerOverride(), configuration.getConnectionTimeout(), configuration.getMaxErrorRetry(), configuration.getSocketTimeout(),
                clientOptions.getUseTCPKeepAlive(), clientOptions.getConnectionTtl()));

        final AmazonS3Client client = new AmazonS3Client(basicAWSCredentials, configuration);

        if (isNotBlank(clientOptions.getEndPoint())) {
            LOGGER.debug(format("Setting the end point for S3 client with access key %1$s to %2$s.", clientOptions.getAccessKey(), clientOptions.getEndPoint()));

            client.setEndpoint(clientOptions.getEndPoint());
        }

        TRANSFERMANAGER_ACCESSKEY_MAP.put(clientOptions.getAccessKey(), new TransferManager(client));

        return TRANSFERMANAGER_ACCESSKEY_MAP.get(clientOptions.getAccessKey());
    }