Java Code Examples for com.amazonaws.services.s3.AmazonS3ClientBuilder#withRegion()
The following examples show how to use
com.amazonaws.services.s3.AmazonS3ClientBuilder#withRegion() .
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
| Source File: JceksAmazonS3ClientFactory.java From circus-train with Apache License 2.0 | 6 votes |
|
private AmazonS3 newInstance(
String region,
S3S3CopierOptions s3s3CopierOptions,
HadoopAWSCredentialProviderChain credentialsChain) {
AmazonS3ClientBuilder builder = AmazonS3ClientBuilder
.standard()
.withCredentials(credentialsChain);
applyClientConfigurations(builder, s3s3CopierOptions);
URI s3Endpoint = s3s3CopierOptions.getS3Endpoint(region);
if (s3Endpoint != null) {
EndpointConfiguration endpointConfiguration = new EndpointConfiguration(s3Endpoint.toString(), region);
builder.withEndpointConfiguration(endpointConfiguration);
} else {
builder.withRegion(region);
}
return builder.build();
}
Example 2
| Source File: AmazonS3Provider.java From emodb with Apache License 2.0 | 6 votes |
|
private static AmazonS3 getAmazonS3(final S3BucketConfiguration s3BucketConfiguration, Clock clock) {
AmazonS3ClientBuilder amazonS3ClientBuilder = AmazonS3ClientBuilder.standard()
.withCredentials(getAwsCredentialsProvider(s3BucketConfiguration))
.withAccelerateModeEnabled(s3BucketConfiguration.getAccelerateModeEnabled());
S3ClientConfiguration.RateLimitConfiguration rateLimitConfiguration = new S3ClientConfiguration.RateLimitConfiguration();
if (null != s3BucketConfiguration.getRegion()) {
amazonS3ClientBuilder
.withRegion(Regions.fromName(s3BucketConfiguration.getRegion()));
} else if (null != s3BucketConfiguration.getS3ClientConfiguration()) {
S3ClientConfiguration.EndpointConfiguration endpointConfiguration = s3BucketConfiguration.getS3ClientConfiguration().getEndpointConfiguration();
amazonS3ClientBuilder
.withEndpointConfiguration(new AwsClientBuilder.EndpointConfiguration(endpointConfiguration.getServiceEndpoint(), endpointConfiguration.getSigningRegion()));
rateLimitConfiguration = s3BucketConfiguration.getS3ClientConfiguration().getRateLimitConfiguration();
}
AmazonS3 amazonS3 = amazonS3ClientBuilder
.build();
return new S3RateLimiter(clock, rateLimitConfiguration)
.rateLimit(amazonS3);
}
Example 3
| Source File: S3.java From rdf-delta with Apache License 2.0 | 6 votes |
|
public static AmazonS3 buildS3(LocalServerConfig configuration) {
String region = configuration.getProperty(pRegion);
String endpoint = configuration.getProperty(pEndpoint);
String credentialsFile = configuration.getProperty(pCredentialFile);
String credentialsProfile = configuration.getProperty(pCredentialProfile);
AmazonS3ClientBuilder builder = AmazonS3ClientBuilder.standard();
if ( endpoint == null )
builder.withRegion(region);
else {
// Needed for S3mock
builder.withPathStyleAccessEnabled(true);
builder.withEndpointConfiguration(new EndpointConfiguration(endpoint, region));
builder.withCredentials(new AWSStaticCredentialsProvider(new AnonymousAWSCredentials()));
}
if ( credentialsFile != null )
builder.withCredentials(new ProfileCredentialsProvider(credentialsFile, credentialsProfile));
return builder.build();
}
Example 4
| Source File: DefaultS3ClientBuilderFactory.java From beam with Apache License 2.0 | 6 votes |
|
@Override
public AmazonS3ClientBuilder createBuilder(S3Options s3Options) {
AmazonS3ClientBuilder builder =
AmazonS3ClientBuilder.standard().withCredentials(s3Options.getAwsCredentialsProvider());
if (s3Options.getClientConfiguration() != null) {
builder = builder.withClientConfiguration(s3Options.getClientConfiguration());
}
if (!Strings.isNullOrEmpty(s3Options.getAwsServiceEndpoint())) {
builder =
builder.withEndpointConfiguration(
new AwsClientBuilder.EndpointConfiguration(
s3Options.getAwsServiceEndpoint(), s3Options.getAwsRegion()));
} else if (!Strings.isNullOrEmpty(s3Options.getAwsRegion())) {
builder = builder.withRegion(s3Options.getAwsRegion());
} else {
LOG.info(
"The AWS S3 Beam extension was included in this build, but the awsRegion flag "
+ "was not specified. If you don't plan to use S3, then ignore this message.");
}
return builder;
}
Example 5
| Source File: S3KeyGenerator.java From hadoop-ozone with Apache License 2.0 | 5 votes |
|
@Override
public Void call() throws Exception {
if (multiPart && fileSize < OM_MULTIPART_MIN_SIZE) {
throw new IllegalArgumentException(
"Size of multipart upload parts should be at least 5MB (5242880)");
}
init();
AmazonS3ClientBuilder amazonS3ClientBuilder =
AmazonS3ClientBuilder.standard()
.withCredentials(new EnvironmentVariableCredentialsProvider());
if (endpoint.length() > 0) {
amazonS3ClientBuilder
.withPathStyleAccessEnabled(true)
.withEndpointConfiguration(
new EndpointConfiguration(endpoint, "us-east-1"));
} else {
amazonS3ClientBuilder.withRegion(Regions.DEFAULT_REGION);
}
s3 = amazonS3ClientBuilder.build();
content = RandomStringUtils.randomAscii(fileSize);
timer = getMetrics().timer("key-create");
System.setProperty(DISABLE_PUT_OBJECT_MD5_VALIDATION_PROPERTY, "true");
runTests(this::createKey);
return null;
}
Example 6
| Source File: AwsS3BuildCacheServiceFactory.java From gradle-s3-build-cache with Apache License 2.0 | 5 votes |
|
private AmazonS3 createS3Client(AwsS3BuildCache config) {
AmazonS3 s3;
try {
AmazonS3ClientBuilder s3Builder = AmazonS3ClientBuilder.standard();
if (!isNullOrEmpty(config.getAwsAccessKeyId()) && !isNullOrEmpty(config.getAwsSecretKey()) &&
!isNullOrEmpty(config.getSessionToken())) {
s3Builder.withCredentials(new AWSStaticCredentialsProvider(
new BasicSessionCredentials(config.getAwsAccessKeyId(), config.getAwsSecretKey(),
config.getSessionToken())));
} else if (!isNullOrEmpty(config.getAwsAccessKeyId()) && !isNullOrEmpty(config.getAwsSecretKey())) {
s3Builder.withCredentials(new AWSStaticCredentialsProvider(
new BasicAWSCredentials(config.getAwsAccessKeyId(), config.getAwsSecretKey())));
}
addHttpHeaders(s3Builder, config);
if (isNullOrEmpty(config.getEndpoint())) {
s3Builder.withRegion(config.getRegion());
} else {
s3Builder.withEndpointConfiguration(
new AwsClientBuilder.EndpointConfiguration(config.getEndpoint(), config.getRegion()));
}
s3 = s3Builder.build();
} catch (SdkClientException e) {
logger.debug("Error while building AWS S3 client: {}", e.getMessage());
throw new GradleException("Creation of S3 build cache failed; cannot create S3 client", e);
}
return s3;
}
Example 7
| Source File: AwsS3ClientFactory.java From circus-train with Apache License 2.0 | 5 votes |
|
public AmazonS3 newInstance(Configuration conf) {
int maxErrorRetry = conf.getInt(ConfigurationVariable.UPLOAD_RETRY_COUNT.getName(),
ConfigurationVariable.UPLOAD_RETRY_COUNT.defaultIntValue());
long errorRetryDelay = conf.getLong(ConfigurationVariable.UPLOAD_RETRY_DELAY_MS.getName(),
ConfigurationVariable.UPLOAD_RETRY_DELAY_MS.defaultLongValue());
LOG.info("Creating AWS S3 client with a retry policy of {} retries and {} ms of exponential backoff delay",
maxErrorRetry, errorRetryDelay);
RetryPolicy retryPolicy = new RetryPolicy(new CounterBasedRetryCondition(maxErrorRetry),
new ExponentialBackoffStrategy(errorRetryDelay), maxErrorRetry, true);
ClientConfiguration clientConfiguration = new ClientConfiguration();
clientConfiguration.setRetryPolicy(retryPolicy);
clientConfiguration.setMaxErrorRetry(maxErrorRetry);
AmazonS3ClientBuilder builder = AmazonS3ClientBuilder
.standard()
.withCredentials(new HadoopAWSCredentialProviderChain(conf))
.withClientConfiguration(clientConfiguration);
EndpointConfiguration endpointConfiguration = getEndpointConfiguration(conf);
if (endpointConfiguration != null) {
builder.withEndpointConfiguration(endpointConfiguration);
} else {
builder.withRegion(getRegion(conf));
}
return builder.build();
}
Example 8
| Source File: S3RepositoryAutoConfiguration.java From hawkbit-extensions with Eclipse Public License 1.0 | 5 votes |
|
/**
* @return the {@link AmazonS3Client} if no other {@link AmazonS3} bean is
* registered.
*/
@Bean
@ConditionalOnMissingBean
public AmazonS3 amazonS3() {
final AmazonS3ClientBuilder s3ClientBuilder = AmazonS3ClientBuilder.standard()
.withCredentials(awsCredentialsProvider()).withClientConfiguration(awsClientConfiguration());
if (!StringUtils.isEmpty(endpoint)) {
final String signingRegion = StringUtils.isEmpty(region) ? "" : region;
s3ClientBuilder.withEndpointConfiguration(new EndpointConfiguration(endpoint, signingRegion));
} else if (!StringUtils.isEmpty(region)) {
s3ClientBuilder.withRegion(region);
}
return s3ClientBuilder.build();
}
Example 9
| Source File: S3Accessor.java From datacollector with Apache License 2.0 | 5 votes |
|
AmazonS3Client createS3Client() throws StageException {
AmazonS3ClientBuilder builder = createAmazonS3ClientBuilder().withClientConfiguration(createClientConfiguration())
.withChunkedEncodingDisabled(connectionConfigs
.isChunkedEncodingEnabled())
.withPathStyleAccessEnabled(true);
AWSCredentialsProvider awsCredentialsProvider = getCredentialsProvider();
// If we don't call build.withCredentials(...) then we will not overwrite the default credentials provider
// already set in the builder when doing AmazonS3ClientBuilder.standard() so only calling build.withCredentials(...)
// if our own provider exists
if (awsCredentialsProvider != null) {
builder.withCredentials(awsCredentialsProvider);
}
String region = (connectionConfigs.getRegion() == null || connectionConfigs.getRegion().isEmpty())
? null
: connectionConfigs.getRegion();
if (connectionConfigs.isUseEndpoint()) {
builder.withEndpointConfiguration(new AwsClientBuilder.EndpointConfiguration(connectionConfigs.getEndpoint(),
region
));
} else if (region != null) {
builder.withRegion(connectionConfigs.getRegion());
} else {
builder.withRegion(AwsRegion.US_WEST_1.getId());
builder.withForceGlobalBucketAccessEnabled(true);
}
return (AmazonS3Client) builder.build();
}
Example 10
| Source File: S3ConnectionBaseConfig.java From datacollector with Apache License 2.0 | 5 votes |
|
private void createConnection(
Stage.Context context,
String configPrefix,
ProxyConfig proxyConfig,
List<Stage.ConfigIssue> issues,
int maxErrorRetries
) throws StageException {
AWSCredentialsProvider credentials = AWSUtil.getCredentialsProvider(awsConfig);
ClientConfiguration clientConfig = AWSUtil.getClientConfiguration(proxyConfig);
if (maxErrorRetries >= 0) {
clientConfig.setMaxErrorRetry(maxErrorRetries);
}
AmazonS3ClientBuilder builder = AmazonS3ClientBuilder
.standard()
.withCredentials(credentials)
.withClientConfiguration(clientConfig)
.withChunkedEncodingDisabled(awsConfig.disableChunkedEncoding)
.withPathStyleAccessEnabled(usePathAddressModel);
if (region == AwsRegion.OTHER) {
if (endpoint == null || endpoint.isEmpty()) {
issues.add(context.createConfigIssue(Groups.S3.name(), configPrefix + "endpoint", Errors.S3_SPOOLDIR_10));
return;
}
builder.withEndpointConfiguration(new AwsClientBuilder.EndpointConfiguration(endpoint, null));
} else {
builder.withRegion(region.getId());
}
s3Client = builder.build();
}
Example 11
| Source File: S3ArtifactStore.java From gocd-s3-artifacts with Apache License 2.0 | 5 votes |
|
public static AmazonS3 getS3client(GoEnvironment env) {
AmazonS3ClientBuilder amazonS3ClientBuilder = AmazonS3ClientBuilder.standard();
if (env.has(AWS_REGION)) {
amazonS3ClientBuilder.withRegion(env.get(AWS_REGION));
}
if (env.hasAWSUseIamRole()) {
amazonS3ClientBuilder.withCredentials(new InstanceProfileCredentialsProvider(false));
} else if (env.has(AWS_ACCESS_KEY_ID) && env.has(AWS_SECRET_ACCESS_KEY)) {
BasicAWSCredentials basicCreds = new BasicAWSCredentials(env.get(AWS_ACCESS_KEY_ID), env.get(AWS_SECRET_ACCESS_KEY));
amazonS3ClientBuilder.withCredentials(new AWSStaticCredentialsProvider(basicCreds));
}
return amazonS3ClientBuilder.build();
}
Example 12
| Source File: AwsS3EnvironmentRepositoryFactory.java From spring-cloud-config with Apache License 2.0 | 5 votes |
|
@Override
public AwsS3EnvironmentRepository build(
AwsS3EnvironmentProperties environmentProperties) {
final AmazonS3ClientBuilder clientBuilder = AmazonS3ClientBuilder.standard();
if (environmentProperties.getRegion() != null) {
clientBuilder.withRegion(environmentProperties.getRegion());
}
final AmazonS3 client = clientBuilder.build();
if (environmentProperties.getEndpoint() != null) {
client.setEndpoint(environmentProperties.getEndpoint());
}
AwsS3EnvironmentRepository repository = new AwsS3EnvironmentRepository(client,
environmentProperties.getBucket(), server);
return repository;
}
Example 13
| Source File: AmazonS3ClientFactory.java From spring-cloud-aws with Apache License 2.0 | 5 votes |
|
private AmazonS3ClientBuilder buildAmazonS3ForRegion(AmazonS3 prototype,
String region) {
AmazonS3ClientBuilder clientBuilder = AmazonS3ClientBuilder.standard();
AmazonS3Client target = getAmazonS3ClientFromProxy(prototype);
if (target != null) {
AWSCredentialsProvider awsCredentialsProvider = (AWSCredentialsProvider) ReflectionUtils
.getField(this.credentialsProviderField, target);
clientBuilder.withCredentials(awsCredentialsProvider);
}
return clientBuilder.withRegion(region);
}
Example 14
| Source File: S3ArtifactStore.java From gocd-s3-artifacts with Apache License 2.0 | 5 votes |
|
public static AmazonS3 getS3client(GoEnvironment env) {
AmazonS3ClientBuilder amazonS3ClientBuilder = AmazonS3ClientBuilder.standard();
if (env.has(AWS_REGION)) {
amazonS3ClientBuilder.withRegion(env.get(AWS_REGION));
}
if (env.hasAWSUseIamRole()) {
amazonS3ClientBuilder.withCredentials(new InstanceProfileCredentialsProvider(false));
} else if (env.has(AWS_ACCESS_KEY_ID) && env.has(AWS_SECRET_ACCESS_KEY)) {
BasicAWSCredentials basicCreds = new BasicAWSCredentials(env.get(AWS_ACCESS_KEY_ID), env.get(AWS_SECRET_ACCESS_KEY));
amazonS3ClientBuilder.withCredentials(new AWSStaticCredentialsProvider(basicCreds));
}
return amazonS3ClientBuilder.build();
}
Example 15
| Source File: S3Module.java From cassandra-backup with Apache License 2.0 | 4 votes |
|
private AmazonS3 provideAmazonS3(final Provider<CoreV1Api> coreV1ApiProvider, final AbstractOperationRequest operationRequest) {
final S3Configuration s3Conf = resolveS3Configuration(coreV1ApiProvider, operationRequest);
final AmazonS3ClientBuilder builder = AmazonS3ClientBuilder.standard();
if (s3Conf.awsEndpoint != null) {
// AWS_REGION must be set if AWS_ENDPOINT is set
if (s3Conf.awsRegion == null) {
throw new IllegalArgumentException("AWS_REGION must be set if AWS_ENDPOINT is set.");
}
builder.withEndpointConfiguration(new EndpointConfiguration(s3Conf.awsEndpoint, s3Conf.awsRegion.toLowerCase()));
} else if (s3Conf.awsRegion != null) {
builder.withRegion(Regions.fromName(s3Conf.awsRegion.toLowerCase()));
}
// if we are not running against Kubernetes, credentials should be fetched from ~/.aws/...
if (isRunningInKubernetes()) {
// it is possible that we have not set any secrets for s3 so the last
// resort is to fallback to AWS instance credentials.
if (s3Conf.awsAccessKeyId != null && s3Conf.awsSecretKey != null) {
builder.setCredentials(new AWSCredentialsProvider() {
@Override
public AWSCredentials getCredentials() {
return new AWSCredentials() {
@Override
public String getAWSAccessKeyId() {
return s3Conf.awsAccessKeyId;
}
@Override
public String getAWSSecretKey() {
return s3Conf.awsSecretKey;
}
};
}
@Override
public void refresh() {
}
});
}
}
return builder.build();
}
Example 16
| Source File: AwsS3Storage.java From ecs-sync with Apache License 2.0 | 4 votes |
|
@Override
public void configure(SyncStorage source, Iterator<SyncFilter> filters, SyncStorage target) {
super.configure(source, filters, target);
Assert.hasText(config.getAccessKey(), "accessKey is required");
Assert.hasText(config.getSecretKey(), "secretKey is required");
Assert.hasText(config.getBucketName(), "bucketName is required");
Assert.isTrue(config.getBucketName().matches("[A-Za-z0-9._-]+"), config.getBucketName() + " is not a valid bucket name");
AWSCredentials creds = new BasicAWSCredentials(config.getAccessKey(), config.getSecretKey());
ClientConfiguration cc = new ClientConfiguration();
if (config.getProtocol() != null)
cc.setProtocol(Protocol.valueOf(config.getProtocol().toString().toUpperCase()));
if (config.isLegacySignatures()) cc.setSignerOverride("S3SignerType");
if (config.getSocketTimeoutMs() >= 0) cc.setSocketTimeout(config.getSocketTimeoutMs());
AmazonS3ClientBuilder builder = AmazonS3ClientBuilder.standard()
.withCredentials(new AWSStaticCredentialsProvider(creds))
.withClientConfiguration(cc);
if (config.getHost() != null) {
String endpoint = "";
if (config.getProtocol() != null) endpoint += config.getProtocol() + "://";
endpoint += config.getHost();
if (config.getPort() > 0) endpoint += ":" + config.getPort();
builder.setEndpointConfiguration(new AwsClientBuilder.EndpointConfiguration(endpoint, config.getRegion()));
} else if (config.getRegion() != null) {
builder.withRegion(config.getRegion());
}
if (config.isDisableVHosts()) {
log.info("The use of virtual hosted buckets has been DISABLED. Path style buckets will be used.");
builder.withPathStyleAccessEnabled(true);
}
s3 = builder.build();
boolean bucketExists = s3.doesBucketExistV2(config.getBucketName());
boolean bucketHasVersions = false;
if (bucketExists && config.isIncludeVersions()) {
// check if versioning has ever been enabled on the bucket (versions will not be collected unless required)
BucketVersioningConfiguration versioningConfig = s3.getBucketVersioningConfiguration(config.getBucketName());
List<String> versionedStates = Arrays.asList(BucketVersioningConfiguration.ENABLED, BucketVersioningConfiguration.SUSPENDED);
bucketHasVersions = versionedStates.contains(versioningConfig.getStatus());
}
if (config.getKeyPrefix() == null) config.setKeyPrefix(""); // make sure keyPrefix isn't null
if (source == this) {
if (config.getExcludedKeys() != null) {
excludedKeyPatterns = new ArrayList<>();
for (String pattern : config.getExcludedKeys()) {
excludedKeyPatterns.add(Pattern.compile(pattern));
}
}
}
if (target == this) {
// create bucket if it doesn't exist
if (!bucketExists && config.isCreateBucket()) {
s3.createBucket(config.getBucketName());
bucketExists = true;
if (config.isIncludeVersions()) {
s3.setBucketVersioningConfiguration(new SetBucketVersioningConfigurationRequest(config.getBucketName(),
new BucketVersioningConfiguration(BucketVersioningConfiguration.ENABLED)));
bucketHasVersions = true;
}
}
// make sure MPU settings are valid
if (config.getMpuThresholdMb() > MAX_PUT_SIZE_MB) {
log.warn("{}MB is above the maximum PUT size of {}MB. the maximum will be used instead",
config.getMpuThresholdMb(), MAX_PUT_SIZE_MB);
config.setMpuThresholdMb(MAX_PUT_SIZE_MB);
}
if (config.getMpuPartSizeMb() < MIN_PART_SIZE_MB) {
log.warn("{}MB is below the minimum MPU part size of {}MB. the minimum will be used instead",
config.getMpuPartSizeMb(), MIN_PART_SIZE_MB);
config.setMpuPartSizeMb(MIN_PART_SIZE_MB);
}
if (source != null) sourceReadWindow = source.getReadWindow();
}
// make sure bucket exists
if (!bucketExists)
throw new ConfigurationException("The bucket " + config.getBucketName() + " does not exist.");
// if syncing versions, make sure plugins support it and bucket has versioning enabled
if (config.isIncludeVersions()) {
if (!(source instanceof AbstractS3Storage && target instanceof AbstractS3Storage))
throw new ConfigurationException("Version migration is only supported between two S3 plugins");
if (!bucketHasVersions)
throw new ConfigurationException("The specified bucket does not have versioning enabled.");
}
}
