software.amazon.awssdk.services.s3.S3ClientBuilder Java Examples

@Test
void buildClientWithDefaults()
{
    AwsS3Sender.Config config = new AwsS3Sender.Config();

    S3Client s3Client = mock(S3Client.class);
    S3ClientBuilder s3ClientBuilder = mock(S3ClientBuilder.class);
    doReturn(s3Client).when(s3ClientBuilder).build();

    new AwsS3Sender(s3ClientBuilder, config);

    verify(s3ClientBuilder, times(1)).build();
    verify(s3ClientBuilder, times(0)).endpointOverride(any());
    verify(s3ClientBuilder, times(0)).region(any());
    verify(s3ClientBuilder, times(0)).credentialsProvider(any());
}
 

@VisibleForTesting
protected S3Client buildClient(S3ClientBuilder s3ClientBuilder)
{
    if (config.getEndpoint() != null) {
        try {
            URI uri = new URI(config.getEndpoint());
            s3ClientBuilder.endpointOverride(uri);
        }
        catch (URISyntaxException e) {
            throw new NonRetryableException(
                    String.format("Invalid endpoint. %s", config.getEndpoint()), e);
        }
    }

    if (config.getRegion() != null) {
        s3ClientBuilder.region(Region.of(config.getRegion()));
    }

    if (config.getAwsAccessKeyId() != null && config.getAwsSecretAccessKey() != null) {
        AwsBasicCredentials credentials =
                AwsBasicCredentials.create(config.getAwsAccessKeyId(), config.getAwsSecretAccessKey());
        s3ClientBuilder.credentialsProvider(StaticCredentialsProvider.create(credentials));
    }

    return s3ClientBuilder.build();
}
 

/**
 * @return Client builder instance preconfigured with credentials and region using the {@link #mockHttpClient} for transport
 * and {@link #mockSigner} for signing. Using actual AwsS3V4Signer results in NPE as the execution goes into payload signing
 * due to "http" protocol and input stream is not mark supported.
 */
private S3ClientBuilder clientBuilderWithMockSigner() {
    return S3Client.builder()
                   .credentialsProvider(StaticCredentialsProvider.create(AwsBasicCredentials.create("akid", "skid")))
                   .region(Region.AP_SOUTH_1)
                   .overrideConfiguration(ClientOverrideConfiguration.builder()
                                                                     .putAdvancedOption(SdkAdvancedClientOption.SIGNER,
                                                                                        mockSigner)
                                                                     .build())
                   .httpClient(mockHttpClient);
}
 

@Test
void close()
{
    S3Client s3Client = mock(S3Client.class);
    S3ClientBuilder s3ClientBuilder = mock(S3ClientBuilder.class);
    doReturn(s3Client).when(s3ClientBuilder).build();

    AwsS3Sender sender = new AwsS3Sender(s3ClientBuilder);
    sender.close();

    verify(s3Client, times(1)).close();
}
 

private void testSend(
        AwsS3Sender.Config config,
        boolean gzipCompressed,
        int failures)
        throws IOException
{
    S3Client s3Client = mock(S3Client.class);
    S3ClientBuilder s3ClientBuilder = mock(S3ClientBuilder.class);
    doReturn(s3Client).when(s3ClientBuilder).build();

    AtomicInteger retryCount = new AtomicInteger();
    doAnswer(invocation -> {
        PutObjectRequest request = invocation.getArgument(0);

        assertEquals("hello.world", request.bucket());
        assertEquals("2345/01/31/23/59-59-99999.data", request.key());

        RequestBody body = invocation.getArgument(1);
        try (InputStream s3In = body.contentStreamProvider().newStream();
                InputStream in = gzipCompressed ? new GZIPInputStream(s3In) : s3In) {
            byte[] content = ByteStreams.toByteArray(in);
            assertEquals("0123456789", new String(content, StandardCharsets.UTF_8));
        }

        if (retryCount.getAndIncrement() < failures) {
            throw new RuntimeException("Something happened");
        }

        return null;
    }).when(s3Client).putObject(any(PutObjectRequest.class), any(RequestBody.class));

    AwsS3Sender sender = new AwsS3Sender(s3ClientBuilder, config);

    sender.send("hello.world", "2345/01/31/23/59-59-99999.data",
            ByteBuffer.wrap("0123456789".getBytes(StandardCharsets.UTF_8)));

    verify(s3Client, times(failures + 1)).putObject(any(PutObjectRequest.class), any(RequestBody.class));

    sender.close();
}
 

@Test
void buildClientWithCustomizedConfig()
{
    AwsS3Sender.Config config = new AwsS3Sender.Config();
    config.setEndpoint("https://another.s3endpoi.nt");
    config.setRegion("ap-northeast-1");
    config.setAwsAccessKeyId("foo");
    config.setAwsSecretAccessKey("bar");

    S3Client s3Client = mock(S3Client.class);
    S3ClientBuilder s3ClientBuilder = mock(S3ClientBuilder.class);
    doReturn(s3Client).when(s3ClientBuilder).build();
    doAnswer(invocation -> {
        AwsCredentialsProvider provider = invocation.getArgument(0);
        AwsCredentials awsCredentials = provider.resolveCredentials();
        assertEquals("foo", awsCredentials.accessKeyId());
        assertEquals("bar", awsCredentials.secretAccessKey());
        return null;
    }).when(s3ClientBuilder).credentialsProvider(any());

    new AwsS3Sender(s3ClientBuilder, config);

    verify(s3ClientBuilder, times(1)).build();
    verify(s3ClientBuilder, times(1)).endpointOverride(eq(URI.create("https://another.s3endpoi.nt")));
    verify(s3ClientBuilder, times(1)).region(eq(Region.AP_NORTHEAST_1));
    verify(s3ClientBuilder, times(1)).credentialsProvider(any());
}
 

public AwsS3Sender(S3ClientBuilder s3ClientBuilder, Config config)
{
    config.validateValues();
    this.config = config;
    this.retryPolicy =
            new RetryPolicy<Void>().
                    handleIf(ex -> {
                        if (ex == null) {
                            // Success. Shouldn't retry.
                            return false;
                        }

                        ErrorHandler errorHandler = config.getErrorHandler();

                        if (errorHandler != null) {
                            errorHandler.handle(ex);
                        }

                        if (ex instanceof InterruptedException || ex instanceof NonRetryableException) {
                            return false;
                        }

                        return true;
                    }).
                    withBackoff(
                            getRetryInternalMs(),
                            getMaxRetryInternalMs(),
                            ChronoUnit.MILLIS,
                            getRetryFactor()).
                    withMaxRetries(getRetryMax());

    this.client = buildClient(s3ClientBuilder);
}
 

protected S3Client createS3Client(final ProviderConfigurationContext configurationContext) {

        final S3ClientBuilder builder = S3Client.builder()
                .region(getRegion(configurationContext))
                .credentialsProvider(getCredentialsProvider(configurationContext));

        final URI s3EndpointOverride = getS3EndpointOverride(configurationContext);
        if (s3EndpointOverride != null) {
            builder.endpointOverride(s3EndpointOverride);
        }

        return builder.build();

    }
 

private S3ClientBuilder getSyncClientBuilder() {

        return S3Client.builder()
                       .region(Region.US_EAST_1)
                       .endpointOverride(HTTP_LOCALHOST_URI)
                       .credentialsProvider(
                           StaticCredentialsProvider.create(AwsBasicCredentials.create("key", "secret")));
    }
 

private S3ClientBuilder getSyncClientBuilder() {

        return S3Client.builder()
                       .region(Region.US_EAST_1)
                       .endpointOverride(HTTP_LOCALHOST_URI)
                       .credentialsProvider(
                           StaticCredentialsProvider.create(AwsBasicCredentials.create("key", "secret")));
    }
 

/**
 * @return Client builder instance preconfigured with credentials and region using the {@link #mockHttpClient} for transport.
 */
private S3ClientBuilder clientBuilder() {
    return S3Client.builder()
                   .credentialsProvider(StaticCredentialsProvider.create(AwsBasicCredentials.create("akid", "skid")))
                   .region(Region.AP_SOUTH_1)
                   .httpClient(mockHttpClient);
}
 

public RuntimeValue<S3Client> buildClient(RuntimeValue<? extends AwsClientBuilder> builder,
        BeanContainer beanContainer,
        ShutdownContext shutdown) {
    S3ClientProducer producer = beanContainer.instance(S3ClientProducer.class);
    producer.setSyncConfiguredBuilder((S3ClientBuilder) builder.getValue());
    shutdown.addShutdownTask(producer::destroy);
    return new RuntimeValue<>(producer.client());
}
 

public RuntimeValue<AwsClientBuilder> createSyncBuilder(S3Config config, RuntimeValue<Builder> transport) {
    S3ClientBuilder builder = S3Client.builder();
    configureS3Client(builder, config);

    if (transport != null) {
        builder.httpClientBuilder(transport.getValue());
    }
    return new RuntimeValue<>(builder);
}
 

@Override
public S3ClientBuilder onCreated(BeanCreatedEvent<S3ClientBuilder> event) {
    S3ClientBuilder builder = event.getBean();
    builder.overrideConfiguration(ClientOverrideConfiguration.builder().retryPolicy(RetryMode.LEGACY).build());

    return builder;
}
 

private S3RandomAccessFile(String url) throws IOException {
  super(url, s3BufferSize, s3MaxReadCacheSize);

  // Region is tricky. Since we are using AWS SDK to manage connections to all object stores, we might have users
  // who use netCDF-Java and never touch AWS. If that's they case, they likely have not setup a basic credentials or
  // configuration file, and thus lack a default region. What we will do here is check to see if there is one set.
  // If, by the time we make the client, profileRegion isn't set, we will default to the AWS_GLOBAL region, which is
  // like a no-op region when it comes to S3. This will allow requests to non-AWS-S3 object stores to work, because
  // a region must be set, even if it's useless.
  Optional<Region> profileRegion = ProfileFile.defaultProfileFile().profile("default")
      .map(p -> p.properties().get(ProfileProperty.REGION)).map(Region::of);

  try {
    uri = new CdmS3Uri(url);
  } catch (URISyntaxException urie) {
    // If we are given a string that is not a valid CdmS3Uri
    // throw an IOException
    throw new IOException(urie.getCause());
  }

  Builder httpConfig = ApacheHttpClient.builder().maxConnections(maxConnections)
      .connectionTimeout(Duration.ofMillis(connectionTimeout)).socketTimeout(Duration.ofMillis(socketTimeout));

  S3ClientBuilder s3ClientBuilder = S3Client.builder().httpClientBuilder(httpConfig);

  // if we are accessing an S3 compatible service, we need to override the server endpoint
  uri.getEndpoint().ifPresent(s3ClientBuilder::endpointOverride);

  // build up a chain of credentials providers
  AwsCredentialsProviderChain.Builder cdmCredentialsProviderChainBuilder = AwsCredentialsProviderChain.builder();

  // if uri has a profile name, we need setup a credentials provider to look for potential credentials, and see if a
  // region has been set
  if (uri.getProfile().isPresent()) {
    // get the profile name
    String profileName = uri.getProfile().get();

    ProfileCredentialsProvider namedProfileCredentials =
        ProfileCredentialsProvider.builder().profileName(profileName).build();

    // add it to the chain that it is the first thing checked for credentials
    cdmCredentialsProviderChainBuilder.addCredentialsProvider(namedProfileCredentials);

    // Read the region associated with the profile, if set
    // Note: the java sdk does not do this by default
    Optional<Region> namedProfileRegion = ProfileFile.defaultProfileFile().profile(profileName)
        .map(p -> p.properties().get(ProfileProperty.REGION)).map(Region::of);
    // if the named profile has a region, update profileRegion to use it.
    if (namedProfileRegion.isPresent()) {
      profileRegion = namedProfileRegion;
    }
  }

  // Add the Default Credentials Provider Chain:
  // https://docs.aws.amazon.com/sdk-for-java/v2/developer-guide/credentials.html
  cdmCredentialsProviderChainBuilder.addCredentialsProvider(DefaultCredentialsProvider.create());

  // Add the AnonymousCredentialsProvider last
  cdmCredentialsProviderChainBuilder.addCredentialsProvider(AnonymousCredentialsProvider.create());

  // build the credentials provider that we'll use
  AwsCredentialsProviderChain cdmCredentialsProviderChain = cdmCredentialsProviderChainBuilder.build();

  // Add the credentials provider to the client builder
  s3ClientBuilder.credentialsProvider(cdmCredentialsProviderChain);

  // Set the region for the client builder (default to AWS_GLOBAL)
  s3ClientBuilder.region(profileRegion.orElse(Region.AWS_GLOBAL));

  // Build the client
  client = s3ClientBuilder.build();

  // request HEAD for the object
  HeadObjectRequest headdObjectRequest =
      HeadObjectRequest.builder().bucket(uri.getBucket()).key(uri.getKey()).build();

  objectHeadResponse = client.headObject(headdObjectRequest);
}
 

private static S3ClientBuilder getClientBuilder() {
    return S3Client.builder()
                   .region(DEFAULT_REGION)
                   .credentialsProvider(CREDENTIALS_PROVIDER_CHAIN);
}
 

public AwsS3Sender(S3ClientBuilder s3ClientBuilder)
{
    this(s3ClientBuilder, new Config());
}
 

public void setSyncConfiguredBuilder(S3ClientBuilder syncConfiguredBuilder) {
    this.syncConfiguredBuilder = syncConfiguredBuilder;
}
 

@Override
@Bean(preDestroy = "close")
@Singleton
public S3Client syncClient(S3ClientBuilder builder) {
    return super.syncClient(builder);
}
 

@Override
@Singleton
public S3ClientBuilder syncBuilder(SdkHttpClient httpClient) {
    return super.syncBuilder(httpClient);
}
 

@Override
protected S3ClientBuilder createSyncBuilder() {
    return S3Client.builder()
            .serviceConfiguration(configuration.getBuilder().build());
}