software.amazon.awssdk.services.s3.S3AsyncClient Java Examples

@Before
public void setup() {
    StaticCredentialsProvider credentials = StaticCredentialsProvider.create(AwsBasicCredentials.create("akid", "skid"));
    s3Client = S3Client.builder()
                       .credentialsProvider(credentials)
                       .region(Region.US_WEST_2)
                       .endpointOverride(URI.create("http://localhost:" + mockServer.port()))
                       .build();

    s3AsyncClient = S3AsyncClient.builder()
                                 .credentialsProvider(credentials)
                                 .region(Region.US_WEST_2)
                                 .endpointOverride(URI.create("http://localhost:" + mockServer.port()))
                                 .build();

    body = "foo".getBytes(StandardCharsets.UTF_8);
    String checksumAsHexString = "acbd18db4cc2f85cedef654fccc4a4d8";
    bodyEtag = "\"" + checksumAsHexString + "\"";
    bodyWithTrailingChecksum = ArrayUtils.addAll(body, BinaryUtils.fromHex(checksumAsHexString));
}
 

@Bean
public S3AsyncClient s3client(S3ClientConfigurarionProperties s3props, AwsCredentialsProvider credentialsProvider) {

    SdkAsyncHttpClient httpClient = NettyNioAsyncHttpClient.builder()
        .writeTimeout(Duration.ZERO)
        .maxConcurrency(64)
        .build();

    S3Configuration serviceConfiguration = S3Configuration.builder()
        .checksumValidationEnabled(false)
        .chunkedEncodingEnabled(true)
        .build();

    S3AsyncClientBuilder b = S3AsyncClient.builder()
        .httpClient(httpClient)
        .region(s3props.getRegion())
        .credentialsProvider(credentialsProvider)
        .serviceConfiguration(serviceConfiguration);

    if (s3props.getEndpoint() != null) {
        b = b.endpointOverride(s3props.getEndpoint());
    }

    return b.build();
}
 

public Downloader(S3AsyncClient s3Client, ApplicationStatusRepository applicationStatusRepository,
                  NetworkAddressBook networkAddressBook, DownloaderProperties downloaderProperties,
                  MeterRegistry meterRegistry) {
    this.s3Client = s3Client;
    this.applicationStatusRepository = applicationStatusRepository;
    this.networkAddressBook = networkAddressBook;
    this.downloaderProperties = downloaderProperties;
    this.meterRegistry = meterRegistry;
    signatureDownloadThreadPool = Executors.newFixedThreadPool(downloaderProperties.getThreads());
    Runtime.getRuntime().addShutdownHook(new Thread(signatureDownloadThreadPool::shutdown));

    signatureVerificationMetric = Counter.builder("hedera.mirror.signature.verification")
            .description("The number of signatures verified from a particular node")
            .tag("type", downloaderProperties.getStreamType().toString());

    streamVerificationMetric = Timer.builder("hedera.mirror.stream.verification")
            .description("The duration in seconds it took to verify consensus and hash chain of a stream file")
            .tag("type", downloaderProperties.getStreamType().toString());
}
 

public static void main(String[] args) {
	S3AsyncClient client = S3AsyncClient.create();
    CompletableFuture<PutObjectResponse> future = client.putObject(
            PutObjectRequest.builder()
                            .bucket(BUCKET)
                            .key(KEY)
                            .build(),
            AsyncRequestBody.fromFile(Paths.get("myfile.in"))
    );
    future.whenComplete((resp, err) -> {
        try {
            if (resp != null) {
                System.out.println("my response: " + resp);
            } else {
                // Handle error
                err.printStackTrace();
            }
        } finally {
            // Lets the application shut down. Only close the client when you are completely done with it.
            client.close();
        }
    });

    future.join();
}
 

@Bean
@ConditionalOnProperty(prefix = "hedera.mirror.importer.downloader", name = "cloudProvider", havingValue = "GCP")
public S3AsyncClient gcpCloudStorageClient() {
    log.info("Configured to download from GCP with bucket name '{}'", downloaderProperties.getBucketName());
    // Any valid region for aws client. Ignored by GCP.
    S3AsyncClientBuilder clientBuilder = asyncClientBuilder("us-east-1")
            .endpointOverride(URI.create(downloaderProperties.getCloudProvider().getEndpoint()));
    String projectId = downloaderProperties.getGcpProjectId();
    if (StringUtils.isNotBlank(projectId)) {
        clientBuilder.overrideConfiguration(builder -> builder.addExecutionInterceptor(new ExecutionInterceptor() {
            @Override
            public SdkHttpRequest modifyHttpRequest(
                    Context.ModifyHttpRequest context, ExecutionAttributes executionAttributes) {
                return context.httpRequest().toBuilder()
                        .appendRawQueryParameter("userProject", projectId).build();
            }
        }));
    }
    return clientBuilder.build();
}
 

@Test
public void completeMultipartUpload_asyncClient_errorInResponseBody_invalidErrorXml() {
    String bucket = "Example-Bucket";
    String key = "Example-Object";
    String xmlResponseBody = "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n"
                             + "<Error>\n"
                             + "<SomethingWeird></SomethingWeird>"
                             + "</Error>";

    stubFor(any(anyUrl()).willReturn(aResponse().withStatus(200).withBody(xmlResponseBody)));

    S3AsyncClient s3Client = getAsyncClientBuilder().build();

    assertThatThrownBy(() -> s3Client.completeMultipartUpload(r -> r.bucket(bucket)
                                                                    .key(key)
                                                                    .uploadId("upload-id"))
                      .join())
        .isInstanceOf(CompletionException.class)
        .hasCauseInstanceOf(S3Exception.class);
}
 

@Test
public void completeMultipartUpload_asyncClient_errorInResponseBody_correctMessage() {
    String bucket = "Example-Bucket";
    String key = "Example-Object";
    String xmlResponseBody = "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n"
                             + "<Error>\n"
                             + "<Code>CustomError</Code>\n"
                             + "<Message>Foo bar</Message>\n"
                             + "<RequestId>656c76696e6727732072657175657374</RequestId>\n"
                             + "<HostId>Uuag1LuByRx9e6j5Onimru9pO4ZVKnJ2Qz7/C1NPcfTWAtRPfTaOFg==</HostId>\n"
                             + "</Error>";

    stubFor(any(anyUrl()).willReturn(aResponse().withStatus(200).withBody(xmlResponseBody)));

    S3AsyncClient s3Client = getAsyncClientBuilder().build();

    assertThatThrownBy(() -> s3Client.completeMultipartUpload(r -> r.bucket(bucket)
                                                                    .key(key)
                                                                    .uploadId("upload-id"))
                      .join())
        .satisfies(e -> {
            S3Exception s3Exception = (S3Exception) e.getCause();
            assertThat(s3Exception.awsErrorDetails().errorMessage()).isEqualTo("Foo bar");
        });
}
 

@Test
public void completeMultipartUpload_asyncClient_errorInResponseBody_correctCode() {
    String bucket = "Example-Bucket";
    String key = "Example-Object";
    String xmlResponseBody = "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n"
                             + "<Error>\n"
                             + "<Code>CustomError</Code>\n"
                             + "<Message>We encountered an internal error. Please try again.</Message>\n"
                             + "<RequestId>656c76696e6727732072657175657374</RequestId>\n"
                             + "<HostId>Uuag1LuByRx9e6j5Onimru9pO4ZVKnJ2Qz7/C1NPcfTWAtRPfTaOFg==</HostId>\n"
                             + "</Error>";

    stubFor(any(anyUrl()).willReturn(aResponse().withStatus(200).withBody(xmlResponseBody)));

    S3AsyncClient s3Client = getAsyncClientBuilder().build();

    assertThatThrownBy(() -> s3Client.completeMultipartUpload(r -> r.bucket(bucket)
                                                                    .key(key)
                                                                    .uploadId("upload-id"))
                                     .join())
        .satisfies(e -> {
            S3Exception s3Exception = (S3Exception) e.getCause();
            assertThat(s3Exception.awsErrorDetails().errorCode()).isEqualTo("CustomError");
        });
}
 

@Test
public void completeMultipartUpload_asyncClient_errorInResponseBody_correctType() {
    String bucket = "Example-Bucket";
    String key = "Example-Object";
    String xmlResponseBody = "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n"
                             + "<Error>\n"
                             + "<Code>InternalError</Code>\n"
                             + "<Message>We encountered an internal error. Please try again.</Message>\n"
                             + "<RequestId>656c76696e6727732072657175657374</RequestId>\n"
                             + "<HostId>Uuag1LuByRx9e6j5Onimru9pO4ZVKnJ2Qz7/C1NPcfTWAtRPfTaOFg==</HostId>\n"
                             + "</Error>";

    stubFor(any(anyUrl()).willReturn(aResponse().withStatus(200).withBody(xmlResponseBody)));

    S3AsyncClient s3Client = getAsyncClientBuilder().build();

    assertThatThrownBy(() -> s3Client.completeMultipartUpload(r -> r.bucket(bucket)
                                                                    .key(key)
                                                                    .uploadId("upload-id"))
                                     .join())
        .isInstanceOf(CompletionException.class)
        .hasCauseInstanceOf(S3Exception.class);
}
 

@Override
@Bean(preDestroy = "close")
@Singleton
@Requires(beans = SdkAsyncHttpClient.class)
public S3AsyncClient asyncClient(S3AsyncClientBuilder builder) {
    return super.asyncClient(builder);
}
 

public static void main(String[] args) {

        final String USAGE = "\n" +
                "Usage:\n" +
                "    S3AsyncOps <bucketname> <objectname> <path>\n\n" +
                "Where:\n" +
                "    bucketname - the name of the bucket (i.e., bucket1)\n\n" +
                "    objectname - the name pf the object object (i.e., book.pdf)\n" +
                "    path - the local path to the file (i.e., C:\\AWS\\book.pdf)\n" +
                "Example:\n" +
                "    bucket1 book.pdf  C:\\AWS\\book.pdf\n";

         String bucketName = args[0];
         String objectKey = args[1];
         String path = args[2];

        S3AsyncClient client = S3AsyncClient.create();
        final CompletableFuture<GetObjectResponse> futureGet = client.getObject(
                GetObjectRequest.builder()
                        .bucket(bucketName)
                        .key(objectKey)
                        .build(),
                AsyncResponseTransformer.toFile(Paths.get(path)));
        futureGet.whenComplete((resp, err) -> {
            try {
                if (resp != null) {
                    System.out.println(resp);
                } else {
                    // Handle error
                    err.printStackTrace();
                }
            } finally {
                // Lets the application shut down. Only close the client when you are completely done with it
                client.close();
            }
        });

        futureGet.join();
    }
 

@Test
public void completeMultipartUpload_asyncClient_completeResponse() {
    String location = "http://Example-Bucket.s3.amazonaws.com/Example-Object";
    String bucket = "Example-Bucket";
    String key = "Example-Object";
    String eTag = "\"3858f62230ac3c915f300c664312c11f-9\"";
    String xmlResponseBody = String.format(
        "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n"
        + "<CompleteMultipartUploadResult xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\">\n"
        + "<Location>%s</Location>\n"
        + "<Bucket>%s</Bucket>\n"
        + "<Key>%s</Key>\n"
        + "<ETag>%s</ETag>\n"
        + "</CompleteMultipartUploadResult>", location, bucket, key, eTag);

    stubFor(any(anyUrl()).willReturn(aResponse().withStatus(200).withBody(xmlResponseBody)));

    S3AsyncClient s3Client = getAsyncClientBuilder().build();

    CompleteMultipartUploadResponse response = s3Client.completeMultipartUpload(
        r -> r.bucket(bucket).key(key).uploadId("upload-id")).join();

    assertThat(response.location()).isEqualTo(location);
    assertThat(response.bucket()).isEqualTo(bucket);
    assertThat(response.key()).isEqualTo(key);
    assertThat(response.eTag()).isEqualTo(eTag);
}
 

private S3AsyncClientBuilder getAsyncClientBuilder() {
    return S3AsyncClient.builder()
                        .region(Region.US_EAST_1)
                        .endpointOverride(HTTP_LOCALHOST_URI)
                        .credentialsProvider(
                            StaticCredentialsProvider.create(AwsBasicCredentials.create("key", "secret")));

}
 

@Test
public void getBucketPolicy_asyncClient() {
    stubFor(any(anyUrl()).willReturn(aResponse().withStatus(200).withBody(EXAMPLE_POLICY)));

    S3AsyncClient s3Client = getAsyncClientBuilder().build();

    GetBucketPolicyResponse response = s3Client.getBucketPolicy(r -> r.bucket(EXAMPLE_BUCKET)).join();
    assertThat(response.policy()).isEqualTo(EXAMPLE_POLICY);
}
 

private S3AsyncClientBuilder getAsyncClientBuilder() {
    return S3AsyncClient.builder()
                        .region(Region.US_EAST_1)
                        .endpointOverride(HTTP_LOCALHOST_URI)
                        .credentialsProvider(
                            StaticCredentialsProvider.create(AwsBasicCredentials.create("key", "secret")));

}
 

public RuntimeValue<S3AsyncClient> buildAsyncClient(RuntimeValue<? extends AwsClientBuilder> builder,
        BeanContainer beanContainer,
        ShutdownContext shutdown) {
    S3ClientProducer producer = beanContainer.instance(S3ClientProducer.class);
    producer.setAsyncConfiguredBuilder((S3AsyncClientBuilder) builder.getValue());
    shutdown.addShutdownTask(producer::destroy);
    return new RuntimeValue<>(producer.asyncClient());
}
 

public RuntimeValue<AwsClientBuilder> createAsyncBuilder(S3Config config,
        RuntimeValue<SdkAsyncHttpClient.Builder> transport) {

    S3AsyncClientBuilder builder = S3AsyncClient.builder();
    configureS3Client(builder, config);

    if (transport != null) {
        builder.httpClientBuilder(transport.getValue());
    }
    return new RuntimeValue<>(builder);
}
 

public static CompletableFuture<Boolean> createBucketAsync(final S3AsyncClient s3, String bucket) {
    return s3.createBucket(createBucketRequest(bucket))
            .thenApply(resp -> true)
            .exceptionally(th -> {
                if (th.getCause() instanceof BucketAlreadyExistsException) {
                    LOG.info("Bucket already exists");
                    return true;
                } else {
                    LOG.error("Failed table bucket", th);
                    return false;
                }
            });
}
 

private S3AsyncClientBuilder asyncClientBuilder(String region) {
    SdkAsyncHttpClient httpClient = NettyNioAsyncHttpClient.builder()
            .maxConcurrency(downloaderProperties.getMaxConcurrency())
            .connectionMaxIdleTime(Duration.ofSeconds(5))  // https://github.com/aws/aws-sdk-java-v2/issues/1122
            .build();

    return S3AsyncClient.builder()
            .region(Region.of(region))
            .credentialsProvider(awsCredentialsProvider(
                    downloaderProperties.getAccessKey(), downloaderProperties.getSecretKey()))
            .httpClient(httpClient)
            .overrideConfiguration(c -> c.addExecutionInterceptor(metricsExecutionInterceptor));
}
 

@Bean
@ConditionalOnProperty(prefix = "hedera.mirror.importer.downloader", name = "cloudProvider", havingValue = "S3",
        matchIfMissing = true)
public S3AsyncClient s3CloudStorageClient() {
    log.info("Configured to download from S3 in region {} with bucket name '{}'",
            downloaderProperties.getRegion(), downloaderProperties.getBucketName());
    S3AsyncClientBuilder clientBuilder = asyncClientBuilder(downloaderProperties.getRegion());
    String endpointOverride = downloaderProperties.getEndpointOverride();
    if (endpointOverride != null) {
        log.info("Overriding s3 client endpoint to {}", endpointOverride);
        clientBuilder.endpointOverride(URI.create(endpointOverride));
    }
    return clientBuilder.build();
}
 

@Produces
@ApplicationScoped
public S3AsyncClient asyncClient() {
    asyncClient = asyncConfiguredBuilder.build();
    return asyncClient;
}
 

@Override
protected DotName asyncClientName() {
    return DotName.createSimple(S3AsyncClient.class.getName());
}
 

public S3AsyncByteReader(S3AsyncClient client, String bucket, String path) {
  super();
  this.client = client;
  this.bucket = bucket;
  this.path = path;
}
 

public RecordFileDownloader(
        S3AsyncClient s3Client, ApplicationStatusRepository applicationStatusRepository,
        NetworkAddressBook networkAddressBook, RecordDownloaderProperties downloaderProperties,
        MeterRegistry meterRegistry) {
    super(s3Client, applicationStatusRepository, networkAddressBook, downloaderProperties, meterRegistry);
}
 

public AccountBalancesDownloader(
        S3AsyncClient s3Client, ApplicationStatusRepository applicationStatusRepository,
        NetworkAddressBook networkAddressBook, BalanceDownloaderProperties downloaderProperties,
        MeterRegistry meterRegistry) {
    super(s3Client, applicationStatusRepository, networkAddressBook, downloaderProperties, meterRegistry);
}
 

@Override
protected S3AsyncClientBuilder createAsyncBuilder() {
    return S3AsyncClient.builder()
            .serviceConfiguration(configuration.getBuilder().build());
}
 

public DownloadResource(S3AsyncClient s3client, S3ClientConfigurarionProperties s3config) {
    this.s3client = s3client;
    this.s3config = s3config;        
}
 

public UploadResource(S3AsyncClient s3client, S3ClientConfigurarionProperties s3config) {
    this.s3client = s3client;
    this.s3config = s3config;        
}