software.amazon.awssdk.services.s3.S3Client Java Examples

@Override
public void init(Configuration config) {
  Preconditions.checkArgument(!isNullOrEmpty(config.getString(REGION)));
  String region = config.getString(REGION);

  AwsCredentialsProvider awsCredentialsProvider;
  try {

    if (!isNullOrEmpty(config.getString(ACCESS_KEY)) && !isNullOrEmpty(config.getString(SECRET_KEY))) {
      String accessKey = config.getString(ACCESS_KEY);
      String secretKey = config.getString(SECRET_KEY);
      AwsBasicCredentials awsBasicCredentials = AwsBasicCredentials.create(accessKey, secretKey);
      awsCredentialsProvider = StaticCredentialsProvider.create(awsBasicCredentials);
    } else {
      awsCredentialsProvider =
          AwsCredentialsProviderChain.builder().addCredentialsProvider(SystemPropertyCredentialsProvider.create())
              .addCredentialsProvider(EnvironmentVariableCredentialsProvider.create()).build();
    }

    _s3Client = S3Client.builder().region(Region.of(region)).credentialsProvider(awsCredentialsProvider).build();
  } catch (S3Exception e) {
    throw new RuntimeException("Could not initialize S3PinotFS", e);
  }
}
 

@Test
public void regionalSettingEnabled_usesRegionalIadEndpoint() throws UnsupportedEncodingException {
    EnvironmentVariableHelper environmentVariableHelper = new EnvironmentVariableHelper();
    environmentVariableHelper.set(SdkSystemSetting.AWS_S3_US_EAST_1_REGIONAL_ENDPOINT.environmentVariable(), "regional");

    mockHttpClient.stubNextResponse(mockListObjectsResponse());

    S3Client s3Client = S3Client.builder()
            .credentialsProvider(StaticCredentialsProvider.create(AwsBasicCredentials.create("akid", "skid")))
            .httpClient(mockHttpClient)
            .region(Region.US_EAST_1)
            .serviceConfiguration(S3Configuration.builder()
                    .pathStyleAccessEnabled(true)
                    .build())
            .build();
    try {
        s3Client.listObjects(ListObjectsRequest.builder().bucket(BUCKET).build());
        assertThat(mockHttpClient.getLastRequest().getUri().getHost()).isEqualTo("s3.us-east-1.amazonaws.com");
    } finally {
        environmentVariableHelper.reset();
    }
}
 

@Before
public void setup() {
    StaticCredentialsProvider credentials = StaticCredentialsProvider.create(AwsBasicCredentials.create("akid", "skid"));
    s3Client = S3Client.builder()
                       .credentialsProvider(credentials)
                       .region(Region.US_WEST_2)
                       .endpointOverride(URI.create("http://localhost:" + mockServer.port()))
                       .build();

    s3AsyncClient = S3AsyncClient.builder()
                                 .credentialsProvider(credentials)
                                 .region(Region.US_WEST_2)
                                 .endpointOverride(URI.create("http://localhost:" + mockServer.port()))
                                 .build();

    body = "foo".getBytes(StandardCharsets.UTF_8);
    String checksumAsHexString = "acbd18db4cc2f85cedef654fccc4a4d8";
    bodyEtag = "\"" + checksumAsHexString + "\"";
    bodyWithTrailingChecksum = ArrayUtils.addAll(body, BinaryUtils.fromHex(checksumAsHexString));
}
 

public static void main(String[] args) {

      if (args.length < 3) {
            System.out.println("Please specify a bucket name, a key name that represents a PDF file (ie, book.pdf), and a path (ie, C:\\AWS\\AdobePDF.pdf)");
            System.exit(1);
        }

        String bucketName = args[0];
        String keyName = args[1];
        String path = args[2];

        Region region = Region.US_WEST_2;
        S3Client s3 = S3Client.builder()
                .region(region)
                .build();

        getObjectBytes(s3,bucketName,keyName, path);
    }
 

public static String getPolicy(S3Client s3, String bucketName) {

        String policyText = "";
        System.out.format("Getting policy for bucket: \"%s\"\n\n", bucketName);

        GetBucketPolicyRequest policyReq = GetBucketPolicyRequest.builder()
                .bucket(bucketName)
                .build();

        try {
            GetBucketPolicyResponse policyRes = s3.getBucketPolicy(policyReq);
            policyText = policyRes.policy();
            return policyText;
        } catch (S3Exception e) {
            System.err.println(e.awsErrorDetails().errorMessage());
            System.exit(1);
        }
        return "";
    }
 

public static String putS3Object(S3Client s3, String bucketName, String objectKey, String objectPath) {

        try {
            //Put a file into the bucket
            PutObjectResponse response = s3.putObject(PutObjectRequest.builder()
                            .bucket(bucketName)
                            .key(objectKey)
                            .build(),
                    RequestBody.fromBytes(getObjectFile(objectPath)));

            return response.eTag();
        } catch (S3Exception | FileNotFoundException e) {
            System.err.println(e.getMessage());
            System.exit(1);
        }
        return "";
    }
 

private S3Client getSyncClient(String bucket) throws IOException {
  try {
    return syncClientCache.get(bucket);
  } catch (ExecutionException | SdkClientException e ) {
    final Throwable cause = e.getCause();
    final Throwable toChain;
    if (cause == null) {
      toChain = e;
    } else {
      Throwables.throwIfInstanceOf(cause, UserException.class);
      Throwables.throwIfInstanceOf(cause, IOException.class);

      toChain = cause;
    }

    throw new IOException(String.format("Unable to create a sync S3 client for bucket %s", bucket), toChain);
  }
}
 

@Override
public Object handle (TaskExecution aTask) throws Exception {
  
  AmazonS3URI s3Uri = new AmazonS3URI(aTask.getRequiredString("uri"));
  
  String bucketName = s3Uri.getBucket();
  String key = s3Uri.getKey();
  
  S3Client s3 = S3Client.builder().build();
  
  s3.putObject(PutObjectRequest.builder()
                               .bucket(bucketName)
                               .key(key)
                               .acl(aTask.getString("acl")!=null?ObjectCannedACL.fromValue(aTask.getString("acl")):null)
                               .build(), Paths.get(aTask.getRequiredString("filepath")));
  
  return null;
}
 

@Test
public void completeMultipartUpload_syncClient_errorInResponseBody_correctMessage() {
    String bucket = "Example-Bucket";
    String key = "Example-Object";
    String xmlResponseBody = "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n"
                             + "<Error>\n"
                             + "<Code>CustomError</Code>\n"
                             + "<Message>Foo bar</Message>\n"
                             + "<RequestId>656c76696e6727732072657175657374</RequestId>\n"
                             + "<HostId>Uuag1LuByRx9e6j5Onimru9pO4ZVKnJ2Qz7/C1NPcfTWAtRPfTaOFg==</HostId>\n"
                             + "</Error>";

    stubFor(any(anyUrl()).willReturn(aResponse().withStatus(200).withBody(xmlResponseBody)));

    S3Client s3Client = getSyncClientBuilder().build();

    assertThatThrownBy(() -> s3Client.completeMultipartUpload(r -> r.bucket(bucket)
                                                                    .key(key)
                                                                    .uploadId("upload-id")))
        .satisfies(e -> assertThat(((S3Exception)e).awsErrorDetails().errorMessage()).isEqualTo("Foo bar"));
}
 

@Test
public void completeMultipartUpload_syncClient_errorInResponseBody_invalidErrorXml() {
    String bucket = "Example-Bucket";
    String key = "Example-Object";
    String xmlResponseBody = "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n"
                             + "<Error>\n"
                             + "<SomethingWeird></SomethingWeird>"
                             + "</Error>";

    stubFor(any(anyUrl()).willReturn(aResponse().withStatus(200).withBody(xmlResponseBody)));

    S3Client s3Client = getSyncClientBuilder().build();

    assertThatThrownBy(() -> s3Client.completeMultipartUpload(r -> r.bucket(bucket)
                                                                    .key(key)
                                                                    .uploadId("upload-id")))
        .isInstanceOf(S3Exception.class);
}
 

public static S3Client createS3Client(final Integer mappedPort) {
    final AwsBasicCredentials credentials = AwsBasicCredentials.create("test", "test");
    final StaticCredentialsProvider credentialsProvider = StaticCredentialsProvider.create(credentials);

    return S3Client.builder()
            .credentialsProvider(credentialsProvider)
            .endpointOverride(URI.create(String.format("http://localhost:%d", mappedPort)))
            .region(Region.EU_CENTRAL_1)
            .serviceConfiguration(S3Configuration.builder()
                    .pathStyleAccessEnabled(true)
                    .build())
            .overrideConfiguration(ClientOverrideConfiguration
                    .builder()
                    .putAdvancedOption(SdkAdvancedClientOption.SIGNER, new NoOpSigner())
                    .build())
            .build();
}
 

public static void deleteBucketObjects(S3Client s3, String bucketName, String objectName) {

        // Create a S3Client object
        ArrayList<ObjectIdentifier> toDelete = new ArrayList<ObjectIdentifier>();
        toDelete.add(ObjectIdentifier.builder().key(objectName).build());

        try {
            DeleteObjectsRequest dor = DeleteObjectsRequest.builder()
                    .bucket(bucketName)
                    .delete(Delete.builder().objects(toDelete).build())
                    .build();
            s3.deleteObjects(dor);
        } catch (S3Exception e) {
            System.err.println(e.awsErrorDetails().errorMessage());
            System.exit(1);
        }
        System.out.println("Done!");
    }
 

public static void main(String[] args) {
    final String USAGE = "\n" +
            "To run this example, supply the name of an S3 bucket and at least\n" +
            "one object name (key) to delete.\n" +
            "\n" +
            "Ex: DeleteObjects <bucketname> <objectname>\n";

    if (args.length < 2) {
        System.out.println(USAGE);
        System.exit(1);
    }

    String bucketName = args[0];
    String objectName = args[1];

    System.out.println("Deleting an object from the S3 bucket: " + bucketName);

    // Create a S3Client object
    Region region = Region.US_WEST_2;
    S3Client s3 = S3Client.builder().region(region).build();
    deleteBucketObjects(s3, bucketName, objectName);
}
 

/**
 * Only APIs that operate on buckets uses virtual addressing. Service level operations like ListBuckets will use the normal
 * endpoint.
 */
@Test
public void serviceLevelOperation_UsesStandardEndpoint() throws Exception {
    mockHttpClient.stubNextResponse(mockListBucketsResponse());
    S3Client s3Client = buildClient(null);

    s3Client.listBuckets();

    assertThat(mockHttpClient.getLastRequest().getUri())
            .as("Uses regional S3 endpoint without bucket")
            .isEqualTo(URI.create(ENDPOINT_WITHOUT_BUCKET + "/"));

    assertThat(mockHttpClient.getLastRequest().encodedPath())
            .as("Bucket is not in resource path")
            .isEqualTo("/");
}
 

public static void main(String[] args) {
    final String USAGE = "\n" +
            "Usage:\n" +
            "    DeleteBucketPolicy <bucket>\n\n" +
            "Where:\n" +
            "    bucket - the bucket to delete the policy from (i.e., bucket1)\n\n" +
            "Example:\n" +
            "    bucket1\n\n";

    if (args.length < 1) {
        System.out.println(USAGE);
        System.exit(1);
    }

    String bucketName = args[0];
    System.out.format("Deleting policy from bucket: \"%s\"\n\n", bucketName);

    //Create a S3Client object
    Region region = Region.US_WEST_2;
    S3Client s3 = S3Client.builder().region(region).build();

    //Delete the bucket policy
    deleteS3BucketPolicy(s3, bucketName);
}
 

@Test
public void regionalSettingUnset_usesGlobalEndpoint() throws UnsupportedEncodingException {
    mockHttpClient.stubNextResponse(mockListObjectsResponse());

    S3Client s3Client = S3Client.builder()
            .credentialsProvider(StaticCredentialsProvider.create(AwsBasicCredentials.create("akid", "skid")))
            .httpClient(mockHttpClient)
            .region(Region.US_EAST_1)
            .serviceConfiguration(S3Configuration.builder()
                    .pathStyleAccessEnabled(true)
                    .build())
            .build();

    s3Client.listObjects(ListObjectsRequest.builder().bucket(BUCKET).build());
    assertThat(mockHttpClient.getLastRequest().getUri().getHost()).isEqualTo("s3.amazonaws.com");
}
 

public static void createBucket( S3Client s3Client, String bucketName, Region region) {
    
    try{
        s3Client.createBucket(CreateBucketRequest
            .builder()
            .bucket(bucketName)
            .createBucketConfiguration(
                    CreateBucketConfiguration.builder()
                            .locationConstraint(region.id())
                            .build())
            .build());

        System.out.println(bucketName);
     } catch (S3Exception e) {
        System.err.println(e.awsErrorDetails().errorMessage());
        System.exit(1);
    }
}
 

@Test
public void regionalSettingDisabled_usesGlobalEndpoint() throws UnsupportedEncodingException {
    EnvironmentVariableHelper environmentVariableHelper = new EnvironmentVariableHelper();
    environmentVariableHelper.set(SdkSystemSetting.AWS_S3_US_EAST_1_REGIONAL_ENDPOINT.environmentVariable(), "nonregional");

    mockHttpClient.stubNextResponse(mockListObjectsResponse());

    S3Client s3Client = S3Client.builder()
            .credentialsProvider(StaticCredentialsProvider.create(AwsBasicCredentials.create("akid", "skid")))
            .httpClient(mockHttpClient)
            .region(Region.US_EAST_1)
            .serviceConfiguration(S3Configuration.builder()
                    .pathStyleAccessEnabled(true)
                    .build())
            .build();
    try {
        s3Client.listObjects(ListObjectsRequest.builder().bucket(BUCKET).build());
        assertThat(mockHttpClient.getLastRequest().getUri().getHost()).isEqualTo("s3.amazonaws.com");
    } finally {
        environmentVariableHelper.reset();
    }
}
 

public static void setWebsiteConfig(
        String bucketName, String indexDoc, String errorDoc) {

    WebsiteConfiguration websiteConfig = WebsiteConfiguration.builder()
            .indexDocument(IndexDocument.builder().suffix(indexDoc).build())
            .errorDocument(ErrorDocument.builder().key(errorDoc).build())
            .build();

    Region region = Region.US_WEST_2;
    S3Client s3 = S3Client.builder().region(region).build();

    PutBucketWebsiteRequest pubWebsiteReq = PutBucketWebsiteRequest.builder()
            .bucket(bucketName)
            .websiteConfiguration(websiteConfig)
            .build();

    try {
        s3.putBucketWebsite(pubWebsiteReq);
    } catch (S3Exception e) {
        System.out.format(
                "Failed to set the website configuration for bucket '%s'!\n",
                bucketName);
        System.err.println(e.awsErrorDetails().errorMessage());
        System.exit(1);
    }
}
 

@Test
void buildClientWithDefaults()
{
    AwsS3Sender.Config config = new AwsS3Sender.Config();

    S3Client s3Client = mock(S3Client.class);
    S3ClientBuilder s3ClientBuilder = mock(S3ClientBuilder.class);
    doReturn(s3Client).when(s3ClientBuilder).build();

    new AwsS3Sender(s3ClientBuilder, config);

    verify(s3ClientBuilder, times(1)).build();
    verify(s3ClientBuilder, times(0)).endpointOverride(any());
    verify(s3ClientBuilder, times(0)).region(any());
    verify(s3ClientBuilder, times(0)).credentialsProvider(any());
}
 

public static void deleteBucketWebsiteConfig(S3Client s3,String bucketName ) {

        //Create a DeleteBucketWebsiteRequest object
        DeleteBucketWebsiteRequest delReq = DeleteBucketWebsiteRequest.builder()
                .bucket(bucketName)
                .build();
        try {
            s3.deleteBucketWebsite(delReq);

        } catch (S3Exception e) {
            System.err.println(e.awsErrorDetails().errorMessage());
            System.out.println("Failed to delete website configuration!");
            System.exit(1);
        }
        System.out.println("Done!");
    }
 

public static void main(String[] args) {

         if (args.length < 2) {
              System.out.println("Please specify a bucket name and key name");
              System.exit(1);
          }

        String bucketName = args[0];
        String keyName = args[1];

        Region region = Region.US_WEST_2;
        S3Client s3 = S3Client.builder()
                .region(region)
                .build();

        listTags(s3,bucketName,keyName );
    }
 

public static void listBucketObjects(S3Client s3, String bucketName ) {

       try {
            ListObjectsRequest listObjects = ListObjectsRequest
                    .builder()
                    .bucket(bucketName)
                    .build();

            ListObjectsResponse res = s3.listObjects(listObjects);
            List<S3Object> objects = res.contents();

            for (ListIterator iterVals = objects.listIterator(); iterVals.hasNext(); ) {
                S3Object myValue = (S3Object) iterVals.next();
                System.out.print("\n The name of the key is " + myValue.key());
                System.out.print("\n The object is " + calKb(myValue.size()) + " KBs");
                System.out.print("\n The owner is " + myValue.owner());
                }
        } catch (S3Exception e) {
            System.err.println(e.awsErrorDetails().errorMessage());
            System.exit(1);
        }
    }
 

public static void main(String[] args) {
    final String USAGE = "\n" +
            "DeleteWebsiteConfiguration - delete the website configuration for an S3 bucket\n\n" +
            "Usage: DeleteWebsiteConfiguration <bucket>\n\n" +
            "Where:\n" +
            "   bucket   - the bucket to delete the website configuration from\n";

    if (args.length < 1) {
        System.out.println(USAGE);
        System.exit(1);
    }

    final String bucketName = args[0];

    System.out.format("Deleting website configuration for bucket: %s\n",
            bucketName);

    // Create a S3Client object
    Region region = Region.US_WEST_2;
    S3Client s3 = S3Client.builder().region(region).build();

    deleteBucketWebsiteConfig(s3, bucketName);
}
 

@Test
public void accessPointArn_customEndpoint_throwsIllegalArgumentException() throws Exception {
    URI customEndpoint = URI.create("https://foobar.amazonaws.com");
    mockHttpClient.stubNextResponse(mockListObjectsResponse());
    S3Client s3Client = clientBuilder().endpointOverride(customEndpoint).build();
    String accessPointArn = "arn:aws:s3:ap-south-1:12345678910:accesspoint:foobar";

    assertThatThrownBy(() -> s3Client.listObjects(ListObjectsRequest.builder().bucket(accessPointArn).build()))
        .isInstanceOf(IllegalArgumentException.class)
        .hasMessageContaining("endpoint override");
}
 

/**
 * If customer is using HTTP we need to preserve that scheme when switching to virtual addressing.
 */
@Test
public void customHttpEndpoint_PreservesSchemeWhenSwitchingToVirtualAddressing() throws Exception {
    URI customEndpoint = URI.create("http://s3-external-1.amazonaws.com");
    mockHttpClient.stubNextResponse(mockListObjectsResponse());
    S3Client s3Client = clientBuilderWithMockSigner().endpointOverride(customEndpoint).build();

    s3Client.listObjects(ListObjectsRequest.builder().bucket(BUCKET).build());

    assertEndpointMatches(mockHttpClient.getLastRequest(),
                          String.format("http://%s.s3-external-1.amazonaws.com", BUCKET));
}
 

/**
 * If a custom S3 endpoint is provided (like s3-external-1 or a FIPS endpoint) then we should still use virtual addressing
 * when possible.
 */
@Test
public void customS3EndpointProvided_UsesVirtualAddressing() throws Exception {
    URI customEndpoint = URI.create("https://s3-external-1.amazonaws.com");
    mockHttpClient.stubNextResponse(mockListObjectsResponse());
    S3Client s3Client = clientBuilder().endpointOverride(customEndpoint).build();

    s3Client.listObjects(ListObjectsRequest.builder().bucket(BUCKET).build());

    assertEndpointMatches(mockHttpClient.getLastRequest(),
                          String.format("https://%s.s3-external-1.amazonaws.com", BUCKET));
}
 

/**
 * Creates all the test resources for the tests.
 */
@BeforeClass
public static void createResources() throws Exception {
    s3 = S3Client.builder()
                 .region(REGION)
                 .httpClient(UrlConnectionHttpClient.builder().build())
                 .credentialsProvider(CREDENTIALS_PROVIDER_CHAIN)
                 .overrideConfiguration(o -> o.addExecutionInterceptor(new UserAgentVerifyingInterceptor()))
                 .build();

    createBucket(BUCKET_NAME, REGION);
}
 

@Bean
@ConditionalOnMissingBean
public S3Client s3Client(final AwsProperties awsProperties,
                         final AwsCredentialsProvider awsCredentialsProvider) {
    return S3Client
            .builder()
            .region(of(awsProperties.getRegion()))
            .credentialsProvider(awsCredentialsProvider)
            .build();
}
 

public static void main(String[] args) {
    final String USAGE = "\n" +
            "Usage:\n" +
            "  SetAcl <bucket> [object] <email> <permission>\n\n" +
            "Where:\n" +
            "  bucket     - the bucket to grant permissions on\n" +
            "  object     - the object grant permissions on\n" +
            " owner id      - The ID of the owner of this bucket (you can get this value from the AWS Console)\n" +
            "  permission - The permission(s) to set. Can be one of:\n" +
            "               FULL_CONTROL, READ, WRITE, READ_ACP, WRITE_ACP\n\n" +
            "Examples:\n" +
            "   SetAcl testbucket testobject <uda0cb5d2c39f310136ead6278...> WRITE\n\n";

    if (args.length < 4) {
         System.out.println(USAGE);
        System.exit(1);
    }

    String bucketName = args[0];
    String objectKey = args[1];
    String id = args[2];
    String access = args[3];

    System.out.format("Setting %s access for %s\n", access, id);
    System.out.println("for object: " + objectKey);
    System.out.println(" in bucket: " + bucketName);

    //Create the S3Client object
    Region region = Region.US_WEST_2;
    S3Client s3 = S3Client.builder().region(region).build();

    setBucketAcl(s3, bucketName, id,access);
    System.out.println("Done!");
}