software.amazon.awssdk.services.s3.model.ListObjectsRequest Java Examples

@Test
public void regionalSettingEnabled_usesRegionalIadEndpoint() throws UnsupportedEncodingException {
    EnvironmentVariableHelper environmentVariableHelper = new EnvironmentVariableHelper();
    environmentVariableHelper.set(SdkSystemSetting.AWS_S3_US_EAST_1_REGIONAL_ENDPOINT.environmentVariable(), "regional");

    mockHttpClient.stubNextResponse(mockListObjectsResponse());

    S3Client s3Client = S3Client.builder()
            .credentialsProvider(StaticCredentialsProvider.create(AwsBasicCredentials.create("akid", "skid")))
            .httpClient(mockHttpClient)
            .region(Region.US_EAST_1)
            .serviceConfiguration(S3Configuration.builder()
                    .pathStyleAccessEnabled(true)
                    .build())
            .build();
    try {
        s3Client.listObjects(ListObjectsRequest.builder().bucket(BUCKET).build());
        assertThat(mockHttpClient.getLastRequest().getUri().getHost()).isEqualTo("s3.us-east-1.amazonaws.com");
    } finally {
        environmentVariableHelper.reset();
    }
}
 

@Test
public void shouldThrowExceptionOnMissingBucket() {
    //given
    when(mock.listObjects(any(ListObjectsRequest.class))).thenThrow(NoSuchBucketException.builder().build());

    this.context.register(TogglzAutoConfiguration.class);
    this.context.register(S3TestConfiguration.class);

    TestPropertyValues
            .of("edison.togglz.s3.enabled=true")
            .and("edison.togglz.s3.bucket-name=togglz-bucket")
            .and("edison.togglz.s3.check-bucket=true")
            .applyTo(context);

    //expect
    Assertions.assertThrows(BeanCreationException.class, this.context::refresh);
}
 

public static void listBucketObjects(S3Client s3, String bucketName ) {

       try {
            ListObjectsRequest listObjects = ListObjectsRequest
                    .builder()
                    .bucket(bucketName)
                    .build();

            ListObjectsResponse res = s3.listObjects(listObjects);
            List<S3Object> objects = res.contents();

            for (ListIterator iterVals = objects.listIterator(); iterVals.hasNext(); ) {
                S3Object myValue = (S3Object) iterVals.next();
                System.out.print("\n The name of the key is " + myValue.key());
                System.out.print("\n The object is " + calKb(myValue.size()) + " KBs");
                System.out.print("\n The owner is " + myValue.owner());
                }
        } catch (S3Exception e) {
            System.err.println(e.awsErrorDetails().errorMessage());
            System.exit(1);
        }
    }
 

@Test
public void listObjectsNoParameters() {
    ListObjectsResponse result = s3.listObjects(ListObjectsRequest.builder().bucket(bucketName).build());
    List<S3Object> objects = result.contents();

    assertEquals(keys.size(), objects.size());
    assertEquals(bucketName, result.name());
    assertS3ObjectSummariesAreValid(objects);
    assertNotNull(result.maxKeys());

    // We didn't use a delimiter, so we expect these to be empty/null
    assertNull(result.delimiter());

    // We don't expect any truncated results
    assertFalse(result.isTruncated());

    // We didn't set other request parameters, so we expect them to be empty
    assertNull(result.encodingType());
    assertThat(result.prefix()).isEmpty();
}
 

@Test
public void listObjectsWithAllElements() {
    String delimiter = "/";
    String marker = "aaa";
    ListObjectsResponse result = s3.listObjects(ListObjectsRequest.builder()
                                                                  .bucket(bucketName)
                                                                  .prefix(KEY_NAME_WITH_SPECIAL_CHARS)
                                                                  .marker(marker)
                                                                  .encodingType(EncodingType.URL)
                                                                  .delimiter(delimiter)
                                                                  .build());
    List<S3Object> objects = result.contents();

    assertEquals(bucketName, result.name());
    assertS3ObjectSummariesAreValid(objects);
    assertEquals(marker, result.marker());
    assertEquals(delimiter, result.delimiter());
    assertEquals(KEY_NAME_WITH_SPECIAL_CHARS, result.prefix());

    assertFalse(result.isTruncated());
    assertTrue(result.maxKeys() >= 1000);
}
 

@Test
public void regionalSettingUnset_usesGlobalEndpoint() throws UnsupportedEncodingException {
    mockHttpClient.stubNextResponse(mockListObjectsResponse());

    S3Client s3Client = S3Client.builder()
            .credentialsProvider(StaticCredentialsProvider.create(AwsBasicCredentials.create("akid", "skid")))
            .httpClient(mockHttpClient)
            .region(Region.US_EAST_1)
            .serviceConfiguration(S3Configuration.builder()
                    .pathStyleAccessEnabled(true)
                    .build())
            .build();

    s3Client.listObjects(ListObjectsRequest.builder().bucket(BUCKET).build());
    assertThat(mockHttpClient.getLastRequest().getUri().getHost()).isEqualTo("s3.amazonaws.com");
}
 

@Test
public void regionalSettingDisabled_usesGlobalEndpoint() throws UnsupportedEncodingException {
    EnvironmentVariableHelper environmentVariableHelper = new EnvironmentVariableHelper();
    environmentVariableHelper.set(SdkSystemSetting.AWS_S3_US_EAST_1_REGIONAL_ENDPOINT.environmentVariable(), "nonregional");

    mockHttpClient.stubNextResponse(mockListObjectsResponse());

    S3Client s3Client = S3Client.builder()
            .credentialsProvider(StaticCredentialsProvider.create(AwsBasicCredentials.create("akid", "skid")))
            .httpClient(mockHttpClient)
            .region(Region.US_EAST_1)
            .serviceConfiguration(S3Configuration.builder()
                    .pathStyleAccessEnabled(true)
                    .build())
            .build();
    try {
        s3Client.listObjects(ListObjectsRequest.builder().bucket(BUCKET).build());
        assertThat(mockHttpClient.getLastRequest().getUri().getHost()).isEqualTo("s3.amazonaws.com");
    } finally {
        environmentVariableHelper.reset();
    }
}
 

@Test
public void regionalSettingEnabledViaProfile_usesRegionalIadEndpoint() throws UnsupportedEncodingException {
    String profile =
        "[profile test]\n" +
        "s3_us_east_1_regional_endpoint = regional";

    ProfileFile profileFile = ProfileFile.builder()
                                         .content(new StringInputStream(profile))
                                         .type(ProfileFile.Type.CONFIGURATION)
                                         .build();

    mockHttpClient.stubNextResponse(mockListObjectsResponse());

    S3Client s3Client = S3Client.builder()
                                .credentialsProvider(StaticCredentialsProvider.create(AwsBasicCredentials.create("akid", "skid")))
                                .httpClient(mockHttpClient)
                                .region(Region.US_EAST_1)
                                .overrideConfiguration(c -> c.defaultProfileFile(profileFile)
                                                             .defaultProfileName("test"))
                                .serviceConfiguration(c -> c.pathStyleAccessEnabled(true))
                                .build();

    s3Client.listObjects(ListObjectsRequest.builder().bucket(BUCKET).build());
    assertThat(mockHttpClient.getLastRequest().getUri().getHost()).isEqualTo("s3.us-east-1.amazonaws.com");
}
 

/**
 * S3 accelerate has a global endpoint, we use that when accelerate mode is enabled in the advanced configuration.
 */
@Test
public void accelerateEnabled_UsesVirtualAddressingWithAccelerateEndpoint() throws Exception {
    mockHttpClient.stubNextResponse(mockListObjectsResponse());
    S3Client s3Client = buildClient(withAccelerateEnabled());

    s3Client.listObjects(ListObjectsRequest.builder().bucket(BUCKET).build());

    assertEndpointMatches(mockHttpClient.getLastRequest(),
                          String.format("https://%s.s3-accelerate.amazonaws.com", BUCKET));
}
 

/**
 * By default we use virtual addressing when possible.
 */
@Test
public void noServiceConfigurationProvided_UsesVirtualAddressingWithStandardEndpoint() throws Exception {
    mockHttpClient.stubNextResponse(mockListObjectsResponse());
    S3Client s3Client = buildClient(null);

    s3Client.listObjects(ListObjectsRequest.builder().bucket(BUCKET).build());

    assertEndpointMatches(mockHttpClient.getLastRequest(), ENDPOINT_WITH_BUCKET);
}
 

/**
 * Dualstack uses regional endpoints that support virtual addressing.
 */
@Test
public void dualstackEnabled_UsesVirtualAddressingWithDualstackEndpoint() throws Exception {
    mockHttpClient.stubNextResponse(mockListObjectsResponse());
    S3Client s3Client = buildClient(withDualstackEnabled());

    s3Client.listObjects(ListObjectsRequest.builder().bucket(BUCKET).build());

    assertEndpointMatches(mockHttpClient.getLastRequest(),
                          String.format("https://%s.s3.dualstack.ap-south-1.amazonaws.com", BUCKET));
}
 

/**
 * Dualstack also supports path style endpoints just like the normal endpoints.
 */
@Test
public void dualstackAndPathStyleEnabled_UsesPathStyleAddressingWithDualstackEndpoint() throws Exception {
    mockHttpClient.stubNextResponse(mockListObjectsResponse());
    S3Client s3Client = buildClient(withDualstackAndPathStyleEnabled());

    s3Client.listObjects(ListObjectsRequest.builder().bucket(BUCKET).build());

    assertEndpointMatches(mockHttpClient.getLastRequest(), "https://s3.dualstack.ap-south-1.amazonaws.com/" + BUCKET);
}
 

/**
 * When dualstack and accelerate are both enabled there is a special, global dualstack endpoint we must use.
 */
@Test
public void dualstackAndAccelerateEnabled_UsesDualstackAccelerateEndpoint() throws Exception {
    mockHttpClient.stubNextResponse(mockListObjectsResponse());
    S3Client s3Client = buildClient(withDualstackAndAccelerateEnabled());

    s3Client.listObjects(ListObjectsRequest.builder().bucket(BUCKET).build());

    assertEndpointMatches(mockHttpClient.getLastRequest(),
                          String.format("https://%s.s3-accelerate.dualstack.amazonaws.com", BUCKET));
}
 

@Test
public void assertTestCase() throws UnsupportedEncodingException {
    final String bucket = testCaseModel.getBucket();
    final String expectedUri = testCaseModel.getExpectedUri();

    S3Client s3Client = constructClient(testCaseModel);

    mockHttpClient.stubNextResponse(mockListObjectsResponse());
    s3Client.listObjects(ListObjectsRequest.builder().bucket(bucket).build());

    assertThat(mockHttpClient.getLastRequest().getUri())
        .isEqualTo(URI.create(expectedUri));
}
 

public static void deleteBucketAndAllContents(String bucketName) {
    System.out.println("Deleting S3 bucket: " + bucketName);
    ListObjectsResponse response = s3.listObjects(ListObjectsRequest.builder().bucket(bucketName).build());

    while (true) {
        if (response.contents() == null) {
            break;
        }
        for (Iterator<?> iterator = response.contents().iterator(); iterator
                .hasNext(); ) {
            S3Object objectSummary = (S3Object) iterator.next();
            s3.deleteObject(DeleteObjectRequest.builder().bucket(bucketName).key(objectSummary.key()).build());
        }

        if (response.isTruncated()) {
            response = s3.listObjects(ListObjectsRequest.builder().marker(response.nextMarker()).build());
        } else {
            break;
        }
    }

    ListObjectVersionsResponse versionsResponse = s3
            .listObjectVersions(ListObjectVersionsRequest.builder().bucket(bucketName).build());
    if (versionsResponse.versions() != null) {
        for (ObjectVersion s : versionsResponse.versions()) {
            s3.deleteObject(DeleteObjectRequest.builder()
                                               .bucket(bucketName)
                                               .key(s.key())
                                               .versionId(s.versionId())
                                               .build());
        }
    }

    s3.deleteBucket(DeleteBucketRequest.builder().bucket(bucketName).build());
}
 

@Override
public synchronized void deleteAllBundleVersions(final BundleCoordinate bundleCoordinate) throws BundlePersistenceException {
    final String basePrefix = s3KeyPrefix == null ? "" : s3KeyPrefix + "/";
    final String bundlePrefix = getBundlePrefix(bundleCoordinate.getBucketId(), bundleCoordinate.getGroupId(), bundleCoordinate.getArtifactId());

    final String prefix = basePrefix + bundlePrefix;
    LOGGER.debug("Deleting all bundle versions from S3 bucket '{}' with prefix '{}'", new Object[]{s3BucketName, prefix});

    try {
        // List all the objects in the bucket with the given prefix of group/artifact...
        final ListObjectsResponse objectsResponse = s3Client.listObjects(
                ListObjectsRequest.builder()
                        .bucket(s3BucketName)
                        .prefix(prefix)
                        .build()
        );

        // Now delete each object, might be able to do this more efficiently with bulk delete
        for (final S3Object s3Object : objectsResponse.contents()) {
            final String s3ObjectKey = s3Object.key();
            s3Client.deleteObject(DeleteObjectRequest.builder()
                    .bucket(s3BucketName)
                    .key(s3ObjectKey)
                    .build()
            );
            LOGGER.debug("Successfully object from S3 bucket '{}' with key '{}'", new Object[]{s3BucketName, s3ObjectKey});
        }

        LOGGER.debug("Successfully deleted all bundle versions from S3 bucket '{}' with prefix '{}'", new Object[]{s3BucketName, prefix});
    } catch (Exception e) {
        throw new BundlePersistenceException("Error deleting bundle versions from S3 due to: " + e.getMessage(), e);
    }
}
 

@Override
public List<S3ObjectDescription> handle (TaskExecution aTask) throws Exception {
  
  S3Client s3 = S3Client.builder().build();
  
  ListObjectsResponse response = s3.listObjects(ListObjectsRequest.builder()
                                   .bucket(aTask.getRequiredString("bucket"))
                                   .prefix(aTask.getRequiredString("prefix"))
                                   .build());
  
  return response.contents()
                 .stream()
                 .map(o->new S3ObjectDescription(aTask.getRequiredString("bucket"),o))
                 .collect(Collectors.toList());
}
 

/**
 * By default we use virtual addressing when possible.
 */
@Test
public void emptyServiceConfigurationProvided_UsesVirtualAddressingWithStandardEndpoint() throws Exception {
    mockHttpClient.stubNextResponse(mockListObjectsResponse());
    S3Client s3Client = buildClient(S3Configuration.builder().build());

    s3Client.listObjects(ListObjectsRequest.builder().bucket(BUCKET).build());

    assertEndpointMatches(mockHttpClient.getLastRequest(), ENDPOINT_WITH_BUCKET);
}
 

private Artifact listFiles() {
    ListObjectsRequest req =
            ListObjectsRequest.builder()
                    .bucket(bucket)
                    .maxKeys(100)
                    .prefix(prefix)
                    .delimiter("/")
                    .build();

    ListObjectsResponse resp = client.listObjects(req);
    List<S3Object> list = resp.contents();
    if (list.isEmpty()) {
        return null;
    }

    Artifact artifact = new Artifact();
    artifact.setName(modelName);
    Map<String, Artifact.Item> files = new ConcurrentHashMap<>();
    for (S3Object obj : list) {
        Artifact.Item item = new Artifact.Item();
        String key = obj.key();
        if (!key.endsWith("/")) {
            item.setUri(key);
            item.setSize(obj.size());
            item.setArtifact(artifact);
            if ("dir".equals(item.getType())) {
                item.setName(""); // avoid creating extra folder
            }
            files.put(key, item);
        }
    }
    artifact.setFiles(files);
    return artifact;
}
 

/**
 * When path style is enabled in the advanced configuration we should always use it.
 */
@Test
public void pathStyleConfigured_UsesPathStyleAddressing() throws Exception {
    mockHttpClient.stubNextResponse(mockListObjectsResponse());
    S3Client s3Client = buildClient(withPathStyle());

    s3Client.listObjects(ListObjectsRequest.builder().bucket(BUCKET).build());

    assertEndpointMatches(mockHttpClient.getLastRequest(), ENDPOINT_WITHOUT_BUCKET + "/" + BUCKET);
}
 

/**
 * In us-east-1 buckets can have non-DNS compliant names. For those buckets we must always use path style even when it
 * is disabled per the advanced configuration.
 */
@Test
public void pathStyleDisabled_NonDnsCompatibleBucket_StillUsesPathStyleAddressing() throws Exception {
    mockHttpClient.stubNextResponse(mockListObjectsResponse());
    S3Client s3Client = buildClient(null);

    s3Client.listObjects(ListObjectsRequest.builder().bucket(NON_DNS_COMPATIBLE_BUCKET).build());

    assertEndpointMatches(mockHttpClient.getLastRequest(), ENDPOINT_WITHOUT_BUCKET + "/" + NON_DNS_COMPATIBLE_BUCKET);
}
 

/**
 * If customer is using HTTP we need to preserve that scheme when switching to virtual addressing.
 */
@Test
public void customHttpEndpoint_PreservesSchemeWhenSwitchingToVirtualAddressing() throws Exception {
    URI customEndpoint = URI.create("http://s3-external-1.amazonaws.com");
    mockHttpClient.stubNextResponse(mockListObjectsResponse());
    S3Client s3Client = clientBuilderWithMockSigner().endpointOverride(customEndpoint).build();

    s3Client.listObjects(ListObjectsRequest.builder().bucket(BUCKET).build());

    assertEndpointMatches(mockHttpClient.getLastRequest(),
                          String.format("http://%s.s3-external-1.amazonaws.com", BUCKET));
}
 

/**
 * If a custom S3 endpoint is provided (like s3-external-1 or a FIPS endpoint) then we should still use virtual addressing
 * when possible.
 */
@Test
public void customS3EndpointProvided_UsesVirtualAddressing() throws Exception {
    URI customEndpoint = URI.create("https://s3-external-1.amazonaws.com");
    mockHttpClient.stubNextResponse(mockListObjectsResponse());
    S3Client s3Client = clientBuilder().endpointOverride(customEndpoint).build();

    s3Client.listObjects(ListObjectsRequest.builder().bucket(BUCKET).build());

    assertEndpointMatches(mockHttpClient.getLastRequest(),
                          String.format("https://%s.s3-external-1.amazonaws.com", BUCKET));
}
 

/**
 * If a custom, non-s3 endpoint is used we revert to path style addressing. This is useful for alternative S3 implementations
 * like Ceph that do not support virtual style addressing.
 */
@Test
public void nonS3EndpointProvided_DoesNotUseVirtualAddressing() throws Exception {
    URI customEndpoint = URI.create("https://foobar.amazonaws.com");
    mockHttpClient.stubNextResponse(mockListObjectsResponse());
    S3Client s3Client = clientBuilder().endpointOverride(customEndpoint).build();

    s3Client.listObjects(ListObjectsRequest.builder().bucket(BUCKET).build());

    assertEndpointMatches(mockHttpClient.getLastRequest(), customEndpoint.toString() + "/" + BUCKET);
}
 

@Test
public void accessPointArn_differentRegion_useArnRegionTrue() throws Exception {
    URI customEndpoint = URI.create("https://foobar-12345678910.s3-accesspoint.us-west-2.amazonaws.com");
    mockHttpClient.stubNextResponse(mockListObjectsResponse());
    S3Client s3Client = clientBuilder().serviceConfiguration(b -> b.useArnRegionEnabled(true)).build();
    String accessPointArn = "arn:aws:s3:us-west-2:12345678910:accesspoint:foobar";

    s3Client.listObjects(ListObjectsRequest.builder().bucket(accessPointArn).build());

    assertEndpointMatches(mockHttpClient.getLastRequest(), customEndpoint.toString());
}
 

@Test
public void accessPointArn_differentRegion_useArnRegionFalse_throwsIllegalArgumentException() throws Exception {
    mockHttpClient.stubNextResponse(mockListObjectsResponse());
    S3Client s3Client = clientBuilder().build();
    String accessPointArn = "arn:aws:s3:us-west-2:12345678910:accesspoint:foobar";

    assertThatThrownBy(() -> s3Client.listObjects(ListObjectsRequest.builder().bucket(accessPointArn).build()))
        .isInstanceOf(IllegalArgumentException.class)
        .hasMessageContaining("region");
}
 

@Test
public void accessPointArn_customEndpoint_throwsIllegalArgumentException() throws Exception {
    URI customEndpoint = URI.create("https://foobar.amazonaws.com");
    mockHttpClient.stubNextResponse(mockListObjectsResponse());
    S3Client s3Client = clientBuilder().endpointOverride(customEndpoint).build();
    String accessPointArn = "arn:aws:s3:ap-south-1:12345678910:accesspoint:foobar";

    assertThatThrownBy(() -> s3Client.listObjects(ListObjectsRequest.builder().bucket(accessPointArn).build()))
        .isInstanceOf(IllegalArgumentException.class)
        .hasMessageContaining("endpoint override");
}
 

@Test
public void accessPointArn_correctlyRewritesEndpoint() throws Exception {
    URI customEndpoint = URI.create("https://foobar-12345678910.s3-accesspoint.ap-south-1.amazonaws.com");
    mockHttpClient.stubNextResponse(mockListObjectsResponse());
    S3Client s3Client = clientBuilder().build();
    String accessPointArn = "arn:aws:s3:ap-south-1:12345678910:accesspoint:foobar";

    s3Client.listObjects(ListObjectsRequest.builder().bucket(accessPointArn).build());

    assertEndpointMatches(mockHttpClient.getLastRequest(), customEndpoint.toString());
}
 

@Test
public void listObjectsWithMaxKeys() {
    int maxKeys = 4;
    ListObjectsResponse result = s3.listObjects(ListObjectsRequest.builder()
                                                                  .bucket(bucketName)
                                                                  .maxKeys(maxKeys)
                                                                  .build());

    List<S3Object> objects = result.contents();

    assertEquals(maxKeys, objects.size());
    assertEquals(bucketName, result.name());
    assertThat(maxKeys).isEqualTo(result.maxKeys());
    assertS3ObjectSummariesAreValid(objects);

    // We didn't use a delimiter, so we expect this to be empty/null
    assertNull(result.delimiter());

    // We expect truncated results since we set maxKeys
    assertTrue(result.isTruncated());

    // URL encoding is requested by default

    // We didn't set other request parameters, so we expect them to be empty
    assertNull(result.encodingType());
    assertThat(result.prefix()).isEmpty();
    assertNull(result.delimiter());
}
 

@GET
@Produces(MediaType.APPLICATION_JSON)
public Uni<List<FileObject>> listFiles() {
    ListObjectsRequest listRequest = ListObjectsRequest.builder()
            .bucket(bucketName)
            .build();

    return Uni.createFrom().completionStage(() -> s3.listObjects(listRequest))
            .onItem().apply(result -> toFileItems(result));
}