software.amazon.awssdk.services.s3.model.ListObjectsV2Response Java Examples

@Test
public void testDeleteFolder()
    throws Exception {
  String[] originalFiles = new String[]{"a-delete-2.txt", "b-delete-2.txt", "c-delete-2.txt"};
  String folderName = "my-files";

  for (String fileName : originalFiles) {
    createEmptyFile(folderName, fileName);
  }

  _s3PinotFS.delete(URI.create(String.format(FILE_FORMAT, SCHEME, BUCKET, folderName)), true);

  ListObjectsV2Response listObjectsV2Response =
      _s3Client.listObjectsV2(S3TestUtils.getListObjectRequest(BUCKET, "", true));
  String[] actualResponse =
      listObjectsV2Response.contents().stream().map(S3Object::key).filter(x -> x.contains("delete-2"))
          .toArray(String[]::new);

  Assert.assertEquals(0, actualResponse.length);
}
 

@Test
public void testDeleteFile()
    throws Exception {
  String[] originalFiles = new String[]{"a-delete.txt", "b-delete.txt", "c-delete.txt"};
  String fileToDelete = "a-delete.txt";

  List<String> expectedResultList = new ArrayList<>();
  for (String fileName : originalFiles) {
    createEmptyFile("", fileName);
    if (!fileName.equals(fileToDelete)) {
      expectedResultList.add(fileName);
    }
  }

  _s3PinotFS.delete(URI.create(String.format(FILE_FORMAT, SCHEME, BUCKET, fileToDelete)), false);

  ListObjectsV2Response listObjectsV2Response =
      _s3Client.listObjectsV2(S3TestUtils.getListObjectRequest(BUCKET, "", true));
  String[] actualResponse =
      listObjectsV2Response.contents().stream().map(x -> x.key().substring(1)).filter(x -> x.contains("delete"))
          .toArray(String[]::new);

  Assert.assertEquals(actualResponse.length, 2);
  Assert.assertTrue(Arrays.equals(actualResponse, expectedResultList.toArray()));
}
 

@Test
public void testTouchFilesInFolder()
    throws Exception {

  String folder = "my-files";
  String[] originalFiles = new String[]{"a-touch.txt", "b-touch.txt", "c-touch.txt"};

  for (String fileName : originalFiles) {
    String fileNameWithFolder = folder + DELIMITER + fileName;
    _s3PinotFS.touch(URI.create(String.format(FILE_FORMAT, SCHEME, BUCKET, fileNameWithFolder)));
  }
  ListObjectsV2Response listObjectsV2Response =
      _s3Client.listObjectsV2(S3TestUtils.getListObjectRequest(BUCKET, folder, false));

  String[] response = listObjectsV2Response.contents().stream().map(S3Object::key).filter(x -> x.contains("touch"))
      .toArray(String[]::new);
  Assert.assertEquals(response.length, originalFiles.length);

  Assert.assertTrue(Arrays.equals(response, Arrays.stream(originalFiles).map(x -> folder + DELIMITER + x).toArray()));
}
 

@Test
public void testListNoParameters() {
    ListObjectsV2Response result = s3.listObjectsV2(ListObjectsV2Request.builder().bucket(bucketName).build());
    List<S3Object> objects = result.contents();

    assertEquals(keys.size(), objects.size());
    assertThat(keys.size(), equalTo(result.keyCount()));
    assertEquals(bucketName, result.name());
    assertS3ObjectSummariesAreValid(objects, false);
    assertNotNull(result.maxKeys());

    // We didn't use a delimiter, so we expect these to be empty/null
    assertNull(result.delimiter());

    // We don't expect any truncated results
    assertFalse(result.isTruncated());
    assertNull(result.nextContinuationToken());

    // We didn't set other request parameters, so we expect them to be empty
    assertNull(result.encodingType());
    assertThat(result.prefix(), equalTo(""));
    assertNull(result.continuationToken());
}
 

@Test
public void testTouchFileInBucket()
    throws Exception {

  String[] originalFiles = new String[]{"a-touch.txt", "b-touch.txt", "c-touch.txt"};

  for (String fileName : originalFiles) {
    _s3PinotFS.touch(URI.create(String.format(FILE_FORMAT, SCHEME, BUCKET, fileName)));
  }
  ListObjectsV2Response listObjectsV2Response =
      _s3Client.listObjectsV2(S3TestUtils.getListObjectRequest(BUCKET, "", true));

  String[] response = listObjectsV2Response.contents().stream().map(S3Object::key).filter(x -> x.contains("touch"))
      .toArray(String[]::new);

  Assert.assertEquals(response.length, originalFiles.length);
  Assert.assertTrue(Arrays.equals(response, originalFiles));
}
 

@Test
public void testListWithEncodingType() {
    String encodingType = "url";
    ListObjectsV2Response result = s3.listObjectsV2(ListObjectsV2Request.builder()
                                                                        .bucket(bucketName)
                                                                        .prefix(KEY_NAME_WITH_SPECIAL_CHARS)
                                                                        .encodingType(encodingType)
                                                                        .build());
    List<S3Object> objects = result.contents();

    // EncodingType should be returned in the response.
    assertEquals(encodingType, result.encodingTypeAsString());

    System.out.println(result.contents().get(0).key());

    // The key name returned in the response should have been decoded
    // from the URL encoded form S3 returned us.
    assertEquals(KEY_NAME_WITH_SPECIAL_CHARS,
                 objects.get(0).key());
}
 

@Override
public SdkResponse modifyResponse(Context.ModifyResponse context,
                                  ExecutionAttributes executionAttributes) {
    SdkResponse response = context.response();
    if (shouldHandle(response)) {
        if (response instanceof ListObjectsResponse) {
            return modifyListObjectsResponse((ListObjectsResponse) response);
        }

        if (response instanceof ListObjectsV2Response) {
            return modifyListObjectsV2Response((ListObjectsV2Response) response);
        }

        if (response instanceof ListObjectVersionsResponse) {
            return modifyListObjectVersionsResponse((ListObjectVersionsResponse) response);
        }

        if (response instanceof ListMultipartUploadsResponse) {
            return modifyListMultipartUploadsResponse((ListMultipartUploadsResponse) response);
        }
    }
    return response;
}
 

@Test
public void testListPagination() {
    int firstRequestMaxKeys = 4;
    String prefix = "key";
    ListObjectsV2Response result = s3.listObjectsV2(ListObjectsV2Request.builder()
                                                                        .bucket(bucketName)
                                                                        .prefix(prefix)
                                                                        .maxKeys(firstRequestMaxKeys)
                                                                        .build());
    List<S3Object> objects = result.contents();

    assertEquals(firstRequestMaxKeys, objects.size());
    assertEquals(bucketName, result.name());
    assertEquals(prefix, result.prefix());
    assertNotNull(result.nextContinuationToken());
    assertTrue(result.isTruncated());
    assertS3ObjectSummariesAreValid(objects, false);

    for (int i = 0; i < firstRequestMaxKeys; i++) {
        assertEquals(keys.get(i), objects.get(i).key());
    }

    ListObjectsV2Response nextResults = s3.listObjectsV2(ListObjectsV2Request.builder()
                                                                             .bucket(bucketName)
                                                                             .prefix(prefix)
                                                                             .continuationToken(
                                                                                     result.nextContinuationToken())
                                                                             .build());
    List<S3Object> nextObjects = nextResults.contents();

    assertNull(nextResults.nextContinuationToken());
    assertEquals(nextResults.continuationToken(), result.nextContinuationToken());
    assertFalse(nextResults.isTruncated());
    assertEquals(prefix, nextResults.prefix());
    assertS3ObjectSummariesAreValid(nextObjects, false);
    assertEquals(nextObjects.size(), BUCKET_OBJECTS - firstRequestMaxKeys);
}
 

/**
 * Delete a S3 bucket with the given name
 *
 * @param name
 * @return
 */
public boolean deleteS3Bucket(String name) {
    boolean deleted = true;
    ListObjectsV2Request listObjectsV2Request =
            ListObjectsV2Request.builder().bucket(name).build();
    ListObjectsV2Response listObjectsV2Response;
    S3Client s3Client = S3Client.builder().region(mAWSRegion).build();
    do {
        listObjectsV2Response = s3Client.listObjectsV2(listObjectsV2Request);
        if (listObjectsV2Response.contents() != null) {
            // detelete the files in the bucket
            for (S3Object s3Object : listObjectsV2Response.contents()) {
                mLogger.debug("Deleteing file " + s3Object.key() + " from bucket " + name);
                s3Client.deleteObject(
                        DeleteObjectRequest.builder().bucket(name).key(s3Object.key()).build());
            }
        }

        listObjectsV2Request =
                ListObjectsV2Request.builder()
                        .bucket(name)
                        .continuationToken(listObjectsV2Response.nextContinuationToken())
                        .build();

    } while (listObjectsV2Response.isTruncated());

    // Delete empty bucket
    DeleteBucketRequest deleteBucketRequest =
            DeleteBucketRequest.builder().bucket(name).build();
    s3Client.deleteBucket(deleteBucketRequest);
    return deleted;
}
 

@Override
public String[] listFiles(URI fileUri, boolean recursive)
    throws IOException {
  try {
    ImmutableList.Builder<String> builder = ImmutableList.builder();
    String prefix = normalizeToDirectoryPrefix(fileUri);

    ListObjectsV2Response listObjectsV2Response;
    ListObjectsV2Request.Builder listObjectsV2RequestBuilder =
        ListObjectsV2Request.builder().bucket(fileUri.getHost());

    if (!prefix.equals(DELIMITER)) {
      listObjectsV2RequestBuilder = listObjectsV2RequestBuilder.prefix(prefix);
    }

    if (!recursive) {
      listObjectsV2RequestBuilder = listObjectsV2RequestBuilder.delimiter(DELIMITER);
    }

    ListObjectsV2Request listObjectsV2Request = listObjectsV2RequestBuilder.build();
    listObjectsV2Response = _s3Client.listObjectsV2(listObjectsV2Request);

    listObjectsV2Response.contents().stream().forEach(object -> {
      //Only add files and not directories
      if (!object.key().equals(fileUri.getPath()) && !object.key().endsWith(DELIMITER)) {
        builder.add(object.key());
      }
    });
    return builder.build().toArray(new String[0]);
  } catch (Throwable t) {
    throw new IOException(t);
  }
}
 

private boolean isEmptyDirectory(URI uri)
    throws IOException {
  if (!isDirectory(uri)) {
    return false;
  }
  String prefix = normalizeToDirectoryPrefix(uri);
  boolean isEmpty = true;
  ListObjectsV2Response listObjectsV2Response;
  ListObjectsV2Request.Builder listObjectsV2RequestBuilder = ListObjectsV2Request.builder().bucket(uri.getHost());

  if (!prefix.equals(DELIMITER)) {
    listObjectsV2RequestBuilder = listObjectsV2RequestBuilder.prefix(prefix);
  }

  ListObjectsV2Request listObjectsV2Request = listObjectsV2RequestBuilder.build();
  listObjectsV2Response = _s3Client.listObjectsV2(listObjectsV2Request);

  for (S3Object s3Object : listObjectsV2Response.contents()) {
    if (s3Object.key().equals(prefix)) {
      continue;
    } else {
      isEmpty = false;
      break;
    }
  }
  return isEmpty;
}
 

@Test
public void encodingTypeNotSet_doesNotDecodeListObjectsV2ResponseParts() {
    ListObjectsV2Response original = V2_TEST_ENCODED_RESPONSE.toBuilder()
            .encodingType((String) null)
            .build();

    Context.ModifyResponse ctx = newContext(original);

    ListObjectsV2Response fromInterceptor = (ListObjectsV2Response) INTERCEPTOR.modifyResponse(ctx, new ExecutionAttributes());

    assertThat(fromInterceptor).isEqualTo(original);
}
 

@Test
public void encodingTypeSet_decodesListObjectsV2ResponseParts() {
    Context.ModifyResponse ctx = newContext(V2_TEST_ENCODED_RESPONSE);

    ListObjectsV2Response decoded = (ListObjectsV2Response) INTERCEPTOR.modifyResponse(ctx, new ExecutionAttributes());

    assertDecoded(decoded::delimiter, " delimiter");
    assertDecoded(decoded::prefix, " prefix");
    assertDecoded(decoded::startAfter, " startafter");
    assertKeysAreDecoded(decoded.contents());
    assertCommonPrefixesAreDecoded(decoded.commonPrefixes());
}
 

@Test
public void listObjectV2WithUrlEncodingType_shouldDecode() {
    ListObjectsV2Response listObjectsV2Response =
        s3.listObjectsV2(b -> b.bucket(BUCKET_NAME).encodingType(EncodingType.URL));

    listObjectsV2Response.contents().forEach(c -> assertKeyIsDecoded(c.key()));
    ListObjectVersionsResponse asyncResponse =
        s3Async.listObjectVersions(b -> b.bucket(BUCKET_NAME).encodingType(EncodingType.URL)).join();

    asyncResponse.versions().forEach(v -> assertKeyIsDecoded(v.key()));
}
 

@Test
public void testListWithFetchOwner() {
    ListObjectsV2Response result = s3.listObjectsV2(ListObjectsV2Request.builder()
                                                                        .bucket(bucketName)
                                                                        .fetchOwner(true)
                                                                        .build());
    List<S3Object> objects = result.contents();
    assertS3ObjectSummariesAreValid(objects, true);
}
 

@Test
public void testListWithMaxKeys() {
    int maxKeys = 4;
    ListObjectsV2Response result = s3.listObjectsV2(ListObjectsV2Request.builder()
                                                                        .bucket(bucketName)
                                                                        .maxKeys(maxKeys)
                                                                        .build());

    List<S3Object> objects = result.contents();

    assertEquals(maxKeys, objects.size());
    assertEquals(bucketName, result.name());
    assertThat(maxKeys, equalTo(result.maxKeys()));
    assertS3ObjectSummariesAreValid(objects, false);

    // We didn't use a delimiter, so we expect this to be empty/null
    assertNull(result.delimiter());

    // We expect truncated results since we set maxKeys
    assertTrue(result.isTruncated());
    assertNotNull(result.nextContinuationToken());
    assertTrue(result.nextContinuationToken().length() > 1);

    // URL encoding is requested by default

    // We didn't set other request parameters, so we expect them to be empty
    assertNull(result.encodingType());
    assertThat(result.prefix(), isEmptyString());
    assertNull(result.startAfter());
    assertNull(result.delimiter());
}
 

@Test
public void testListWithPrefixAndDelimiter() {
    String prefix = "a";
    String delimiter = "/";
    ListObjectsV2Response result = s3.listObjectsV2(ListObjectsV2Request.builder()
                                                                        .bucket(bucketName)
                                                                        .prefix(prefix)
                                                                        .delimiter(delimiter)
                                                                        .build());

    List<S3Object> objects = result.contents();

    assertEquals(1, objects.size());
    assertEquals(bucketName, result.name());
    assertS3ObjectSummariesAreValid(objects, false);
    assertEquals(prefix, result.prefix());
    assertEquals(delimiter, result.delimiter());

    // We don't expect any truncated results
    assertFalse(result.isTruncated());
    assertNull(result.nextContinuationToken());

    // We didn't set other request parameters, so we expect them to be empty
    assertNull(result.startAfter());
    assertNull(result.encodingType());
    assertTrue(result.maxKeys() >= 1000);
}
 

@Test
public void testListWithPrefixAndStartAfter() {
    String prefix = "key";
    String startAfter = "key-01";
    ListObjectsV2Response result = s3.listObjectsV2(ListObjectsV2Request.builder()
                                                                        .bucket(bucketName)
                                                                        .prefix(prefix)
                                                                        .startAfter(startAfter)
                                                                        .build());
    List<S3Object> objects = result.contents();

    assertEquals(BUCKET_OBJECTS - 1, objects.size());
    assertEquals(bucketName, result.name());
    assertS3ObjectSummariesAreValid(objects, false);
    assertEquals(startAfter, result.startAfter());
    assertEquals(prefix, result.prefix());

    // We didn't use a delimiter, so we expect it to be empty/null
    assertNull(result.delimiter());

    // We don't expect any truncated results
    assertFalse(result.isTruncated());
    assertNull(result.nextContinuationToken());

    // We didn't set any other request parameters, so we expect them to be
    // set to the defaults.
    assertTrue(result.maxKeys() >= 1000);
    assertNull(result.encodingType());
}
 

@Test
public void test_AsyncResponse_UsingRxJava() {
    ListObjectsV2Publisher publisher = s3Async.listObjectsV2Paginator(requestBuilder.bucket(bucketName).build());

    Single<List<S3Object>> objects = Flowable.fromPublisher(publisher)
                                             .flatMapIterable(ListObjectsV2Response::contents)
                                             .toList();

    // There are multiple fluent methods to convert Single type to a different form
    List<S3Object> objectList = objects.blockingGet();
    assertThat(Long.valueOf(objectList.size()), equalTo(OBJECT_COUNT));
}
 

private static SdkResponse modifyListObjectsV2Response(ListObjectsV2Response response) {
    return response.toBuilder()
                   .delimiter(urlDecode(response.delimiter()))
                   .prefix(urlDecode(response.prefix()))
                   .startAfter(urlDecode(response.startAfter()))
                   .contents(decodeContents(response.contents()))
                   .commonPrefixes(decodeCommonPrefixes(response.commonPrefixes()))
                   .build();
}
 

public static void main(String[] args) throws Exception {

        final String USAGE = "\n" +
                "Usage:\n" +
                "    S3BucketDeletion <bucket>\n\n" +
                "Where:\n" +
                "    bucket - the bucket to delete  (i.e., bucket1)\n\n" +
                "Example:\n" +
                "    bucket1\n\n";

        if (args.length < 1) {
            System.out.println(USAGE);
            System.exit(1);
        }
        String bucket = args[0];

        //Create the S3Client object
        Region region = Region.US_WEST_2;
        s3 = S3Client.builder().region(region).build();

        try {
            // snippet-start:[s3.java2.s3_bucket_ops.delete_bucket]
            // To delete a bucket, all the objects in the bucket must be deleted first
            ListObjectsV2Request listObjectsV2Request = ListObjectsV2Request.builder().bucket(bucket).build();
            ListObjectsV2Response listObjectsV2Response;

            do {
                listObjectsV2Response = s3.listObjectsV2(listObjectsV2Request);
                for (S3Object s3Object : listObjectsV2Response.contents()) {
                    s3.deleteObject(DeleteObjectRequest.builder().bucket(bucket).key(s3Object.key()).build());
                }

                listObjectsV2Request = ListObjectsV2Request.builder().bucket(bucket)
                    .continuationToken(listObjectsV2Response.nextContinuationToken())
                    .build();

            } while(listObjectsV2Response.isTruncated());
            // snippet-end:[s3.java2.s3_bucket_ops.delete_bucket]

          DeleteBucketRequest deleteBucketRequest = DeleteBucketRequest.builder().bucket(bucket).build();
          s3.deleteBucket(deleteBucketRequest);

        } catch (S3Exception e) {
            System.err.println(e.awsErrorDetails().errorMessage());
            System.exit(1);
        }
    }
 

public ListObjectsV2Response getLastPage() {
    return lastPage;
}
 

@Override
public void onNext(ListObjectsV2Response response) {
    lastPage = response;
    keyCount += response.keyCount();
    requestsCompleted++;
}