software.amazon.awssdk.services.s3.model.S3Object Java Examples

/**
 * Asserts that a list of S3Object objects are valid, by checking
 * that expected fields are not null or empty, that ETag values don't
 * contain leading or trailing quotes, that the last modified date is
 * recent, etc.
 *
 * @param objectSummaries    The list of objects to validate.
 * @param shouldIncludeOwner Whether owner information was requested and should be present in results.
 */

private void assertS3ObjectSummariesAreValid(List<S3Object> objectSummaries,
                                             boolean shouldIncludeOwner) {

    for (java.util.Iterator iterator = objectSummaries.iterator(); iterator.hasNext(); ) {
        S3Object obj = (S3Object) iterator.next();
        assertTrue(obj.eTag().length() > 1);
        assertTrue(obj.key().length() > 1);

        // Verify that the last modified date is within an hour
        assertNotNull(obj.lastModified());
        long offset = obj.lastModified().toEpochMilli() - Instant.now().toEpochMilli();
        assertTrue(offset < ONE_HOUR_IN_MILLISECONDS);

        assertTrue(obj.storageClassAsString().length() > 1);

        if (shouldIncludeOwner) {
            assertNotNull(obj.owner());
            assertTrue(obj.owner().displayName().length() > 1);
            assertTrue(obj.owner().id().length() > 1);
        }
    }
}
 

@Test
public void testDeleteFolder()
    throws Exception {
  String[] originalFiles = new String[]{"a-delete-2.txt", "b-delete-2.txt", "c-delete-2.txt"};
  String folderName = "my-files";

  for (String fileName : originalFiles) {
    createEmptyFile(folderName, fileName);
  }

  _s3PinotFS.delete(URI.create(String.format(FILE_FORMAT, SCHEME, BUCKET, folderName)), true);

  ListObjectsV2Response listObjectsV2Response =
      _s3Client.listObjectsV2(S3TestUtils.getListObjectRequest(BUCKET, "", true));
  String[] actualResponse =
      listObjectsV2Response.contents().stream().map(S3Object::key).filter(x -> x.contains("delete-2"))
          .toArray(String[]::new);

  Assert.assertEquals(0, actualResponse.length);
}
 

@Test
public void testTouchFilesInFolder()
    throws Exception {

  String folder = "my-files";
  String[] originalFiles = new String[]{"a-touch.txt", "b-touch.txt", "c-touch.txt"};

  for (String fileName : originalFiles) {
    String fileNameWithFolder = folder + DELIMITER + fileName;
    _s3PinotFS.touch(URI.create(String.format(FILE_FORMAT, SCHEME, BUCKET, fileNameWithFolder)));
  }
  ListObjectsV2Response listObjectsV2Response =
      _s3Client.listObjectsV2(S3TestUtils.getListObjectRequest(BUCKET, folder, false));

  String[] response = listObjectsV2Response.contents().stream().map(S3Object::key).filter(x -> x.contains("touch"))
      .toArray(String[]::new);
  Assert.assertEquals(response.length, originalFiles.length);

  Assert.assertTrue(Arrays.equals(response, Arrays.stream(originalFiles).map(x -> folder + DELIMITER + x).toArray()));
}
 

@Test
public void testTouchFileInBucket()
    throws Exception {

  String[] originalFiles = new String[]{"a-touch.txt", "b-touch.txt", "c-touch.txt"};

  for (String fileName : originalFiles) {
    _s3PinotFS.touch(URI.create(String.format(FILE_FORMAT, SCHEME, BUCKET, fileName)));
  }
  ListObjectsV2Response listObjectsV2Response =
      _s3Client.listObjectsV2(S3TestUtils.getListObjectRequest(BUCKET, "", true));

  String[] response = listObjectsV2Response.contents().stream().map(S3Object::key).filter(x -> x.contains("touch"))
      .toArray(String[]::new);

  Assert.assertEquals(response.length, originalFiles.length);
  Assert.assertTrue(Arrays.equals(response, originalFiles));
}
 

public static void listBucketObjects(S3Client s3, String bucketName ) {

       try {
            ListObjectsRequest listObjects = ListObjectsRequest
                    .builder()
                    .bucket(bucketName)
                    .build();

            ListObjectsResponse res = s3.listObjects(listObjects);
            List<S3Object> objects = res.contents();

            for (ListIterator iterVals = objects.listIterator(); iterVals.hasNext(); ) {
                S3Object myValue = (S3Object) iterVals.next();
                System.out.print("\n The name of the key is " + myValue.key());
                System.out.print("\n The object is " + calKb(myValue.size()) + " KBs");
                System.out.print("\n The owner is " + myValue.owner());
                }
        } catch (S3Exception e) {
            System.err.println(e.awsErrorDetails().errorMessage());
            System.exit(1);
        }
    }
 

@Test
public void listObjectsNoParameters() {
    ListObjectsResponse result = s3.listObjects(ListObjectsRequest.builder().bucket(bucketName).build());
    List<S3Object> objects = result.contents();

    assertEquals(keys.size(), objects.size());
    assertEquals(bucketName, result.name());
    assertS3ObjectSummariesAreValid(objects);
    assertNotNull(result.maxKeys());

    // We didn't use a delimiter, so we expect these to be empty/null
    assertNull(result.delimiter());

    // We don't expect any truncated results
    assertFalse(result.isTruncated());

    // We didn't set other request parameters, so we expect them to be empty
    assertNull(result.encodingType());
    assertThat(result.prefix()).isEmpty();
}
 

@Test
public void listObjectsWithAllElements() {
    String delimiter = "/";
    String marker = "aaa";
    ListObjectsResponse result = s3.listObjects(ListObjectsRequest.builder()
                                                                  .bucket(bucketName)
                                                                  .prefix(KEY_NAME_WITH_SPECIAL_CHARS)
                                                                  .marker(marker)
                                                                  .encodingType(EncodingType.URL)
                                                                  .delimiter(delimiter)
                                                                  .build());
    List<S3Object> objects = result.contents();

    assertEquals(bucketName, result.name());
    assertS3ObjectSummariesAreValid(objects);
    assertEquals(marker, result.marker());
    assertEquals(delimiter, result.delimiter());
    assertEquals(KEY_NAME_WITH_SPECIAL_CHARS, result.prefix());

    assertFalse(result.isTruncated());
    assertTrue(result.maxKeys() >= 1000);
}
 

@Test
public void testListWithEncodingType() {
    String encodingType = "url";
    ListObjectsV2Response result = s3.listObjectsV2(ListObjectsV2Request.builder()
                                                                        .bucket(bucketName)
                                                                        .prefix(KEY_NAME_WITH_SPECIAL_CHARS)
                                                                        .encodingType(encodingType)
                                                                        .build());
    List<S3Object> objects = result.contents();

    // EncodingType should be returned in the response.
    assertEquals(encodingType, result.encodingTypeAsString());

    System.out.println(result.contents().get(0).key());

    // The key name returned in the response should have been decoded
    // from the URL encoded form S3 returned us.
    assertEquals(KEY_NAME_WITH_SPECIAL_CHARS,
                 objects.get(0).key());
}
 

@Test
public void shouldDownloadLatestSnapshotFileFromBucket() throws Exception {
    //given
    s3Helper.upload(S3_UTILS_TEST_BUCKET, createTempFile("compaction-test-snapshot-", ".json.zip").toFile());

    waitSoThatNextSnapshotHasDifferentModificationDate();

    File latest = createTempFile("compaction-test-snapshot-", ".json.zip").toFile();
    s3Helper.upload(S3_UTILS_TEST_BUCKET, latest);

    //when
    Optional<S3Object> s3Object = snapshotService.fetchSnapshotMetadataFromS3(S3_UTILS_TEST_BUCKET, "test");

    //then
    assertThat(s3Object.get().key(), is(latest.getName()));
}
 

@Test
public void testListNoParameters() {
    ListObjectsV2Response result = s3.listObjectsV2(ListObjectsV2Request.builder().bucket(bucketName).build());
    List<S3Object> objects = result.contents();

    assertEquals(keys.size(), objects.size());
    assertThat(keys.size(), equalTo(result.keyCount()));
    assertEquals(bucketName, result.name());
    assertS3ObjectSummariesAreValid(objects, false);
    assertNotNull(result.maxKeys());

    // We didn't use a delimiter, so we expect these to be empty/null
    assertNull(result.delimiter());

    // We don't expect any truncated results
    assertFalse(result.isTruncated());
    assertNull(result.nextContinuationToken());

    // We didn't set other request parameters, so we expect them to be empty
    assertNull(result.encodingType());
    assertThat(result.prefix(), equalTo(""));
    assertNull(result.continuationToken());
}
 

Optional<File> getLatestSnapshot(final String channelName) {
    Optional<S3Object> s3Object = fetchSnapshotMetadataFromS3(snapshotBucketName, channelName);
    if (s3Object.isPresent()) {
        String latestSnapshotKey = s3Object.get().key();
        Path snapshotFile = getTempFile(latestSnapshotKey);

        if (existsAndHasSize(snapshotFile, s3Object.get().size())) {
            LOG.info(marker, "Locally available snapshot file is the same as in S3, skip download and use it: {}", snapshotFile.toAbsolutePath().toString());
            return Optional.of(snapshotFile.toFile());
        }

        removeTempFiles(String.format("*-%s-snapshot-*.json.zip", channelName));
        LOG.info(marker, "Downloading snapshot file to {}", snapshotFile.getFileName().toAbsolutePath().toString());
        if (s3Helper.download(snapshotBucketName, latestSnapshotKey, snapshotFile)) {
            return Optional.of(snapshotFile.toFile());
        }
        return Optional.empty();
    } else {
        return Optional.empty();
    }
}
 

@Test
public void shouldReturnOptionalEmptyWhenNoFileInBucket() {
    //when
    Optional<S3Object> s3Object = snapshotService.fetchSnapshotMetadataFromS3(S3_UTILS_TEST_BUCKET, "DOES_NOT_EXIST");

    //then
    assertThat(s3Object.isPresent(), is(false));
}
 

public static void deleteBucketAndAllContents(String bucketName) {
    System.out.println("Deleting S3 bucket: " + bucketName);
    ListObjectsResponse response = s3.listObjects(ListObjectsRequest.builder().bucket(bucketName).build());

    while (true) {
        if (response.contents() == null) {
            break;
        }
        for (Iterator<?> iterator = response.contents().iterator(); iterator
                .hasNext(); ) {
            S3Object objectSummary = (S3Object) iterator.next();
            s3.deleteObject(DeleteObjectRequest.builder().bucket(bucketName).key(objectSummary.key()).build());
        }

        if (response.isTruncated()) {
            response = s3.listObjects(ListObjectsRequest.builder().marker(response.nextMarker()).build());
        } else {
            break;
        }
    }

    ListObjectVersionsResponse versionsResponse = s3
            .listObjectVersions(ListObjectVersionsRequest.builder().bucket(bucketName).build());
    if (versionsResponse.versions() != null) {
        for (ObjectVersion s : versionsResponse.versions()) {
            s3.deleteObject(DeleteObjectRequest.builder()
                                               .bucket(bucketName)
                                               .key(s.key())
                                               .versionId(s.versionId())
                                               .build());
        }
    }

    s3.deleteBucket(DeleteBucketRequest.builder().bucket(bucketName).build());
}
 

@Override
public synchronized void deleteAllBundleVersions(final BundleCoordinate bundleCoordinate) throws BundlePersistenceException {
    final String basePrefix = s3KeyPrefix == null ? "" : s3KeyPrefix + "/";
    final String bundlePrefix = getBundlePrefix(bundleCoordinate.getBucketId(), bundleCoordinate.getGroupId(), bundleCoordinate.getArtifactId());

    final String prefix = basePrefix + bundlePrefix;
    LOGGER.debug("Deleting all bundle versions from S3 bucket '{}' with prefix '{}'", new Object[]{s3BucketName, prefix});

    try {
        // List all the objects in the bucket with the given prefix of group/artifact...
        final ListObjectsResponse objectsResponse = s3Client.listObjects(
                ListObjectsRequest.builder()
                        .bucket(s3BucketName)
                        .prefix(prefix)
                        .build()
        );

        // Now delete each object, might be able to do this more efficiently with bulk delete
        for (final S3Object s3Object : objectsResponse.contents()) {
            final String s3ObjectKey = s3Object.key();
            s3Client.deleteObject(DeleteObjectRequest.builder()
                    .bucket(s3BucketName)
                    .key(s3ObjectKey)
                    .build()
            );
            LOGGER.debug("Successfully object from S3 bucket '{}' with key '{}'", new Object[]{s3BucketName, s3ObjectKey});
        }

        LOGGER.debug("Successfully deleted all bundle versions from S3 bucket '{}' with prefix '{}'", new Object[]{s3BucketName, prefix});
    } catch (Exception e) {
        throw new BundlePersistenceException("Error deleting bundle versions from S3 due to: " + e.getMessage(), e);
    }
}
 

Optional<S3Object> fetchSnapshotMetadataFromS3(String bucketName, String channelName) {
    return s3Helper.listAll(bucketName)
            .stream()
            .filter(o -> o.key().startsWith(getSnapshotFileNamePrefix(channelName)))
            .filter(o -> o.key().endsWith(COMPACTION_FILE_EXTENSION))
            .min(comparing(S3Object::lastModified, reverseOrder()));
}
 

@Test
public void testListPagination() {
    int firstRequestMaxKeys = 4;
    String prefix = "key";
    ListObjectsV2Response result = s3.listObjectsV2(ListObjectsV2Request.builder()
                                                                        .bucket(bucketName)
                                                                        .prefix(prefix)
                                                                        .maxKeys(firstRequestMaxKeys)
                                                                        .build());
    List<S3Object> objects = result.contents();

    assertEquals(firstRequestMaxKeys, objects.size());
    assertEquals(bucketName, result.name());
    assertEquals(prefix, result.prefix());
    assertNotNull(result.nextContinuationToken());
    assertTrue(result.isTruncated());
    assertS3ObjectSummariesAreValid(objects, false);

    for (int i = 0; i < firstRequestMaxKeys; i++) {
        assertEquals(keys.get(i), objects.get(i).key());
    }

    ListObjectsV2Response nextResults = s3.listObjectsV2(ListObjectsV2Request.builder()
                                                                             .bucket(bucketName)
                                                                             .prefix(prefix)
                                                                             .continuationToken(
                                                                                     result.nextContinuationToken())
                                                                             .build());
    List<S3Object> nextObjects = nextResults.contents();

    assertNull(nextResults.nextContinuationToken());
    assertEquals(nextResults.continuationToken(), result.nextContinuationToken());
    assertFalse(nextResults.isTruncated());
    assertEquals(prefix, nextResults.prefix());
    assertS3ObjectSummariesAreValid(nextObjects, false);
    assertEquals(nextObjects.size(), BUCKET_OBJECTS - firstRequestMaxKeys);
}
 

private boolean isEmptyDirectory(URI uri)
    throws IOException {
  if (!isDirectory(uri)) {
    return false;
  }
  String prefix = normalizeToDirectoryPrefix(uri);
  boolean isEmpty = true;
  ListObjectsV2Response listObjectsV2Response;
  ListObjectsV2Request.Builder listObjectsV2RequestBuilder = ListObjectsV2Request.builder().bucket(uri.getHost());

  if (!prefix.equals(DELIMITER)) {
    listObjectsV2RequestBuilder = listObjectsV2RequestBuilder.prefix(prefix);
  }

  ListObjectsV2Request listObjectsV2Request = listObjectsV2RequestBuilder.build();
  listObjectsV2Response = _s3Client.listObjectsV2(listObjectsV2Request);

  for (S3Object s3Object : listObjectsV2Response.contents()) {
    if (s3Object.key().equals(prefix)) {
      continue;
    } else {
      isEmpty = false;
      break;
    }
  }
  return isEmpty;
}
 

/**
 * Delete a S3 bucket with the given name
 *
 * @param name
 * @return
 */
public boolean deleteS3Bucket(String name) {
    boolean deleted = true;
    ListObjectsV2Request listObjectsV2Request =
            ListObjectsV2Request.builder().bucket(name).build();
    ListObjectsV2Response listObjectsV2Response;
    S3Client s3Client = S3Client.builder().region(mAWSRegion).build();
    do {
        listObjectsV2Response = s3Client.listObjectsV2(listObjectsV2Request);
        if (listObjectsV2Response.contents() != null) {
            // detelete the files in the bucket
            for (S3Object s3Object : listObjectsV2Response.contents()) {
                mLogger.debug("Deleteing file " + s3Object.key() + " from bucket " + name);
                s3Client.deleteObject(
                        DeleteObjectRequest.builder().bucket(name).key(s3Object.key()).build());
            }
        }

        listObjectsV2Request =
                ListObjectsV2Request.builder()
                        .bucket(name)
                        .continuationToken(listObjectsV2Response.nextContinuationToken())
                        .build();

    } while (listObjectsV2Response.isTruncated());

    // Delete empty bucket
    DeleteBucketRequest deleteBucketRequest =
            DeleteBucketRequest.builder().bucket(name).build();
    s3Client.deleteBucket(deleteBucketRequest);
    return deleted;
}
 

private Artifact listFiles() {
    ListObjectsRequest req =
            ListObjectsRequest.builder()
                    .bucket(bucket)
                    .maxKeys(100)
                    .prefix(prefix)
                    .delimiter("/")
                    .build();

    ListObjectsResponse resp = client.listObjects(req);
    List<S3Object> list = resp.contents();
    if (list.isEmpty()) {
        return null;
    }

    Artifact artifact = new Artifact();
    artifact.setName(modelName);
    Map<String, Artifact.Item> files = new ConcurrentHashMap<>();
    for (S3Object obj : list) {
        Artifact.Item item = new Artifact.Item();
        String key = obj.key();
        if (!key.endsWith("/")) {
            item.setUri(key);
            item.setSize(obj.size());
            item.setArtifact(artifact);
            if ("dir".equals(item.getType())) {
                item.setName(""); // avoid creating extra folder
            }
            files.put(key, item);
        }
    }
    artifact.setFiles(files);
    return artifact;
}
 

@Test
public void testListWithFetchOwner() {
    ListObjectsV2Response result = s3.listObjectsV2(ListObjectsV2Request.builder()
                                                                        .bucket(bucketName)
                                                                        .fetchOwner(true)
                                                                        .build());
    List<S3Object> objects = result.contents();
    assertS3ObjectSummariesAreValid(objects, true);
}
 

@Test
public void testListWithMaxKeys() {
    int maxKeys = 4;
    ListObjectsV2Response result = s3.listObjectsV2(ListObjectsV2Request.builder()
                                                                        .bucket(bucketName)
                                                                        .maxKeys(maxKeys)
                                                                        .build());

    List<S3Object> objects = result.contents();

    assertEquals(maxKeys, objects.size());
    assertEquals(bucketName, result.name());
    assertThat(maxKeys, equalTo(result.maxKeys()));
    assertS3ObjectSummariesAreValid(objects, false);

    // We didn't use a delimiter, so we expect this to be empty/null
    assertNull(result.delimiter());

    // We expect truncated results since we set maxKeys
    assertTrue(result.isTruncated());
    assertNotNull(result.nextContinuationToken());
    assertTrue(result.nextContinuationToken().length() > 1);

    // URL encoding is requested by default

    // We didn't set other request parameters, so we expect them to be empty
    assertNull(result.encodingType());
    assertThat(result.prefix(), isEmptyString());
    assertNull(result.startAfter());
    assertNull(result.delimiter());
}
 

@Test
public void testListWithPrefixAndDelimiter() {
    String prefix = "a";
    String delimiter = "/";
    ListObjectsV2Response result = s3.listObjectsV2(ListObjectsV2Request.builder()
                                                                        .bucket(bucketName)
                                                                        .prefix(prefix)
                                                                        .delimiter(delimiter)
                                                                        .build());

    List<S3Object> objects = result.contents();

    assertEquals(1, objects.size());
    assertEquals(bucketName, result.name());
    assertS3ObjectSummariesAreValid(objects, false);
    assertEquals(prefix, result.prefix());
    assertEquals(delimiter, result.delimiter());

    // We don't expect any truncated results
    assertFalse(result.isTruncated());
    assertNull(result.nextContinuationToken());

    // We didn't set other request parameters, so we expect them to be empty
    assertNull(result.startAfter());
    assertNull(result.encodingType());
    assertTrue(result.maxKeys() >= 1000);
}
 

@Test
public void testListWithPrefixAndStartAfter() {
    String prefix = "key";
    String startAfter = "key-01";
    ListObjectsV2Response result = s3.listObjectsV2(ListObjectsV2Request.builder()
                                                                        .bucket(bucketName)
                                                                        .prefix(prefix)
                                                                        .startAfter(startAfter)
                                                                        .build());
    List<S3Object> objects = result.contents();

    assertEquals(BUCKET_OBJECTS - 1, objects.size());
    assertEquals(bucketName, result.name());
    assertS3ObjectSummariesAreValid(objects, false);
    assertEquals(startAfter, result.startAfter());
    assertEquals(prefix, result.prefix());

    // We didn't use a delimiter, so we expect it to be empty/null
    assertNull(result.delimiter());

    // We don't expect any truncated results
    assertFalse(result.isTruncated());
    assertNull(result.nextContinuationToken());

    // We didn't set any other request parameters, so we expect them to be
    // set to the defaults.
    assertTrue(result.maxKeys() >= 1000);
    assertNull(result.encodingType());
}
 

/**
 * Asserts that a list of S3Object objects are valid, by checking
 * that expected fields are not null or empty, that ETag values don't
 * contain leading or trailing quotes, that the last modified date is
 * recent, etc.
 *  @param objectSummaries The list of objects to validate.
 *
 */
private void assertS3ObjectSummariesAreValid(List<S3Object> objectSummaries) {
    for (S3Object obj : objectSummaries) {
        assertTrue(obj.eTag().length() > 1);
        assertTrue(obj.key().length() > 1);

        // Verify that the last modified date is within an hour
        assertNotNull(obj.lastModified());
        long offset = obj.lastModified().toEpochMilli() - Instant.now().toEpochMilli();
        assertTrue(offset < ONE_HOUR_IN_MILLISECONDS);

        assertTrue(obj.storageClassAsString().length() > 1);
    }
}
 

@Test
public void listObjectsWithMaxKeys() {
    int maxKeys = 4;
    ListObjectsResponse result = s3.listObjects(ListObjectsRequest.builder()
                                                                  .bucket(bucketName)
                                                                  .maxKeys(maxKeys)
                                                                  .build());

    List<S3Object> objects = result.contents();

    assertEquals(maxKeys, objects.size());
    assertEquals(bucketName, result.name());
    assertThat(maxKeys).isEqualTo(result.maxKeys());
    assertS3ObjectSummariesAreValid(objects);

    // We didn't use a delimiter, so we expect this to be empty/null
    assertNull(result.delimiter());

    // We expect truncated results since we set maxKeys
    assertTrue(result.isTruncated());

    // URL encoding is requested by default

    // We didn't set other request parameters, so we expect them to be empty
    assertNull(result.encodingType());
    assertThat(result.prefix()).isEmpty();
    assertNull(result.delimiter());
}
 

@Test
public void test_AsyncResponse_UsingRxJava() {
    ListObjectsV2Publisher publisher = s3Async.listObjectsV2Paginator(requestBuilder.bucket(bucketName).build());

    Single<List<S3Object>> objects = Flowable.fromPublisher(publisher)
                                             .flatMapIterable(ListObjectsV2Response::contents)
                                             .toList();

    // There are multiple fluent methods to convert Single type to a different form
    List<S3Object> objectList = objects.blockingGet();
    assertThat(Long.valueOf(objectList.size()), equalTo(OBJECT_COUNT));
}
 

private static List<S3Object> decodeContents(List<S3Object> contents) {
    if (contents == null) {
        return null;
    }
    return Collections.unmodifiableList(contents.stream()
                                                 .map(o -> o.toBuilder().key(urlDecode(o.key())).build())
                                                 .collect(Collectors.toList()));
}
 

public static FileObject from(S3Object s3Object) {
    FileObject file = new FileObject();
    if (s3Object != null) {
        file.setObjectKey(s3Object.key());
        file.setSize(s3Object.size());
    }
    return file;
}
 

@GET
@Produces(MediaType.APPLICATION_JSON)
public List<FileObject> listFiles() {
    ListObjectsRequest listRequest = ListObjectsRequest.builder().bucket(bucketName).build();

    //HEAD S3 objects to get metadata
    return s3.listObjects(listRequest).contents().stream().sorted(Comparator.comparing(S3Object::lastModified).reversed())
            .map(FileObject::from).collect(Collectors.toList());
}
 

private void assertKeysAreDecoded(List<S3Object> objects) {
    objects.forEach(o -> assertDecoded(o::key));
}