com.amazonaws.services.s3.model.S3ObjectSummary Java Examples

@Override
public VersionListing listVersions() {
    VersionListing versionListing = new VersionListing();
    this.objectListing = client.listObjects(listObjectsRequest);
    for(S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) {
        S3VersionSummary versionSummary = new S3VersionSummary();
        versionSummary.setBucketName(objectSummary.getBucketName());
        versionSummary.setETag(objectSummary.getETag());
        versionSummary.setKey(objectSummary.getKey());
        versionSummary.setLastModified(objectSummary.getLastModified());
        versionSummary.setOwner(objectSummary.getOwner());
        versionSummary.setSize(objectSummary.getSize());
        versionSummary.setStorageClass(objectSummary.getStorageClass());
        versionSummary.setIsLatest(true);

        versionListing.getVersionSummaries().add(versionSummary);
    }

    return versionListing;
}
 

@Test
public void testTagObjects()
{
    // Create an S3 file transfer request parameters DTO to access S3 objects.
    S3FileTransferRequestParamsDto s3FileTransferRequestParamsDto = new S3FileTransferRequestParamsDto();

    // Create an S3 file transfer request parameters DTO to tag S3 objects.
    S3FileTransferRequestParamsDto s3ObjectTaggerParamsDto = new S3FileTransferRequestParamsDto();
    s3ObjectTaggerParamsDto.setAwsAccessKeyId(AWS_ASSUMED_ROLE_ACCESS_KEY);
    s3ObjectTaggerParamsDto.setAwsSecretKey(AWS_ASSUMED_ROLE_SECRET_KEY);
    s3ObjectTaggerParamsDto.setSessionToken(AWS_ASSUMED_ROLE_SESSION_TOKEN);

    // Create an S3 object summary.
    S3ObjectSummary s3ObjectSummary = new S3ObjectSummary();
    s3ObjectSummary.setKey(S3_KEY);

    // Create an S3 object tag.
    Tag tag = new Tag(S3_OBJECT_TAG_KEY, S3_OBJECT_TAG_VALUE);

    // Call the method under test.
    s3Service.tagObjects(s3FileTransferRequestParamsDto, s3ObjectTaggerParamsDto, Collections.singletonList(s3ObjectSummary), tag);

    // Verify the external calls.
    verify(s3Dao).tagObjects(s3FileTransferRequestParamsDto, s3ObjectTaggerParamsDto, Collections.singletonList(s3ObjectSummary), tag);
    verifyNoMoreInteractions(s3Dao);
}
 

private Collection<String> getKeys(final String bucket, final int start, final int count) {
    final Set<String> keys = new HashSet<>();
    int index = 0;
    ObjectListing listing = this.s3Client.listObjects(bucket);
    do {
        for (final S3ObjectSummary summary : listing.getObjectSummaries()) {
            if (index < start) {
                logger.debug("skipping {} at index={} start={}", summary.getKey(), index++, start);
                continue;
            }
            keys.add(summary.getKey());

            if (keys.size() == count) {
                logger.debug("retrieved {}/{} keys, returning early", keys.size(), count);
                return keys;
            }
        }

        logger.debug("got {} keys from {}", listing.getObjectSummaries().size(), listing);
        listing = this.s3Client.listNextBatchOfObjects(listing);
    } while (listing.isTruncated());

    return keys;
}
 

public void delete(String bucketName, String pathPrefix) {
    ObjectListing listing = null;
    do {
        listing = listing == null ? helper.client().listObjects(bucketName, pathPrefix) : helper.client().listNextBatchOfObjects(listing);

        DeleteObjectsRequest req = new DeleteObjectsRequest(bucketName);

        List<DeleteObjectsRequest.KeyVersion> keys = new ArrayList<>(listing.getObjectSummaries().size());
        for (S3ObjectSummary summary : listing.getObjectSummaries()) {
            keys.add(new DeleteObjectsRequest.KeyVersion(summary.getKey()));
        }
        req.withKeys(keys);

        helper.client().deleteObjects(req);
    } while (listing.isTruncated());
}
 

public static void emptyAndDeleteBucket(AmazonS3Client client,
        String bucketName) {
    ObjectListing objectListing = client.listObjects(bucketName);

    while (true) {
        for (Iterator<?> iterator = objectListing.getObjectSummaries()
                .iterator(); iterator.hasNext();) {
            S3ObjectSummary objectSummary = (S3ObjectSummary) iterator
                    .next();
            client.deleteObject(bucketName, objectSummary.getKey());
        }

        if (objectListing.isTruncated()) {
            objectListing = client.listNextBatchOfObjects(objectListing);
        } else {
            break;
        }
    }
    client.deleteBucket(bucketName);
}
 

@Override
public ObjectListing listObjects(final ListObjectsRequest request) throws AmazonClientException {
    assertThat(request.getBucketName(), equalTo(bucket));

    final ObjectListing listing = new ObjectListing();
    listing.setBucketName(request.getBucketName());
    listing.setPrefix(request.getPrefix());

    for (Map.Entry<String, byte[]> blob : blobs.entrySet()) {
        if (Strings.isEmpty(request.getPrefix()) || blob.getKey().startsWith(request.getPrefix())) {
            S3ObjectSummary summary = new S3ObjectSummary();
            summary.setBucketName(request.getBucketName());
            summary.setKey(blob.getKey());
            summary.setSize(blob.getValue().length);
            listing.getObjectSummaries().add(summary);
        }
    }
    return listing;
}
 

@Test
public void testTagObjects()
{
    // Create an S3 file transfer request parameters DTO to access S3 objects.
    S3FileTransferRequestParamsDto params = new S3FileTransferRequestParamsDto();
    params.setS3BucketName(S3_BUCKET_NAME);

    // Create an S3 object summary.
    S3ObjectSummary s3ObjectSummary = new S3ObjectSummary();
    s3ObjectSummary.setKey(TARGET_S3_KEY);

    // Create an S3 object tag.
    Tag tag = new Tag(S3_OBJECT_TAG_KEY, S3_OBJECT_TAG_VALUE);

    // Put a file in S3.
    s3Operations.putObject(new PutObjectRequest(S3_BUCKET_NAME, TARGET_S3_KEY, new ByteArrayInputStream(new byte[1]), new ObjectMetadata()), null);

    // Tag the file with an S3 object tag.
    s3Dao.tagObjects(params, new S3FileTransferRequestParamsDto(), Collections.singletonList(s3ObjectSummary), tag);

    // Validate that the object got tagged.
    GetObjectTaggingResult getObjectTaggingResult = s3Operations.getObjectTagging(new GetObjectTaggingRequest(S3_BUCKET_NAME, TARGET_S3_KEY), null);
    assertEquals(Collections.singletonList(tag), getObjectTaggingResult.getTagSet());
}
 

@Override
public VersionListing listVersions() {
    VersionListing versionListing = new VersionListing();
    this.objectListing = client.listObjects(listObjectsRequest);
    for(S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) {
        S3VersionSummary versionSummary = new S3VersionSummary();
        versionSummary.setBucketName(objectSummary.getBucketName());
        versionSummary.setETag(objectSummary.getETag());
        versionSummary.setKey(objectSummary.getKey());
        versionSummary.setLastModified(objectSummary.getLastModified());
        versionSummary.setOwner(objectSummary.getOwner());
        versionSummary.setSize(objectSummary.getSize());
        versionSummary.setStorageClass(objectSummary.getStorageClass());
        versionSummary.setIsLatest(true);

        versionListing.getVersionSummaries().add(versionSummary);
    }

    return versionListing;
}
 

/**
 * This creates a new FileWrapper instance based on the S3ObjectSummary information.
 *
 * @param pathComponentCount The root path component count.
 * @param javaPath           The Path instance for the file.
 * @param entry              The S3ObjectSummary for the file.
 * @return A new FileWrapper instance.
 */
public static FileWrapper createFileWrapperFromFile(int pathComponentCount, Path javaPath, S3ObjectSummary entry) {
	Path pathFileName = javaPath.getFileName();
	if (pathFileName == null) {
		return null;
	}
	return new FileWrapper(
			// Name:
			convertPathToAwsFormat(pathFileName),
			// Path (relative to the `path` parameter):
			convertPathToAwsFormat(javaPath.subpath(pathComponentCount, javaPath.getNameCount())),
			// Directory?
			false,
			// Size:
			entry.getSize(),
			// Last modified (milliseconds):
			entry.getLastModified().getTime()
	);
}
 

/**
 * @throws Exception If error.
 */
@Override protected void afterSpiStopped() throws Exception {
    AWSCredentials cred = new BasicAWSCredentials(IgniteS3TestSuite.getAccessKey(),
        IgniteS3TestSuite.getSecretKey());

    AmazonS3 s3 = new AmazonS3Client(cred);

    String bucketName = S3CheckpointSpi.BUCKET_NAME_PREFIX + "unit-test-bucket";

    try {
        ObjectListing list = s3.listObjects(bucketName);

        while (true) {
            for (S3ObjectSummary sum : list.getObjectSummaries())
                s3.deleteObject(bucketName, sum.getKey());

            if (list.isTruncated())
                list = s3.listNextBatchOfObjects(list);
            else
                break;
        }
    }
    catch (AmazonClientException e) {
        throw new IgniteSpiException("Failed to read checkpoint bucket: " + bucketName, e);
    }
}
 

@Test
public void testListDirectoryIgnoreZeroByteDirectoryMarkers()
{
    // Create an S3 file transfer request parameters DTO.
    S3FileTransferRequestParamsDto s3FileTransferRequestParamsDto = new S3FileTransferRequestParamsDto();

    // Create a list of S3 object summaries.
    List<S3ObjectSummary> s3ObjectSummaries = Collections.singletonList(new S3ObjectSummary());

    // Mock the external calls.
    when(s3Dao.listDirectory(s3FileTransferRequestParamsDto, true)).thenReturn(s3ObjectSummaries);

    // Call the method under test.
    List<S3ObjectSummary> result = s3Service.listDirectory(s3FileTransferRequestParamsDto, true);

    // Verify the external calls.
    verify(s3Dao).listDirectory(s3FileTransferRequestParamsDto, true);
    verifyNoMoreInteractions(s3Dao);

    // Validate the returned object.
    assertEquals(s3ObjectSummaries, result);
}
 

@Test
public void testValidateRegisteredS3FilesUnexpectedNonEmptyS3FileFound() throws IOException
{
    // Create two lists of expected and actual storage files, with an actual file not being added to the list of expected files.
    List<StorageFile> testExpectedFiles = new ArrayList<>();
    List<S3ObjectSummary> testActualFiles = Collections.singletonList(createS3ObjectSummary(TARGET_S3_KEY, FILE_SIZE_1_KB));

    // Create a business object data key.
    BusinessObjectDataKey businessObjectDataKey =
        new BusinessObjectDataKey(BDEF_NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, FORMAT_VERSION, PARTITION_VALUE, SUBPARTITION_VALUES,
            DATA_VERSION);

    // Try to validate S3 files when unexpected non-empty S3 file exists.
    // The validation is expected to fail when detecting an unregistered non-empty S3 file.
    try
    {
        storageFileHelper.validateRegisteredS3Files(testExpectedFiles, testActualFiles, STORAGE_NAME, businessObjectDataKey);
        fail();
    }
    catch (IllegalStateException e)
    {
        assertEquals(String.format("Found unregistered non-empty S3 file \"%s\" in \"%s\" storage. Business object data {%s}", TARGET_S3_KEY, STORAGE_NAME,
            businessObjectDataServiceTestHelper.getExpectedBusinessObjectDataKeyAsString(businessObjectDataKey)), e.getMessage());
    }
}
 

/***
 * Download a S3 object to local directory
 *
 * @param s3ObjectSummary S3 object summary for the object to download
 * @param targetDirectory Local target directory to download the object to
 * @throws IOException If any errors were encountered in downloading the object
 */
public void downloadS3Object(S3ObjectSummary s3ObjectSummary,
    String targetDirectory)
    throws IOException {

  final AmazonS3 amazonS3 = getS3Client();

  final GetObjectRequest getObjectRequest = new GetObjectRequest(
      s3ObjectSummary.getBucketName(),
      s3ObjectSummary.getKey());

  final S3Object s3Object = amazonS3.getObject(getObjectRequest);

  final String targetFile = StringUtils.removeEnd(targetDirectory, File.separator) + File.separator + s3Object.getKey();
  FileUtils.copyInputStreamToFile(s3Object.getObjectContent(), new File(targetFile));

  LOGGER.info("S3 object downloaded to file: " + targetFile);
}
 

@Test
public void testUnicodeObject() throws Exception {
    String blobName = "ŪņЇЌœđЗ/☺ unicode € rocks ™";
    ObjectMetadata metadata = new ObjectMetadata();
    metadata.setContentLength(BYTE_SOURCE.size());
    client.putObject(containerName, blobName, BYTE_SOURCE.openStream(),
            metadata);

    metadata = client.getObjectMetadata(containerName, blobName);
    assertThat(metadata).isNotNull();

    ObjectListing listing = client.listObjects(containerName);
    List<S3ObjectSummary> summaries = listing.getObjectSummaries();
    assertThat(summaries).hasSize(1);
    S3ObjectSummary summary = summaries.iterator().next();
    assertThat(summary.getKey()).isEqualTo(blobName);
}
 

private Answer<ObjectListing> objectListingAnswer(@Nullable final String marker, final String... fileNames) {
    return new Answer<ObjectListing>() {
        @Override
        public ObjectListing answer(InvocationOnMock invocation)
                throws Throwable {
            ListObjectsRequest request = (ListObjectsRequest) invocation.getArguments()[0];

            ObjectListing objectListing = new ObjectListing();
            objectListing.setBucketName(request.getBucketName());
            objectListing.setPrefix(request.getPrefix());

            objectListing.setTruncated(marker != null);
            objectListing.setNextMarker(marker);

            for (String fileName : fileNames) {
                S3ObjectSummary objectSummary = new S3ObjectSummary();
                objectSummary.setKey(request.getPrefix() + fileName);
                objectSummary.setSize(100);
                objectListing.getObjectSummaries().add(objectSummary);
            }

            return objectListing;
        }
    };
}
 

@Test
public void testTagObjectsAmazonServiceException()
{
    // Create an S3 file transfer request parameters DTO to access S3 objects with a mocked S3 bucket name that would trigger an AWS exception.
    S3FileTransferRequestParamsDto params = new S3FileTransferRequestParamsDto();
    params.setS3BucketName(MockS3OperationsImpl.MOCK_S3_BUCKET_NAME_INTERNAL_ERROR);

    // Create an S3 object summary.
    S3ObjectSummary s3ObjectSummary = new S3ObjectSummary();
    s3ObjectSummary.setKey(TARGET_S3_KEY);

    // Create an S3 object tag.
    Tag tag = new Tag(S3_OBJECT_TAG_KEY, S3_OBJECT_TAG_VALUE);

    try
    {
        s3Dao.tagObjects(params, new S3FileTransferRequestParamsDto(), Collections.singletonList(s3ObjectSummary), tag);
        fail();
    }
    catch (IllegalStateException e)
    {
        assertEquals(String.format("Failed to tag S3 object with \"%s\" key and \"null\" version id in \"%s\" bucket. " +
                "Reason: InternalError (Service: null; Status Code: 0; Error Code: InternalError; Request ID: null)", TARGET_S3_KEY,
            MockS3OperationsImpl.MOCK_S3_BUCKET_NAME_INTERNAL_ERROR), e.getMessage());
    }
}
 

@Override
public void deleteByTenant(final String tenant) {
    final String folder = sanitizeTenant(tenant);

    LOG.info("Deleting S3 object folder (tenant) from bucket {} and key {}", s3Properties.getBucketName(), folder);

    // Delete artifacts
    ObjectListing objects = amazonS3.listObjects(s3Properties.getBucketName(), folder + "/");
    do {
        for (final S3ObjectSummary objectSummary : objects.getObjectSummaries()) {
            amazonS3.deleteObject(s3Properties.getBucketName(), objectSummary.getKey());
        }
        objects = amazonS3.listNextBatchOfObjects(objects);
    } while (objects.isTruncated());

}
 

public static List<S3ObjectSummary> listObjects(AmazonS3 client, String bucket) {
  ObjectListing objects = client.listObjects(bucket);
  List<S3ObjectSummary> result = new ArrayList<>(objects.getObjectSummaries().size());
  for (S3ObjectSummary objectSummary : objects.getObjectSummaries()) {
    if (objectSummary.getKey().endsWith("/") || objectSummary.getKey().endsWith("_$folder$")) {
      continue;
    }
    result.add(objectSummary);
  }
  return result;
}
 

@Test
public void createFileWrapperFromFile() throws Exception {
	FileWrapper file;
	S3ObjectSummary s3ObjectSummary = new S3ObjectSummary();
	s3ObjectSummary.setBucketName("my-bucket");
	s3ObjectSummary.setKey("path/to/my/file.ext");
	s3ObjectSummary.setLastModified(new Date(9000));
	s3ObjectSummary.setSize(12);

	file = S3FindFilesStep.Execution.createFileWrapperFromFile(0, Paths.get(s3ObjectSummary.getKey()), s3ObjectSummary);
	Assert.assertEquals("file.ext", file.getName());
	Assert.assertEquals("path/to/my/file.ext", file.getPath());
	Assert.assertFalse(file.isDirectory());
	Assert.assertEquals(12, file.getLength());
	Assert.assertEquals(9000, file.getLastModified());

	file = S3FindFilesStep.Execution.createFileWrapperFromFile(1, Paths.get(s3ObjectSummary.getKey()), s3ObjectSummary);
	Assert.assertEquals("file.ext", file.getName());
	Assert.assertEquals("to/my/file.ext", file.getPath());
	Assert.assertFalse(file.isDirectory());
	Assert.assertEquals(12, file.getLength());
	Assert.assertEquals(9000, file.getLastModified());

	file = S3FindFilesStep.Execution.createFileWrapperFromFile(2, Paths.get(s3ObjectSummary.getKey()), s3ObjectSummary);
	Assert.assertEquals("file.ext", file.getName());
	Assert.assertEquals("my/file.ext", file.getPath());
	Assert.assertFalse(file.isDirectory());
	Assert.assertEquals(12, file.getLength());
	Assert.assertEquals(9000, file.getLastModified());
}
 

public StorageObjectSummary next()
{
  // Get the next S3 summary object and return it as a platform-agnostic object (StorageObjectSummary)
  S3ObjectSummary s3Obj = s3ObjSummariesIterator.next();

  return StorageObjectSummary.createFromS3ObjectSummary(s3Obj);
}
 

/**
 * Get the splits for a record stored in stash.  Each split corresponds to a file in the Stash table's directory.
 */
public List<StashSplit> getSplits(String table)
        throws StashNotAvailableException, TableNotStashedException {
    ImmutableList.Builder<StashSplit> splitsBuilder = ImmutableList.builder();

    Iterator<S3ObjectSummary> objectSummaries = getS3ObjectSummariesForTable(table);
    while (objectSummaries.hasNext()) {
        S3ObjectSummary objectSummary = objectSummaries.next();
        String key = objectSummary.getKey();
        // Strip the common root path prefix from the split since it is constant.
        splitsBuilder.add(new StashSplit(table, key.substring(_rootPath.length() + 1), objectSummary.getSize()));
    }

    return splitsBuilder.build();
}
 

private void sendLineageEvent(S3ObjectSummary s3Object) {
  LineageEvent event = context.createLineageEvent(LineageEventType.ENTITY_READ);
  event.setSpecificAttribute(LineageSpecificAttribute.ENDPOINT_TYPE, EndPointType.S3.name());
  event.setSpecificAttribute(LineageSpecificAttribute.ENTITY_NAME, s3Object.getKey());
  event.setSpecificAttribute(LineageSpecificAttribute.DESCRIPTION, s3ConfigBean.s3Config.bucket);

  context.publishLineageEvent(event);
}
 

/**
 * Creates a new S3 writable configuration.
 * @param s3 An S3 {@link AmazonS3 client}.
 * @param s3ObjectSummary The S3 object {@link S3ObjectSummary summary}.
 * @param version The version of the configuration.
 */
public S3WritableConfiguration(AmazonS3 s3, S3ObjectSummary s3ObjectSummary, String version) {

  this.s3 = s3;
  this.s3Object = s3.getObject(s3ObjectSummary.getBucketName(), s3ObjectSummary.getKey());
  this.version = version;

}
 

private void addKeyUris(List<URI> uris, ObjectListing list, URI uri, Predicate<URI> uriPredicate) {
    List<S3ObjectSummary> summaries = list.getObjectSummaries();
    for (S3ObjectSummary summary : summaries) {
        String key = summary.getKey();
        if (!key.endsWith("/")) {
            URI keyUri = uri.resolve("/" + key);
            if (uriPredicate.apply(keyUri)) {
                uris.add(keyUri);
                if (logger.isDebugEnabled()) {
                    logger.debug("{}", keyUri);
                }
            }
        }
    }
}
 

@Override
public Stream<BlobId> getDirectPathBlobIdStream(final String prefix) {
  String subpath = format("%s/%s", DIRECT_PATH_PREFIX, prefix);
  Iterable<S3ObjectSummary> summaries = S3Objects.withPrefix(s3, getConfiguredBucket(), subpath);
  return stream(summaries.spliterator(), false)
    .map(S3ObjectSummary::getKey)
    .filter(key -> key.endsWith(BLOB_ATTRIBUTE_SUFFIX))
    .map(this::attributePathToDirectPathBlobId);
}
 

@Override
public List<OldNoteInfo> list(AuthenticationInfo subject) throws IOException {
  List<OldNoteInfo> infos = new LinkedList<>();
  OldNoteInfo info;
  try {
    ListObjectsRequest listObjectsRequest = new ListObjectsRequest()
            .withBucketName(bucketName)
            .withPrefix(user + "/" + "notebook");
    ObjectListing objectListing;
    do {
      objectListing = s3client.listObjects(listObjectsRequest);
      for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) {
        if (objectSummary.getKey().endsWith("note.json")) {
          info = getNoteInfo(objectSummary.getKey());
          if (info != null) {
            infos.add(info);
          } else {
            LOG.debug("Unable to get notebook info for key: " + objectSummary.getKey());
          }
        }
      }
      listObjectsRequest.setMarker(objectListing.getNextMarker());
    } while (objectListing.isTruncated());
  } catch (AmazonClientException ace) {
    throw new IOException("Unable to list objects in S3: " + ace, ace);
  }
  return infos;
}
 

private boolean walk(Iterator<S3ObjectSummary> iter, ObjectId file, String path) throws IOException {
    byte[] content;
    byte[] newHash;
    LOG.debug("Start processing file: {}", path);
    try (DigestInputStream is = new DigestInputStream(repository.open(file).openStream(), DigestUtils.getMd5Digest())) {
        // Get content
        content = IOUtils.toByteArray(is);
        // Get hash
        newHash = is.getMessageDigest().digest();
    }
    if (isUploadFile(iter, path, Hex.encodeHexString(newHash))) {
        LOG.info("Uploading file: {}", path);
        ObjectMetadata bucketMetadata = new ObjectMetadata();
        bucketMetadata.setContentMD5(Base64.encodeAsString(newHash));
        bucketMetadata.setContentLength(content.length);
        // Give Tika a few hints for the content detection
        Metadata tikaMetadata = new Metadata();
        tikaMetadata.set(Metadata.RESOURCE_NAME_KEY, FilenameUtils.getName(FilenameUtils.normalize(path)));
        // Fire!
        try (InputStream bis = TikaInputStream.get(content, tikaMetadata)) {
            bucketMetadata.setContentType(TIKA_DETECTOR.detect(bis, tikaMetadata).toString());
            s3.putObject(bucket.getName(), path, bis, bucketMetadata);
            return true;
        }
    }
    LOG.info("Skipping file (same checksum): {}", path);
    return false;
}
 

private List<KeyVersion> getKeysFromListing(ObjectListing listing) {
  Set<String> keys = listing.getObjectSummaries().stream().map(S3ObjectSummary::getKey).collect(Collectors.toSet());
  List<KeyVersion> keysToDelete = keys.stream().map(k -> new KeyVersion(k)).collect(Collectors.toList());

  if (!listing.isTruncated()) {
    return keysToDelete;
  }
  listing.setNextMarker(listing.getNextMarker());
  keysToDelete.addAll(getKeysFromListing(listing));

  return keysToDelete;
}
 

public void deleteBucketAndContent()
{
  //Get the list of objects
  ObjectListing objectListing = client.listObjects(testMeta.bucketKey);
  for ( Iterator<?> iterator = objectListing.getObjectSummaries().iterator(); iterator.hasNext(); ) {
    S3ObjectSummary objectSummary = (S3ObjectSummary)iterator.next();
    LOG.info("Deleting an object: {}",objectSummary.getKey());
    client.deleteObject(testMeta.bucketKey, objectSummary.getKey());
  }
  client.deleteBucket(testMeta.bucketKey);
}
 

/**
 * Returns the size of the dataset, in bytes. Will be zero if this dataset is a collection or non-existent.
 *
 * @return the size of the dataset
 */
@Override
public long length() {
  // If the summary is already in the cache, return it.
  // It'll have been added by a listDatasets() call on the parent directory.
  S3ObjectSummary objectSummary = objectSummaryCache.getIfPresent(s3uri);
  if (objectSummary != null) {
    return objectSummary.getSize();
  }

  /*
   * Get the metadata directly from S3. This will be expensive.
   * We get punished hard if length() and/or lastModified() is called on a bunch of datasets without
   * listDatasets() first being called on their parent directory.
   *
   * So, is the right thing to do here "getParentDataset().listDatasets()" and then query the cache again?
   * Perhaps, but listDatasets() throws an IOException, and length() and lastModified() do not.
   * We would have to change their signatures and the upstream client code to make it work.
   */
  ObjectMetadata metadata = threddsS3Client.getObjectMetadata(s3uri);
  if (metadata != null) {
    return metadata.getContentLength();
  } else {
    // "this" may be a collection or non-existent. In both cases, we return 0.
    return 0;
  }
}