com.amazonaws.services.s3.model.VersionListing Java Examples

@Override
public VersionListing listVersions() {
    VersionListing versionListing = new VersionListing();
    this.objectListing = client.listObjectsV2(listObjectsRequest);
    for(S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) {
        S3VersionSummary versionSummary = new S3VersionSummary();
        versionSummary.setBucketName(objectSummary.getBucketName());
        versionSummary.setETag(objectSummary.getETag());
        versionSummary.setKey(objectSummary.getKey());
        versionSummary.setLastModified(objectSummary.getLastModified());
        versionSummary.setOwner(objectSummary.getOwner());
        versionSummary.setSize(objectSummary.getSize());
        versionSummary.setStorageClass(objectSummary.getStorageClass());
        versionSummary.setIsLatest(true);

        versionListing.getVersionSummaries().add(versionSummary);
    }

    return versionListing;
}
 

@Override
public VersionListing listVersions() {
    VersionListing versionListing = new VersionListing();
    this.objectListing = client.listObjects(listObjectsRequest);
    for(S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) {
        S3VersionSummary versionSummary = new S3VersionSummary();
        versionSummary.setBucketName(objectSummary.getBucketName());
        versionSummary.setETag(objectSummary.getETag());
        versionSummary.setKey(objectSummary.getKey());
        versionSummary.setLastModified(objectSummary.getLastModified());
        versionSummary.setOwner(objectSummary.getOwner());
        versionSummary.setSize(objectSummary.getSize());
        versionSummary.setStorageClass(objectSummary.getStorageClass());
        versionSummary.setIsLatest(true);

        versionListing.getVersionSummaries().add(versionSummary);
    }

    return versionListing;
}
 

@Override
public VersionListing listVersions() {
    VersionListing versionListing = new VersionListing();
    this.objectListing = client.listObjects(listObjectsRequest);
    for(S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) {
        S3VersionSummary versionSummary = new S3VersionSummary();
        versionSummary.setBucketName(objectSummary.getBucketName());
        versionSummary.setETag(objectSummary.getETag());
        versionSummary.setKey(objectSummary.getKey());
        versionSummary.setLastModified(objectSummary.getLastModified());
        versionSummary.setOwner(objectSummary.getOwner());
        versionSummary.setSize(objectSummary.getSize());
        versionSummary.setStorageClass(objectSummary.getStorageClass());
        versionSummary.setIsLatest(true);

        versionListing.getVersionSummaries().add(versionSummary);
    }

    return versionListing;
}
 

private boolean deleteObject(final String bucket, final String key) {
    this.s3Client.deleteObject(bucket, key);
    final VersionListing versionList = this.s3Client.listVersions(bucket, key);
    for (final S3VersionSummary summary : versionList.getVersionSummaries()) {
        logger.debug("deleting version {}", summary.getKey());
        this.s3Client.deleteVersion(bucket, summary.getKey(), summary.getVersionId());
    }

    return true;
}
 

private void deleteAllVersions(String bucketName) {
    logger.info("Deleting all object versions from bucket '{}'", bucketName);
    VersionListing versionListing = s3.listVersions(bucketName, null);
    delete(versionListing);
    while (versionListing.isTruncated()) {
        versionListing = s3.listNextBatchOfVersions(versionListing);
        delete(versionListing);
    }
}
 

@Test
public void testDeleteDirectoryNoS3VersionsExist()
{
    // Create an S3 file transfer request parameters DTO to access S3 objects.
    S3FileTransferRequestParamsDto s3FileTransferRequestParamsDto = new S3FileTransferRequestParamsDto();
    s3FileTransferRequestParamsDto.setS3BucketName(S3_BUCKET_NAME);
    s3FileTransferRequestParamsDto.setS3KeyPrefix(S3_KEY_PREFIX);

    // Create a retry policy.
    RetryPolicy retryPolicy =
        new RetryPolicy(PredefinedRetryPolicies.DEFAULT_RETRY_CONDITION, PredefinedRetryPolicies.DEFAULT_BACKOFF_STRATEGY, INTEGER_VALUE, true);

    // Create an empty version listing.
    VersionListing versionListing = new VersionListing();

    // Mock the external calls.
    when(retryPolicyFactory.getRetryPolicy()).thenReturn(retryPolicy);
    when(s3Operations.listVersions(any(ListVersionsRequest.class), any(AmazonS3Client.class))).thenReturn(versionListing);

    // Call the method under test.
    s3DaoImpl.deleteDirectory(s3FileTransferRequestParamsDto);

    // Verify the external calls.
    verify(retryPolicyFactory).getRetryPolicy();
    verify(s3Operations).listVersions(any(ListVersionsRequest.class), any(AmazonS3Client.class));
    verifyNoMoreInteractionsHelper();
}
 

@Test
public void testTagVersionsTargetTagKeyAlreadyExists()
{
    // Create two S3 object tags having the same tag key.
    List<Tag> tags = Arrays.asList(new Tag(S3_OBJECT_TAG_KEY, S3_OBJECT_TAG_VALUE), new Tag(S3_OBJECT_TAG_KEY, S3_OBJECT_TAG_VALUE_2));

    // Put an S3 file that is already tagged with the first S3 object tag in an S3 bucket that has versioning enabled.
    PutObjectRequest putObjectRequest =
        new PutObjectRequest(MockS3OperationsImpl.MOCK_S3_BUCKET_NAME_VERSIONING_ENABLED, TARGET_S3_KEY, new ByteArrayInputStream(new byte[1]),
            new ObjectMetadata());
    putObjectRequest.setTagging(new ObjectTagging(Collections.singletonList(tags.get(0))));
    s3Operations.putObject(putObjectRequest, null);

    // List S3 versions that match the test S3 key.
    ListVersionsRequest listVersionsRequest =
        new ListVersionsRequest().withBucketName(MockS3OperationsImpl.MOCK_S3_BUCKET_NAME_VERSIONING_ENABLED).withPrefix(TARGET_S3_KEY);
    VersionListing versionListing = s3Operations.listVersions(listVersionsRequest, null);
    assertEquals(1, CollectionUtils.size(versionListing.getVersionSummaries()));
    assertEquals(TARGET_S3_KEY, versionListing.getVersionSummaries().get(0).getKey());
    assertNotNull(versionListing.getVersionSummaries().get(0).getVersionId());

    // Validate that the S3 object is tagged with the first tag only.
    GetObjectTaggingResult getObjectTaggingResult = s3Operations.getObjectTagging(
        new GetObjectTaggingRequest(MockS3OperationsImpl.MOCK_S3_BUCKET_NAME_VERSIONING_ENABLED, TARGET_S3_KEY,
            versionListing.getVersionSummaries().get(0).getVersionId()), null);
    assertEquals(Collections.singletonList(tags.get(0)), getObjectTaggingResult.getTagSet());

    // Create an S3 file transfer request parameters DTO to access S3 objects.
    S3FileTransferRequestParamsDto params = new S3FileTransferRequestParamsDto();
    params.setS3BucketName(MockS3OperationsImpl.MOCK_S3_BUCKET_NAME_VERSIONING_ENABLED);

    // Tag the S3 version with the second S3 object tag.
    s3Dao.tagVersions(params, new S3FileTransferRequestParamsDto(), versionListing.getVersionSummaries(), tags.get(1));

    // Validate that the S3 object is now tagged with the second tag only.
    getObjectTaggingResult = s3Operations.getObjectTagging(
        new GetObjectTaggingRequest(MockS3OperationsImpl.MOCK_S3_BUCKET_NAME_VERSIONING_ENABLED, TARGET_S3_KEY,
            versionListing.getVersionSummaries().get(0).getVersionId()), null);
    assertEquals(Collections.singletonList(tags.get(1)), getObjectTaggingResult.getTagSet());
}
 

@Test
public void testTagVersionsS3BucketWithVersioningEnabled()
{
    // Create an S3 object tag.
    Tag tag = new Tag(S3_OBJECT_TAG_KEY, S3_OBJECT_TAG_VALUE);

    // Put two S3 versions in S3 bucket that has versioning enabled.
    for (int i = 0; i < 2; i++)
    {
        s3Operations.putObject(
            new PutObjectRequest(MockS3OperationsImpl.MOCK_S3_BUCKET_NAME_VERSIONING_ENABLED, TARGET_S3_KEY, new ByteArrayInputStream(new byte[1]),
                new ObjectMetadata()), null);
    }

    // List S3 versions that match the test key.
    ListVersionsRequest listVersionsRequest =
        new ListVersionsRequest().withBucketName(MockS3OperationsImpl.MOCK_S3_BUCKET_NAME_VERSIONING_ENABLED).withPrefix(TARGET_S3_KEY);
    VersionListing versionListing = s3Operations.listVersions(listVersionsRequest, null);
    assertEquals(2, CollectionUtils.size(versionListing.getVersionSummaries()));
    for (int i = 0; i < 2; i++)
    {
        assertNotNull(versionListing.getVersionSummaries().get(i).getVersionId());
    }

    // Create an S3 file transfer request parameters DTO to access S3 objects with a mocked S3 bucket name that enables S3 bucket versioning.
    S3FileTransferRequestParamsDto params = new S3FileTransferRequestParamsDto();
    params.setS3BucketName(MockS3OperationsImpl.MOCK_S3_BUCKET_NAME_VERSIONING_ENABLED);

    // Tag listed S3 version with an S3 object tag.
    s3Dao.tagVersions(params, new S3FileTransferRequestParamsDto(), versionListing.getVersionSummaries(), tag);

    // Validate that both versions got tagged.
    for (int i = 0; i < 2; i++)
    {
        GetObjectTaggingResult getObjectTaggingResult = s3Operations.getObjectTagging(
            new GetObjectTaggingRequest(MockS3OperationsImpl.MOCK_S3_BUCKET_NAME_VERSIONING_ENABLED, TARGET_S3_KEY,
                versionListing.getVersionSummaries().get(i).getVersionId()), null);
        assertEquals(Collections.singletonList(tag), getObjectTaggingResult.getTagSet());
    }
}
 

@Test
public void testTagVersionsS3BucketWithVersioningDisabled()
{
    // Create an S3 object tag.
    Tag tag = new Tag(S3_OBJECT_TAG_KEY, S3_OBJECT_TAG_VALUE);

    // Put S3 objects in S3 bucket that has versioning disabled.
    for (int i = 0; i < 2; i++)
    {
        s3Operations.putObject(new PutObjectRequest(S3_BUCKET_NAME, TARGET_S3_KEY + i, new ByteArrayInputStream(new byte[1]), new ObjectMetadata()), null);
    }

    // List S3 versions that match the test prefix.
    ListVersionsRequest listVersionsRequest = new ListVersionsRequest().withBucketName(S3_BUCKET_NAME).withPrefix(TARGET_S3_KEY);
    VersionListing versionListing = s3Operations.listVersions(listVersionsRequest, null);
    assertEquals(2, CollectionUtils.size(versionListing.getVersionSummaries()));
    for (int i = 0; i < 2; i++)
    {
        assertNull(versionListing.getVersionSummaries().get(i).getVersionId());
    }

    // Create an S3 file transfer request parameters DTO to access S3 objects.
    S3FileTransferRequestParamsDto params = new S3FileTransferRequestParamsDto();
    params.setS3BucketName(S3_BUCKET_NAME);

    // Tag listed S3 versions with an S3 object tag.
    s3Dao.tagVersions(params, new S3FileTransferRequestParamsDto(), versionListing.getVersionSummaries(), tag);

    // Validate that both S3 objects got tagged.
    for (int i = 0; i < 2; i++)
    {
        GetObjectTaggingResult getObjectTaggingResult =
            s3Operations.getObjectTagging(new GetObjectTaggingRequest(S3_BUCKET_NAME, TARGET_S3_KEY + i, null), null);
        assertEquals(Collections.singletonList(tag), getObjectTaggingResult.getTagSet());
    }
}
 

@Test
public void testTagVersionsOtherTagKeyAlreadyExists()
{
    // Create two S3 object tags having different tag keys.
    List<Tag> tags = Arrays.asList(new Tag(S3_OBJECT_TAG_KEY, S3_OBJECT_TAG_VALUE), new Tag(S3_OBJECT_TAG_KEY_2, S3_OBJECT_TAG_VALUE_2));

    // Put an S3 file that is already tagged with the first S3 object tag in an S3 bucket that has versioning enabled.
    PutObjectRequest putObjectRequest =
        new PutObjectRequest(MockS3OperationsImpl.MOCK_S3_BUCKET_NAME_VERSIONING_ENABLED, TARGET_S3_KEY, new ByteArrayInputStream(new byte[1]),
            new ObjectMetadata());
    putObjectRequest.setTagging(new ObjectTagging(Collections.singletonList(tags.get(0))));
    s3Operations.putObject(putObjectRequest, null);

    // List S3 versions that match the test S3 key.
    ListVersionsRequest listVersionsRequest =
        new ListVersionsRequest().withBucketName(MockS3OperationsImpl.MOCK_S3_BUCKET_NAME_VERSIONING_ENABLED).withPrefix(TARGET_S3_KEY);
    VersionListing versionListing = s3Operations.listVersions(listVersionsRequest, null);
    assertEquals(1, CollectionUtils.size(versionListing.getVersionSummaries()));
    assertEquals(TARGET_S3_KEY, versionListing.getVersionSummaries().get(0).getKey());
    assertNotNull(versionListing.getVersionSummaries().get(0).getVersionId());

    // Validate that the S3 object is tagged with the first tag only.
    GetObjectTaggingResult getObjectTaggingResult = s3Operations.getObjectTagging(
        new GetObjectTaggingRequest(MockS3OperationsImpl.MOCK_S3_BUCKET_NAME_VERSIONING_ENABLED, TARGET_S3_KEY,
            versionListing.getVersionSummaries().get(0).getVersionId()), null);
    assertEquals(Collections.singletonList(tags.get(0)), getObjectTaggingResult.getTagSet());

    // Tag the S3 version with the second S3 object tag.
    S3FileTransferRequestParamsDto params = new S3FileTransferRequestParamsDto();
    params.setS3BucketName(MockS3OperationsImpl.MOCK_S3_BUCKET_NAME_VERSIONING_ENABLED);
    s3Dao.tagVersions(params, new S3FileTransferRequestParamsDto(), versionListing.getVersionSummaries(), tags.get(1));

    // Validate that the S3 object is now tagged with both tags.
    getObjectTaggingResult = s3Operations.getObjectTagging(
        new GetObjectTaggingRequest(MockS3OperationsImpl.MOCK_S3_BUCKET_NAME_VERSIONING_ENABLED, TARGET_S3_KEY,
            versionListing.getVersionSummaries().get(0).getVersionId()), null);
    assertEquals(tags.size(), getObjectTaggingResult.getTagSet().size());
    assertTrue(getObjectTaggingResult.getTagSet().containsAll(tags));
}
 

@Test
public void testDeleteDirectoryAssertHandleAmazonClientException()
{
    S3Operations originalS3Operations = (S3Operations) ReflectionTestUtils.getField(s3Dao, "s3Operations");
    S3Operations mockS3Operations = mock(S3Operations.class);
    ReflectionTestUtils.setField(s3Dao, "s3Operations", mockS3Operations);

    try
    {
        S3FileTransferRequestParamsDto s3FileTransferRequestParamsDto = new S3FileTransferRequestParamsDto();
        s3FileTransferRequestParamsDto.setS3BucketName("s3BucketName");
        s3FileTransferRequestParamsDto.setS3KeyPrefix("s3KeyPrefix");

        VersionListing versionListing = new VersionListing();
        versionListing.getVersionSummaries().add(new S3VersionSummary());
        when(mockS3Operations.listVersions(any(), any())).thenReturn(versionListing);
        when(mockS3Operations.deleteObjects(any(), any())).thenThrow(new AmazonClientException("message"));

        try
        {
            s3Dao.deleteDirectory(s3FileTransferRequestParamsDto);
            fail();
        }
        catch (Exception e)
        {
            assertEquals(IllegalStateException.class, e.getClass());
            assertEquals("Failed to delete keys/key versions with prefix \"s3KeyPrefix\" from bucket \"s3BucketName\". Reason: message", e.getMessage());
        }
    }
    finally
    {
        ReflectionTestUtils.setField(s3Dao, "s3Operations", originalS3Operations);
    }
}
 

@Override
public VersionListing listVersions(ListVersionsRequest listVersionsRequest) throws AmazonClientException,
    AmazonServiceException {
  // TODO Auto-generated method stub
  return null;
}
 

@Test
public void testListVersionsAssertKeyAndVersionIdMarker()
{
    S3Operations originalS3Operations = (S3Operations) ReflectionTestUtils.getField(s3Dao, "s3Operations");
    S3Operations mockS3Operations = mock(S3Operations.class);
    ReflectionTestUtils.setField(s3Dao, "s3Operations", mockS3Operations);

    try
    {
        String s3BucketName = "s3BucketName";
        String s3KeyPrefix = "s3KeyPrefix";
        String expectedKey = "key";
        String expectedVersionId = "versionId";

        S3FileTransferRequestParamsDto s3FileTransferRequestParamsDto = new S3FileTransferRequestParamsDto();
        s3FileTransferRequestParamsDto.setS3BucketName(s3BucketName);
        s3FileTransferRequestParamsDto.setS3KeyPrefix(s3KeyPrefix);

        when(mockS3Operations.listVersions(any(), any())).then(new Answer<VersionListing>()
        {
            @Override
            public VersionListing answer(InvocationOnMock invocation) throws Throwable
            {
                ListVersionsRequest listVersionsRequest = invocation.getArgument(0);
                String keyMarker = listVersionsRequest.getKeyMarker();
                String versionIdMarker = listVersionsRequest.getVersionIdMarker();

                VersionListing versionListing = new VersionListing();
                if (keyMarker == null || versionIdMarker == null)
                {
                    versionListing.setTruncated(true);
                    versionListing.setNextKeyMarker("nextKeyMarker");
                    versionListing.setNextVersionIdMarker("nextVersionIdMarker");
                }
                else
                {
                    assertEquals("nextKeyMarker", listVersionsRequest.getKeyMarker());
                    assertEquals("nextVersionIdMarker", listVersionsRequest.getVersionIdMarker());

                    S3VersionSummary s3VersionSummary = new S3VersionSummary();
                    s3VersionSummary.setKey(expectedKey);
                    s3VersionSummary.setVersionId(expectedVersionId);
                    versionListing.getVersionSummaries().add(s3VersionSummary);
                }
                return versionListing;
            }
        });

        List<S3VersionSummary> result = s3Dao.listVersions(s3FileTransferRequestParamsDto);
        assertEquals(1, result.size());
        assertEquals(expectedKey, result.get(0).getKey());
        assertEquals(expectedVersionId, result.get(0).getVersionId());
    }
    finally
    {
        ReflectionTestUtils.setField(s3Dao, "s3Operations", originalS3Operations);
    }
}
 

@Override
public VersionListing listVersions(String bucketName, String prefix, String keyMarker, String versionIdMarker,
    String delimiter, Integer maxResults) throws AmazonClientException, AmazonServiceException {
  // TODO Auto-generated method stub
  return null;
}
 

@Override
public VersionListing listNextBatchOfVersions(VersionListing previousVersionListing) throws AmazonClientException,
    AmazonServiceException {
  // TODO Auto-generated method stub
  return null;
}
 

@Override
public VersionListing listVersions(String bucketName, String prefix) throws AmazonClientException,
    AmazonServiceException {
  // TODO Auto-generated method stub
  return null;
}
 

@Test
public void testListVersions() {
    runner.setProperty(ListS3.REGION, "eu-west-1");
    runner.setProperty(ListS3.BUCKET, "test-bucket");
    runner.setProperty(ListS3.USE_VERSIONS, "true");

    Date lastModified = new Date();
    VersionListing versionListing = new VersionListing();
    S3VersionSummary versionSummary1 = new S3VersionSummary();
    versionSummary1.setBucketName("test-bucket");
    versionSummary1.setKey("test-key");
    versionSummary1.setVersionId("1");
    versionSummary1.setLastModified(lastModified);
    versionListing.getVersionSummaries().add(versionSummary1);
    S3VersionSummary versionSummary2 = new S3VersionSummary();
    versionSummary2.setBucketName("test-bucket");
    versionSummary2.setKey("test-key");
    versionSummary2.setVersionId("2");
    versionSummary2.setLastModified(lastModified);
    versionListing.getVersionSummaries().add(versionSummary2);
    Mockito.when(mockS3Client.listVersions(Mockito.any(ListVersionsRequest.class))).thenReturn(versionListing);

    runner.run();

    ArgumentCaptor<ListVersionsRequest> captureRequest = ArgumentCaptor.forClass(ListVersionsRequest.class);
    Mockito.verify(mockS3Client, Mockito.times(1)).listVersions(captureRequest.capture());
    ListVersionsRequest request = captureRequest.getValue();
    assertEquals("test-bucket", request.getBucketName());
    Mockito.verify(mockS3Client, Mockito.never()).listObjects(Mockito.any(ListObjectsRequest.class));

    runner.assertAllFlowFilesTransferred(ListS3.REL_SUCCESS, 2);
    List<MockFlowFile> flowFiles = runner.getFlowFilesForRelationship(ListS3.REL_SUCCESS);
    MockFlowFile ff0 = flowFiles.get(0);
    ff0.assertAttributeEquals("filename", "test-key");
    ff0.assertAttributeEquals("s3.bucket", "test-bucket");
    ff0.assertAttributeEquals("s3.lastModified", String.valueOf(lastModified.getTime()));
    ff0.assertAttributeEquals("s3.version", "1");
    MockFlowFile ff1 = flowFiles.get(1);
    ff1.assertAttributeEquals("filename", "test-key");
    ff1.assertAttributeEquals("s3.bucket", "test-bucket");
    ff1.assertAttributeEquals("s3.lastModified", String.valueOf(lastModified.getTime()));
    ff1.assertAttributeEquals("s3.version", "2");
}
 

@Override
public VersionListing listVersions() {
    versionListing = client.listVersions(listVersionsRequest);
    return versionListing;
}
 

/**
 * Delete an S3 bucket using the provided client. Coming from AWS documentation:
 * https://docs.aws.amazon.com/AmazonS3/latest/dev/delete-or-empty-bucket.html#delete-bucket-sdk-java
 * @param s3Client the AmazonS3 client instance used to delete the bucket
 * @param bucketName a String containing the bucket name
 */
public static void deleteBucket(AmazonS3 s3Client, String bucketName) {
    // Delete all objects from the bucket. This is sufficient
    // for non versioned buckets. For versioned buckets, when you attempt to delete objects, Amazon S3 inserts
    // delete markers for all objects, but doesn't delete the object versions.
    // To delete objects from versioned buckets, delete all of the object versions before deleting
    // the bucket (see below for an example).
    ObjectListing objectListing = s3Client.listObjects(bucketName);
    while (true) {
        Iterator<S3ObjectSummary> objIter = objectListing.getObjectSummaries().iterator();
        while (objIter.hasNext()) {
            s3Client.deleteObject(bucketName, objIter.next().getKey());
        }

        // If the bucket contains many objects, the listObjects() call
        // might not return all of the objects in the first listing. Check to
        // see whether the listing was truncated. If so, retrieve the next page of objects
        // and delete them.
        if (objectListing.isTruncated()) {
            objectListing = s3Client.listNextBatchOfObjects(objectListing);
        } else {
            break;
        }
    }

    // Delete all object versions (required for versioned buckets).
    VersionListing versionList = s3Client.listVersions(new ListVersionsRequest().withBucketName(bucketName));
    while (true) {
        Iterator<S3VersionSummary> versionIter = versionList.getVersionSummaries().iterator();
        while (versionIter.hasNext()) {
            S3VersionSummary vs = versionIter.next();
            s3Client.deleteVersion(bucketName, vs.getKey(), vs.getVersionId());
        }

        if (versionList.isTruncated()) {
            versionList = s3Client.listNextBatchOfVersions(versionList);
        } else {
            break;
        }
    }

    // After all objects and object versions are deleted, delete the bucket.
    s3Client.deleteBucket(bucketName);
}
 

@Override
public void onTrigger(final ProcessContext context, final ProcessSession session) {
    try {
        restoreState(context);
    } catch (IOException ioe) {
        getLogger().error("Failed to restore processor state; yielding", ioe);
        context.yield();
        return;
    }

    final long startNanos = System.nanoTime();
    final String bucket = context.getProperty(BUCKET).evaluateAttributeExpressions().getValue();

    final AmazonS3 client = getClient();
    int listCount = 0;
    long maxTimestamp = 0L;
    String delimiter = context.getProperty(DELIMITER).getValue();
    String prefix = context.getProperty(PREFIX).evaluateAttributeExpressions().getValue();

    boolean useVersions = context.getProperty(USE_VERSIONS).asBoolean();

    S3BucketLister bucketLister = useVersions
            ? new S3VersionBucketLister(client)
            : new S3ObjectBucketLister(client);

    bucketLister.setBucketName(bucket);

    if (delimiter != null && !delimiter.isEmpty()) {
        bucketLister.setDelimiter(delimiter);
    }
    if (prefix != null && !prefix.isEmpty()) {
        bucketLister.setPrefix(prefix);
    }

    VersionListing versionListing;
    do {
        versionListing = bucketLister.listVersions();
        for (S3VersionSummary versionSummary : versionListing.getVersionSummaries()) {
            long lastModified = versionSummary.getLastModified().getTime();
            if (lastModified < currentTimestamp
                    || lastModified == currentTimestamp && currentKeys.contains(versionSummary.getKey())) {
                continue;
            }

            // Create the attributes
            final Map<String, String> attributes = new HashMap<>();
            attributes.put(CoreAttributes.FILENAME.key(), versionSummary.getKey());
            attributes.put("s3.bucket", versionSummary.getBucketName());
            if (versionSummary.getOwner() != null) { // We may not have permission to read the owner
                attributes.put("s3.owner", versionSummary.getOwner().getId());
            }
            attributes.put("s3.etag", versionSummary.getETag());
            attributes.put("s3.lastModified", String.valueOf(lastModified));
            attributes.put("s3.length", String.valueOf(versionSummary.getSize()));
            attributes.put("s3.storeClass", versionSummary.getStorageClass());
            attributes.put("s3.isLatest", String.valueOf(versionSummary.isLatest()));
            if (versionSummary.getVersionId() != null) {
                attributes.put("s3.version", versionSummary.getVersionId());
            }

            // Create the flowfile
            FlowFile flowFile = session.create();
            flowFile = session.putAllAttributes(flowFile, attributes);
            session.transfer(flowFile, REL_SUCCESS);

            // Update state
            if (lastModified > maxTimestamp) {
                maxTimestamp = lastModified;
                currentKeys.clear();
            }
            if (lastModified == maxTimestamp) {
                currentKeys.add(versionSummary.getKey());
            }
            listCount++;
        }
        bucketLister.setNextMarker();

        commit(context, session, listCount);
        listCount = 0;
    } while (bucketLister.isTruncated());
    currentTimestamp = maxTimestamp;

    final long listMillis = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startNanos);
    getLogger().info("Successfully listed S3 bucket {} in {} millis", new Object[]{bucket, listMillis});

    if (!commit(context, session, listCount)) {
        if (currentTimestamp > 0) {
            persistState(context);
        }
        getLogger().debug("No new objects in S3 bucket {} to list. Yielding.", new Object[]{bucket});
        context.yield();
    }
}
 

private void delete(VersionListing versionListing) {
    for (S3VersionSummary versionSummary : versionListing.getVersionSummaries()) {
        s3.deleteVersion(versionSummary.getBucketName(), versionSummary.getKey(), versionSummary.getVersionId());
    }
}
 

/** Unsupported Operation. */
@Override public VersionListing listVersions(ListVersionsRequest listVersionsReq) throws SdkClientException {
    throw new UnsupportedOperationException("Operation not supported");
}
 

/** Unsupported Operation. */
@Override public VersionListing listVersions(String bucketName, String prefix, String keyMarker, String verIdMarker,
    String delim, Integer maxResults) throws SdkClientException {
    throw new UnsupportedOperationException("Operation not supported");
}
 

/** Unsupported Operation. */
@Override public VersionListing listNextBatchOfVersions(ListNextBatchOfVersionsRequest listNextBatchOfVersionsReq)
    throws SdkClientException {
    throw new UnsupportedOperationException("Operation not supported");
}
 

/** Unsupported Operation. */
@Override public VersionListing listNextBatchOfVersions(VersionListing previousVerListing) throws SdkClientException {
    throw new UnsupportedOperationException("Operation not supported");
}
 

/** Unsupported Operation. */
@Override public VersionListing listVersions(String bucketName, String prefix) throws SdkClientException {
    throw new UnsupportedOperationException("Operation not supported");
}
 

@Override
public VersionListing listVersions() {
    versionListing = client.listVersions(listVersionsRequest);
    return versionListing;
}
 

@Test
public void testDeleteDirectoryMultiObjectDeleteException()
{
    // Create an S3 file transfer request parameters DTO to access S3 objects.
    S3FileTransferRequestParamsDto s3FileTransferRequestParamsDto = new S3FileTransferRequestParamsDto();
    s3FileTransferRequestParamsDto.setS3BucketName(S3_BUCKET_NAME);
    s3FileTransferRequestParamsDto.setS3KeyPrefix(S3_KEY_PREFIX);

    // Create a retry policy.
    RetryPolicy retryPolicy =
        new RetryPolicy(PredefinedRetryPolicies.DEFAULT_RETRY_CONDITION, PredefinedRetryPolicies.DEFAULT_BACKOFF_STRATEGY, INTEGER_VALUE, true);

    // Create an S3 version summary.
    S3VersionSummary s3VersionSummary = new S3VersionSummary();
    s3VersionSummary.setKey(S3_KEY);
    s3VersionSummary.setVersionId(S3_VERSION_ID);

    // Create a version listing.
    VersionListing versionListing = new VersionListing();
    versionListing.setVersionSummaries(Collections.singletonList(s3VersionSummary));

    // Create a delete error.
    MultiObjectDeleteException.DeleteError deleteError = new MultiObjectDeleteException.DeleteError();
    deleteError.setKey(S3_KEY);
    deleteError.setVersionId(S3_VERSION_ID);
    deleteError.setCode(ERROR_CODE);
    deleteError.setMessage(ERROR_MESSAGE);

    // Create a multi object delete exception.
    MultiObjectDeleteException multiObjectDeleteException = new MultiObjectDeleteException(Collections.singletonList(deleteError), new ArrayList<>());

    // Mock the external calls.
    when(retryPolicyFactory.getRetryPolicy()).thenReturn(retryPolicy);
    when(s3Operations.listVersions(any(ListVersionsRequest.class), any(AmazonS3Client.class))).thenReturn(versionListing);
    when(s3Operations.deleteObjects(any(DeleteObjectsRequest.class), any(AmazonS3Client.class))).thenThrow(multiObjectDeleteException);

    // Try to call the method under test.
    try
    {
        s3DaoImpl.deleteDirectory(s3FileTransferRequestParamsDto);
    }
    catch (IllegalStateException e)
    {
        assertEquals(String.format(
            "Failed to delete keys/key versions with prefix \"%s\" from bucket \"%s\". Reason: One or more objects could not be deleted " +
                "(Service: null; Status Code: 0; Error Code: null; Request ID: null; S3 Extended Request ID: null)", S3_KEY_PREFIX, S3_BUCKET_NAME),
            e.getMessage());
    }

    // Verify the external calls.
    verify(retryPolicyFactory, times(2)).getRetryPolicy();
    verify(s3Operations).listVersions(any(ListVersionsRequest.class), any(AmazonS3Client.class));
    verify(s3Operations).deleteObjects(any(DeleteObjectsRequest.class), any(AmazonS3Client.class));
    verifyNoMoreInteractionsHelper();
}
 

/**
 * {@inheritDoc}
 * <p/>
 * If the bucket does not exist, returns a listing with an empty list. If a prefix is specified in listVersionsRequest, only versions starting with the
 * prefix will be returned.
 */
@Override
public VersionListing listVersions(ListVersionsRequest listVersionsRequest, AmazonS3 s3Client)
{
    LOGGER.debug("listVersions(): listVersionsRequest.getBucketName() = " + listVersionsRequest.getBucketName());

    String bucketName = listVersionsRequest.getBucketName();

    if (MOCK_S3_BUCKET_NAME_NO_SUCH_BUCKET_EXCEPTION.equals(bucketName))
    {
        AmazonS3Exception amazonS3Exception = new AmazonS3Exception(MOCK_S3_BUCKET_NAME_NO_SUCH_BUCKET_EXCEPTION);
        amazonS3Exception.setErrorCode("NoSuchBucket");
        throw amazonS3Exception;
    }
    else if (MOCK_S3_BUCKET_NAME_INTERNAL_ERROR.equals(bucketName))
    {
        throw new AmazonServiceException(S3Operations.ERROR_CODE_INTERNAL_ERROR);
    }

    VersionListing versionListing = new VersionListing();
    versionListing.setBucketName(bucketName);

    MockS3Bucket mockS3Bucket = mockS3Buckets.get(bucketName);
    if (mockS3Bucket != null)
    {
        for (MockS3Object mockS3Object : mockS3Bucket.getVersions().values())
        {
            String s3ObjectKey = mockS3Object.getKey();
            if (listVersionsRequest.getPrefix() == null || s3ObjectKey.startsWith(listVersionsRequest.getPrefix()))
            {
                S3VersionSummary s3VersionSummary = new S3VersionSummary();
                s3VersionSummary.setBucketName(bucketName);
                s3VersionSummary.setKey(s3ObjectKey);
                s3VersionSummary.setVersionId(mockS3Object.getVersion());
                s3VersionSummary.setSize(mockS3Object.getData().length);
                s3VersionSummary.setStorageClass(mockS3Object.getObjectMetadata() != null ? mockS3Object.getObjectMetadata().getStorageClass() : null);

                versionListing.getVersionSummaries().add(s3VersionSummary);
            }
        }
    }

    return versionListing;
}
 

@Override
public VersionListing listVersions(ListVersionsRequest listVersionsRequest, AmazonS3 s3Client)
{
    return s3Client.listVersions(listVersionsRequest);
}