com.amazonaws.services.s3.model.ListVersionsRequest Java Examples

The following examples show how to use com.amazonaws.services.s3.model.ListVersionsRequest. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: S3DaoImplTest.java    From herd with Apache License 2.0 5 votes vote down vote up
@Test
public void testDeleteDirectoryNoS3VersionsExist()
{
    // Create an S3 file transfer request parameters DTO to access S3 objects.
    S3FileTransferRequestParamsDto s3FileTransferRequestParamsDto = new S3FileTransferRequestParamsDto();
    s3FileTransferRequestParamsDto.setS3BucketName(S3_BUCKET_NAME);
    s3FileTransferRequestParamsDto.setS3KeyPrefix(S3_KEY_PREFIX);

    // Create a retry policy.
    RetryPolicy retryPolicy =
        new RetryPolicy(PredefinedRetryPolicies.DEFAULT_RETRY_CONDITION, PredefinedRetryPolicies.DEFAULT_BACKOFF_STRATEGY, INTEGER_VALUE, true);

    // Create an empty version listing.
    VersionListing versionListing = new VersionListing();

    // Mock the external calls.
    when(retryPolicyFactory.getRetryPolicy()).thenReturn(retryPolicy);
    when(s3Operations.listVersions(any(ListVersionsRequest.class), any(AmazonS3Client.class))).thenReturn(versionListing);

    // Call the method under test.
    s3DaoImpl.deleteDirectory(s3FileTransferRequestParamsDto);

    // Verify the external calls.
    verify(retryPolicyFactory).getRetryPolicy();
    verify(s3Operations).listVersions(any(ListVersionsRequest.class), any(AmazonS3Client.class));
    verifyNoMoreInteractionsHelper();
}
 
Example #2
Source File: S3DaoTest.java    From herd with Apache License 2.0 5 votes vote down vote up
@Test
public void testTagVersionsOtherTagKeyAlreadyExists()
{
    // Create two S3 object tags having different tag keys.
    List<Tag> tags = Arrays.asList(new Tag(S3_OBJECT_TAG_KEY, S3_OBJECT_TAG_VALUE), new Tag(S3_OBJECT_TAG_KEY_2, S3_OBJECT_TAG_VALUE_2));

    // Put an S3 file that is already tagged with the first S3 object tag in an S3 bucket that has versioning enabled.
    PutObjectRequest putObjectRequest =
        new PutObjectRequest(MockS3OperationsImpl.MOCK_S3_BUCKET_NAME_VERSIONING_ENABLED, TARGET_S3_KEY, new ByteArrayInputStream(new byte[1]),
            new ObjectMetadata());
    putObjectRequest.setTagging(new ObjectTagging(Collections.singletonList(tags.get(0))));
    s3Operations.putObject(putObjectRequest, null);

    // List S3 versions that match the test S3 key.
    ListVersionsRequest listVersionsRequest =
        new ListVersionsRequest().withBucketName(MockS3OperationsImpl.MOCK_S3_BUCKET_NAME_VERSIONING_ENABLED).withPrefix(TARGET_S3_KEY);
    VersionListing versionListing = s3Operations.listVersions(listVersionsRequest, null);
    assertEquals(1, CollectionUtils.size(versionListing.getVersionSummaries()));
    assertEquals(TARGET_S3_KEY, versionListing.getVersionSummaries().get(0).getKey());
    assertNotNull(versionListing.getVersionSummaries().get(0).getVersionId());

    // Validate that the S3 object is tagged with the first tag only.
    GetObjectTaggingResult getObjectTaggingResult = s3Operations.getObjectTagging(
        new GetObjectTaggingRequest(MockS3OperationsImpl.MOCK_S3_BUCKET_NAME_VERSIONING_ENABLED, TARGET_S3_KEY,
            versionListing.getVersionSummaries().get(0).getVersionId()), null);
    assertEquals(Collections.singletonList(tags.get(0)), getObjectTaggingResult.getTagSet());

    // Tag the S3 version with the second S3 object tag.
    S3FileTransferRequestParamsDto params = new S3FileTransferRequestParamsDto();
    params.setS3BucketName(MockS3OperationsImpl.MOCK_S3_BUCKET_NAME_VERSIONING_ENABLED);
    s3Dao.tagVersions(params, new S3FileTransferRequestParamsDto(), versionListing.getVersionSummaries(), tags.get(1));

    // Validate that the S3 object is now tagged with both tags.
    getObjectTaggingResult = s3Operations.getObjectTagging(
        new GetObjectTaggingRequest(MockS3OperationsImpl.MOCK_S3_BUCKET_NAME_VERSIONING_ENABLED, TARGET_S3_KEY,
            versionListing.getVersionSummaries().get(0).getVersionId()), null);
    assertEquals(tags.size(), getObjectTaggingResult.getTagSet().size());
    assertTrue(getObjectTaggingResult.getTagSet().containsAll(tags));
}
 
Example #3
Source File: S3DaoTest.java    From herd with Apache License 2.0 5 votes vote down vote up
@Test
public void testTagVersionsS3BucketWithVersioningDisabled()
{
    // Create an S3 object tag.
    Tag tag = new Tag(S3_OBJECT_TAG_KEY, S3_OBJECT_TAG_VALUE);

    // Put S3 objects in S3 bucket that has versioning disabled.
    for (int i = 0; i < 2; i++)
    {
        s3Operations.putObject(new PutObjectRequest(S3_BUCKET_NAME, TARGET_S3_KEY + i, new ByteArrayInputStream(new byte[1]), new ObjectMetadata()), null);
    }

    // List S3 versions that match the test prefix.
    ListVersionsRequest listVersionsRequest = new ListVersionsRequest().withBucketName(S3_BUCKET_NAME).withPrefix(TARGET_S3_KEY);
    VersionListing versionListing = s3Operations.listVersions(listVersionsRequest, null);
    assertEquals(2, CollectionUtils.size(versionListing.getVersionSummaries()));
    for (int i = 0; i < 2; i++)
    {
        assertNull(versionListing.getVersionSummaries().get(i).getVersionId());
    }

    // Create an S3 file transfer request parameters DTO to access S3 objects.
    S3FileTransferRequestParamsDto params = new S3FileTransferRequestParamsDto();
    params.setS3BucketName(S3_BUCKET_NAME);

    // Tag listed S3 versions with an S3 object tag.
    s3Dao.tagVersions(params, new S3FileTransferRequestParamsDto(), versionListing.getVersionSummaries(), tag);

    // Validate that both S3 objects got tagged.
    for (int i = 0; i < 2; i++)
    {
        GetObjectTaggingResult getObjectTaggingResult =
            s3Operations.getObjectTagging(new GetObjectTaggingRequest(S3_BUCKET_NAME, TARGET_S3_KEY + i, null), null);
        assertEquals(Collections.singletonList(tag), getObjectTaggingResult.getTagSet());
    }
}
 
Example #4
Source File: S3DaoTest.java    From herd with Apache License 2.0 5 votes vote down vote up
@Test
public void testTagVersionsS3BucketWithVersioningEnabled()
{
    // Create an S3 object tag.
    Tag tag = new Tag(S3_OBJECT_TAG_KEY, S3_OBJECT_TAG_VALUE);

    // Put two S3 versions in S3 bucket that has versioning enabled.
    for (int i = 0; i < 2; i++)
    {
        s3Operations.putObject(
            new PutObjectRequest(MockS3OperationsImpl.MOCK_S3_BUCKET_NAME_VERSIONING_ENABLED, TARGET_S3_KEY, new ByteArrayInputStream(new byte[1]),
                new ObjectMetadata()), null);
    }

    // List S3 versions that match the test key.
    ListVersionsRequest listVersionsRequest =
        new ListVersionsRequest().withBucketName(MockS3OperationsImpl.MOCK_S3_BUCKET_NAME_VERSIONING_ENABLED).withPrefix(TARGET_S3_KEY);
    VersionListing versionListing = s3Operations.listVersions(listVersionsRequest, null);
    assertEquals(2, CollectionUtils.size(versionListing.getVersionSummaries()));
    for (int i = 0; i < 2; i++)
    {
        assertNotNull(versionListing.getVersionSummaries().get(i).getVersionId());
    }

    // Create an S3 file transfer request parameters DTO to access S3 objects with a mocked S3 bucket name that enables S3 bucket versioning.
    S3FileTransferRequestParamsDto params = new S3FileTransferRequestParamsDto();
    params.setS3BucketName(MockS3OperationsImpl.MOCK_S3_BUCKET_NAME_VERSIONING_ENABLED);

    // Tag listed S3 version with an S3 object tag.
    s3Dao.tagVersions(params, new S3FileTransferRequestParamsDto(), versionListing.getVersionSummaries(), tag);

    // Validate that both versions got tagged.
    for (int i = 0; i < 2; i++)
    {
        GetObjectTaggingResult getObjectTaggingResult = s3Operations.getObjectTagging(
            new GetObjectTaggingRequest(MockS3OperationsImpl.MOCK_S3_BUCKET_NAME_VERSIONING_ENABLED, TARGET_S3_KEY,
                versionListing.getVersionSummaries().get(i).getVersionId()), null);
        assertEquals(Collections.singletonList(tag), getObjectTaggingResult.getTagSet());
    }
}
 
Example #5
Source File: S3DaoTest.java    From herd with Apache License 2.0 5 votes vote down vote up
@Test
public void testTagVersionsTargetTagKeyAlreadyExists()
{
    // Create two S3 object tags having the same tag key.
    List<Tag> tags = Arrays.asList(new Tag(S3_OBJECT_TAG_KEY, S3_OBJECT_TAG_VALUE), new Tag(S3_OBJECT_TAG_KEY, S3_OBJECT_TAG_VALUE_2));

    // Put an S3 file that is already tagged with the first S3 object tag in an S3 bucket that has versioning enabled.
    PutObjectRequest putObjectRequest =
        new PutObjectRequest(MockS3OperationsImpl.MOCK_S3_BUCKET_NAME_VERSIONING_ENABLED, TARGET_S3_KEY, new ByteArrayInputStream(new byte[1]),
            new ObjectMetadata());
    putObjectRequest.setTagging(new ObjectTagging(Collections.singletonList(tags.get(0))));
    s3Operations.putObject(putObjectRequest, null);

    // List S3 versions that match the test S3 key.
    ListVersionsRequest listVersionsRequest =
        new ListVersionsRequest().withBucketName(MockS3OperationsImpl.MOCK_S3_BUCKET_NAME_VERSIONING_ENABLED).withPrefix(TARGET_S3_KEY);
    VersionListing versionListing = s3Operations.listVersions(listVersionsRequest, null);
    assertEquals(1, CollectionUtils.size(versionListing.getVersionSummaries()));
    assertEquals(TARGET_S3_KEY, versionListing.getVersionSummaries().get(0).getKey());
    assertNotNull(versionListing.getVersionSummaries().get(0).getVersionId());

    // Validate that the S3 object is tagged with the first tag only.
    GetObjectTaggingResult getObjectTaggingResult = s3Operations.getObjectTagging(
        new GetObjectTaggingRequest(MockS3OperationsImpl.MOCK_S3_BUCKET_NAME_VERSIONING_ENABLED, TARGET_S3_KEY,
            versionListing.getVersionSummaries().get(0).getVersionId()), null);
    assertEquals(Collections.singletonList(tags.get(0)), getObjectTaggingResult.getTagSet());

    // Create an S3 file transfer request parameters DTO to access S3 objects.
    S3FileTransferRequestParamsDto params = new S3FileTransferRequestParamsDto();
    params.setS3BucketName(MockS3OperationsImpl.MOCK_S3_BUCKET_NAME_VERSIONING_ENABLED);

    // Tag the S3 version with the second S3 object tag.
    s3Dao.tagVersions(params, new S3FileTransferRequestParamsDto(), versionListing.getVersionSummaries(), tags.get(1));

    // Validate that the S3 object is now tagged with the second tag only.
    getObjectTaggingResult = s3Operations.getObjectTagging(
        new GetObjectTaggingRequest(MockS3OperationsImpl.MOCK_S3_BUCKET_NAME_VERSIONING_ENABLED, TARGET_S3_KEY,
            versionListing.getVersionSummaries().get(0).getVersionId()), null);
    assertEquals(Collections.singletonList(tags.get(1)), getObjectTaggingResult.getTagSet());
}
 
Example #6
Source File: AmazonS3Mock.java    From Scribengin with GNU Affero General Public License v3.0 4 votes vote down vote up
@Override
public VersionListing listVersions(ListVersionsRequest listVersionsRequest) throws AmazonClientException,
    AmazonServiceException {
  // TODO Auto-generated method stub
  return null;
}
 
Example #7
Source File: AWSCommon.java    From camel-kafka-connector with Apache License 2.0 4 votes vote down vote up
/**
 * Delete an S3 bucket using the provided client. Coming from AWS documentation:
 * https://docs.aws.amazon.com/AmazonS3/latest/dev/delete-or-empty-bucket.html#delete-bucket-sdk-java
 * @param s3Client the AmazonS3 client instance used to delete the bucket
 * @param bucketName a String containing the bucket name
 */
public static void deleteBucket(AmazonS3 s3Client, String bucketName) {
    // Delete all objects from the bucket. This is sufficient
    // for non versioned buckets. For versioned buckets, when you attempt to delete objects, Amazon S3 inserts
    // delete markers for all objects, but doesn't delete the object versions.
    // To delete objects from versioned buckets, delete all of the object versions before deleting
    // the bucket (see below for an example).
    ObjectListing objectListing = s3Client.listObjects(bucketName);
    while (true) {
        Iterator<S3ObjectSummary> objIter = objectListing.getObjectSummaries().iterator();
        while (objIter.hasNext()) {
            s3Client.deleteObject(bucketName, objIter.next().getKey());
        }

        // If the bucket contains many objects, the listObjects() call
        // might not return all of the objects in the first listing. Check to
        // see whether the listing was truncated. If so, retrieve the next page of objects
        // and delete them.
        if (objectListing.isTruncated()) {
            objectListing = s3Client.listNextBatchOfObjects(objectListing);
        } else {
            break;
        }
    }

    // Delete all object versions (required for versioned buckets).
    VersionListing versionList = s3Client.listVersions(new ListVersionsRequest().withBucketName(bucketName));
    while (true) {
        Iterator<S3VersionSummary> versionIter = versionList.getVersionSummaries().iterator();
        while (versionIter.hasNext()) {
            S3VersionSummary vs = versionIter.next();
            s3Client.deleteVersion(bucketName, vs.getKey(), vs.getVersionId());
        }

        if (versionList.isTruncated()) {
            versionList = s3Client.listNextBatchOfVersions(versionList);
        } else {
            break;
        }
    }

    // After all objects and object versions are deleted, delete the bucket.
    s3Client.deleteBucket(bucketName);
}
 
Example #8
Source File: TestListS3.java    From nifi with Apache License 2.0 4 votes vote down vote up
@Test
public void testListVersions() {
    runner.setProperty(ListS3.REGION, "eu-west-1");
    runner.setProperty(ListS3.BUCKET, "test-bucket");
    runner.setProperty(ListS3.USE_VERSIONS, "true");

    Date lastModified = new Date();
    VersionListing versionListing = new VersionListing();
    S3VersionSummary versionSummary1 = new S3VersionSummary();
    versionSummary1.setBucketName("test-bucket");
    versionSummary1.setKey("test-key");
    versionSummary1.setVersionId("1");
    versionSummary1.setLastModified(lastModified);
    versionListing.getVersionSummaries().add(versionSummary1);
    S3VersionSummary versionSummary2 = new S3VersionSummary();
    versionSummary2.setBucketName("test-bucket");
    versionSummary2.setKey("test-key");
    versionSummary2.setVersionId("2");
    versionSummary2.setLastModified(lastModified);
    versionListing.getVersionSummaries().add(versionSummary2);
    Mockito.when(mockS3Client.listVersions(Mockito.any(ListVersionsRequest.class))).thenReturn(versionListing);

    runner.run();

    ArgumentCaptor<ListVersionsRequest> captureRequest = ArgumentCaptor.forClass(ListVersionsRequest.class);
    Mockito.verify(mockS3Client, Mockito.times(1)).listVersions(captureRequest.capture());
    ListVersionsRequest request = captureRequest.getValue();
    assertEquals("test-bucket", request.getBucketName());
    Mockito.verify(mockS3Client, Mockito.never()).listObjects(Mockito.any(ListObjectsRequest.class));

    runner.assertAllFlowFilesTransferred(ListS3.REL_SUCCESS, 2);
    List<MockFlowFile> flowFiles = runner.getFlowFilesForRelationship(ListS3.REL_SUCCESS);
    MockFlowFile ff0 = flowFiles.get(0);
    ff0.assertAttributeEquals("filename", "test-key");
    ff0.assertAttributeEquals("s3.bucket", "test-bucket");
    ff0.assertAttributeEquals("s3.lastModified", String.valueOf(lastModified.getTime()));
    ff0.assertAttributeEquals("s3.version", "1");
    MockFlowFile ff1 = flowFiles.get(1);
    ff1.assertAttributeEquals("filename", "test-key");
    ff1.assertAttributeEquals("s3.bucket", "test-bucket");
    ff1.assertAttributeEquals("s3.lastModified", String.valueOf(lastModified.getTime()));
    ff1.assertAttributeEquals("s3.version", "2");
}
 
Example #9
Source File: ListS3.java    From nifi with Apache License 2.0 4 votes vote down vote up
@Override
public void setBucketName(String bucketName) {
    listVersionsRequest = new ListVersionsRequest().withBucketName(bucketName);
}
 
Example #10
Source File: DummyS3Client.java    From ignite with Apache License 2.0 4 votes vote down vote up
/** Unsupported Operation. */
@Override public VersionListing listVersions(ListVersionsRequest listVersionsReq) throws SdkClientException {
    throw new UnsupportedOperationException("Operation not supported");
}
 
Example #11
Source File: S3DaoImplTest.java    From herd with Apache License 2.0 4 votes vote down vote up
@Test
public void testDeleteDirectoryMultiObjectDeleteException()
{
    // Create an S3 file transfer request parameters DTO to access S3 objects.
    S3FileTransferRequestParamsDto s3FileTransferRequestParamsDto = new S3FileTransferRequestParamsDto();
    s3FileTransferRequestParamsDto.setS3BucketName(S3_BUCKET_NAME);
    s3FileTransferRequestParamsDto.setS3KeyPrefix(S3_KEY_PREFIX);

    // Create a retry policy.
    RetryPolicy retryPolicy =
        new RetryPolicy(PredefinedRetryPolicies.DEFAULT_RETRY_CONDITION, PredefinedRetryPolicies.DEFAULT_BACKOFF_STRATEGY, INTEGER_VALUE, true);

    // Create an S3 version summary.
    S3VersionSummary s3VersionSummary = new S3VersionSummary();
    s3VersionSummary.setKey(S3_KEY);
    s3VersionSummary.setVersionId(S3_VERSION_ID);

    // Create a version listing.
    VersionListing versionListing = new VersionListing();
    versionListing.setVersionSummaries(Collections.singletonList(s3VersionSummary));

    // Create a delete error.
    MultiObjectDeleteException.DeleteError deleteError = new MultiObjectDeleteException.DeleteError();
    deleteError.setKey(S3_KEY);
    deleteError.setVersionId(S3_VERSION_ID);
    deleteError.setCode(ERROR_CODE);
    deleteError.setMessage(ERROR_MESSAGE);

    // Create a multi object delete exception.
    MultiObjectDeleteException multiObjectDeleteException = new MultiObjectDeleteException(Collections.singletonList(deleteError), new ArrayList<>());

    // Mock the external calls.
    when(retryPolicyFactory.getRetryPolicy()).thenReturn(retryPolicy);
    when(s3Operations.listVersions(any(ListVersionsRequest.class), any(AmazonS3Client.class))).thenReturn(versionListing);
    when(s3Operations.deleteObjects(any(DeleteObjectsRequest.class), any(AmazonS3Client.class))).thenThrow(multiObjectDeleteException);

    // Try to call the method under test.
    try
    {
        s3DaoImpl.deleteDirectory(s3FileTransferRequestParamsDto);
    }
    catch (IllegalStateException e)
    {
        assertEquals(String.format(
            "Failed to delete keys/key versions with prefix \"%s\" from bucket \"%s\". Reason: One or more objects could not be deleted " +
                "(Service: null; Status Code: 0; Error Code: null; Request ID: null; S3 Extended Request ID: null)", S3_KEY_PREFIX, S3_BUCKET_NAME),
            e.getMessage());
    }

    // Verify the external calls.
    verify(retryPolicyFactory, times(2)).getRetryPolicy();
    verify(s3Operations).listVersions(any(ListVersionsRequest.class), any(AmazonS3Client.class));
    verify(s3Operations).deleteObjects(any(DeleteObjectsRequest.class), any(AmazonS3Client.class));
    verifyNoMoreInteractionsHelper();
}
 
Example #12
Source File: MockS3OperationsImpl.java    From herd with Apache License 2.0 4 votes vote down vote up
/**
 * {@inheritDoc}
 * <p/>
 * If the bucket does not exist, returns a listing with an empty list. If a prefix is specified in listVersionsRequest, only versions starting with the
 * prefix will be returned.
 */
@Override
public VersionListing listVersions(ListVersionsRequest listVersionsRequest, AmazonS3 s3Client)
{
    LOGGER.debug("listVersions(): listVersionsRequest.getBucketName() = " + listVersionsRequest.getBucketName());

    String bucketName = listVersionsRequest.getBucketName();

    if (MOCK_S3_BUCKET_NAME_NO_SUCH_BUCKET_EXCEPTION.equals(bucketName))
    {
        AmazonS3Exception amazonS3Exception = new AmazonS3Exception(MOCK_S3_BUCKET_NAME_NO_SUCH_BUCKET_EXCEPTION);
        amazonS3Exception.setErrorCode("NoSuchBucket");
        throw amazonS3Exception;
    }
    else if (MOCK_S3_BUCKET_NAME_INTERNAL_ERROR.equals(bucketName))
    {
        throw new AmazonServiceException(S3Operations.ERROR_CODE_INTERNAL_ERROR);
    }

    VersionListing versionListing = new VersionListing();
    versionListing.setBucketName(bucketName);

    MockS3Bucket mockS3Bucket = mockS3Buckets.get(bucketName);
    if (mockS3Bucket != null)
    {
        for (MockS3Object mockS3Object : mockS3Bucket.getVersions().values())
        {
            String s3ObjectKey = mockS3Object.getKey();
            if (listVersionsRequest.getPrefix() == null || s3ObjectKey.startsWith(listVersionsRequest.getPrefix()))
            {
                S3VersionSummary s3VersionSummary = new S3VersionSummary();
                s3VersionSummary.setBucketName(bucketName);
                s3VersionSummary.setKey(s3ObjectKey);
                s3VersionSummary.setVersionId(mockS3Object.getVersion());
                s3VersionSummary.setSize(mockS3Object.getData().length);
                s3VersionSummary.setStorageClass(mockS3Object.getObjectMetadata() != null ? mockS3Object.getObjectMetadata().getStorageClass() : null);

                versionListing.getVersionSummaries().add(s3VersionSummary);
            }
        }
    }

    return versionListing;
}
 
Example #13
Source File: S3DaoTest.java    From herd with Apache License 2.0 4 votes vote down vote up
@Test
public void testListVersionsAssertKeyAndVersionIdMarker()
{
    S3Operations originalS3Operations = (S3Operations) ReflectionTestUtils.getField(s3Dao, "s3Operations");
    S3Operations mockS3Operations = mock(S3Operations.class);
    ReflectionTestUtils.setField(s3Dao, "s3Operations", mockS3Operations);

    try
    {
        String s3BucketName = "s3BucketName";
        String s3KeyPrefix = "s3KeyPrefix";
        String expectedKey = "key";
        String expectedVersionId = "versionId";

        S3FileTransferRequestParamsDto s3FileTransferRequestParamsDto = new S3FileTransferRequestParamsDto();
        s3FileTransferRequestParamsDto.setS3BucketName(s3BucketName);
        s3FileTransferRequestParamsDto.setS3KeyPrefix(s3KeyPrefix);

        when(mockS3Operations.listVersions(any(), any())).then(new Answer<VersionListing>()
        {
            @Override
            public VersionListing answer(InvocationOnMock invocation) throws Throwable
            {
                ListVersionsRequest listVersionsRequest = invocation.getArgument(0);
                String keyMarker = listVersionsRequest.getKeyMarker();
                String versionIdMarker = listVersionsRequest.getVersionIdMarker();

                VersionListing versionListing = new VersionListing();
                if (keyMarker == null || versionIdMarker == null)
                {
                    versionListing.setTruncated(true);
                    versionListing.setNextKeyMarker("nextKeyMarker");
                    versionListing.setNextVersionIdMarker("nextVersionIdMarker");
                }
                else
                {
                    assertEquals("nextKeyMarker", listVersionsRequest.getKeyMarker());
                    assertEquals("nextVersionIdMarker", listVersionsRequest.getVersionIdMarker());

                    S3VersionSummary s3VersionSummary = new S3VersionSummary();
                    s3VersionSummary.setKey(expectedKey);
                    s3VersionSummary.setVersionId(expectedVersionId);
                    versionListing.getVersionSummaries().add(s3VersionSummary);
                }
                return versionListing;
            }
        });

        List<S3VersionSummary> result = s3Dao.listVersions(s3FileTransferRequestParamsDto);
        assertEquals(1, result.size());
        assertEquals(expectedKey, result.get(0).getKey());
        assertEquals(expectedVersionId, result.get(0).getVersionId());
    }
    finally
    {
        ReflectionTestUtils.setField(s3Dao, "s3Operations", originalS3Operations);
    }
}
 
Example #14
Source File: S3DaoImpl.java    From herd with Apache License 2.0 4 votes vote down vote up
@Override
public List<S3VersionSummary> listVersions(final S3FileTransferRequestParamsDto params)
{
    Assert.isTrue(!isRootKeyPrefix(params.getS3KeyPrefix()), "Listing of S3 versions from root directory is not allowed.");

    AmazonS3Client s3Client = getAmazonS3(params);
    List<S3VersionSummary> s3VersionSummaries = new ArrayList<>();

    try
    {
        ListVersionsRequest listVersionsRequest = new ListVersionsRequest().withBucketName(params.getS3BucketName()).withPrefix(params.getS3KeyPrefix());
        VersionListing versionListing;

        do
        {
            versionListing = s3Operations.listVersions(listVersionsRequest, s3Client);
            s3VersionSummaries.addAll(versionListing.getVersionSummaries());
            listVersionsRequest.setKeyMarker(versionListing.getNextKeyMarker());
            listVersionsRequest.setVersionIdMarker(versionListing.getNextVersionIdMarker());
        }
        while (versionListing.isTruncated());
    }
    catch (AmazonS3Exception amazonS3Exception)
    {
        if (S3Operations.ERROR_CODE_NO_SUCH_BUCKET.equals(amazonS3Exception.getErrorCode()))
        {
            throw new IllegalArgumentException("The specified bucket '" + params.getS3BucketName() + "' does not exist.", amazonS3Exception);
        }
        throw new IllegalStateException("Error accessing S3", amazonS3Exception);
    }
    catch (AmazonClientException e)
    {
        throw new IllegalStateException(String
            .format("Failed to list S3 versions with prefix \"%s\" from bucket \"%s\". Reason: %s", params.getS3KeyPrefix(), params.getS3BucketName(),
                e.getMessage()), e);
    }
    finally
    {
        // Shutdown the AmazonS3Client instance to release resources.
        s3Client.shutdown();
    }

    return s3VersionSummaries;
}
 
Example #15
Source File: S3OperationsImpl.java    From herd with Apache License 2.0 4 votes vote down vote up
@Override
public VersionListing listVersions(ListVersionsRequest listVersionsRequest, AmazonS3 s3Client)
{
    return s3Client.listVersions(listVersionsRequest);
}
 
Example #16
Source File: ListS3.java    From localization_nifi with Apache License 2.0 4 votes vote down vote up
@Override
public void setBucketName(String bucketName) {
    listVersionsRequest = new ListVersionsRequest().withBucketName(bucketName);
}
 
Example #17
Source File: S3Operations.java    From herd with Apache License 2.0 2 votes vote down vote up
/**
 * Returns a list of summary information about the versions in the specified bucket.
 *
 * @param listVersionsRequest the request object containing all options for listing the versions in a specified bucket
 * @param s3Client the {@link AmazonS3} implementation to use
 *
 * @return the listing of the versions in the specified bucket
 */
public VersionListing listVersions(ListVersionsRequest listVersionsRequest, AmazonS3 s3Client);