Java Code Examples for com.amazonaws.services.s3.model.S3VersionSummary#setKey()

The following examples show how to use com.amazonaws.services.s3.model.S3VersionSummary#setKey() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: ListS3.java    From localization_nifi with Apache License 2.0 6 votes vote down vote up
@Override
public VersionListing listVersions() {
    VersionListing versionListing = new VersionListing();
    this.objectListing = client.listObjects(listObjectsRequest);
    for(S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) {
        S3VersionSummary versionSummary = new S3VersionSummary();
        versionSummary.setBucketName(objectSummary.getBucketName());
        versionSummary.setETag(objectSummary.getETag());
        versionSummary.setKey(objectSummary.getKey());
        versionSummary.setLastModified(objectSummary.getLastModified());
        versionSummary.setOwner(objectSummary.getOwner());
        versionSummary.setSize(objectSummary.getSize());
        versionSummary.setStorageClass(objectSummary.getStorageClass());
        versionSummary.setIsLatest(true);

        versionListing.getVersionSummaries().add(versionSummary);
    }

    return versionListing;
}
 
Example 2
Source File: ListS3.java    From nifi with Apache License 2.0 6 votes vote down vote up
@Override
public VersionListing listVersions() {
    VersionListing versionListing = new VersionListing();
    this.objectListing = client.listObjects(listObjectsRequest);
    for(S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) {
        S3VersionSummary versionSummary = new S3VersionSummary();
        versionSummary.setBucketName(objectSummary.getBucketName());
        versionSummary.setETag(objectSummary.getETag());
        versionSummary.setKey(objectSummary.getKey());
        versionSummary.setLastModified(objectSummary.getLastModified());
        versionSummary.setOwner(objectSummary.getOwner());
        versionSummary.setSize(objectSummary.getSize());
        versionSummary.setStorageClass(objectSummary.getStorageClass());
        versionSummary.setIsLatest(true);

        versionListing.getVersionSummaries().add(versionSummary);
    }

    return versionListing;
}
 
Example 3
Source File: ListS3.java    From nifi with Apache License 2.0 6 votes vote down vote up
@Override
public VersionListing listVersions() {
    VersionListing versionListing = new VersionListing();
    this.objectListing = client.listObjectsV2(listObjectsRequest);
    for(S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) {
        S3VersionSummary versionSummary = new S3VersionSummary();
        versionSummary.setBucketName(objectSummary.getBucketName());
        versionSummary.setETag(objectSummary.getETag());
        versionSummary.setKey(objectSummary.getKey());
        versionSummary.setLastModified(objectSummary.getLastModified());
        versionSummary.setOwner(objectSummary.getOwner());
        versionSummary.setSize(objectSummary.getSize());
        versionSummary.setStorageClass(objectSummary.getStorageClass());
        versionSummary.setIsLatest(true);

        versionListing.getVersionSummaries().add(versionSummary);
    }

    return versionListing;
}
 
Example 4
Source File: S3DaoTest.java    From herd with Apache License 2.0 5 votes vote down vote up
@Test
public void testTagVersionsAmazonServiceException()
{
    // Create an S3 file transfer request parameters DTO to access S3 objects with a mocked S3 bucket name that would trigger an AWS exception.
    S3FileTransferRequestParamsDto params = new S3FileTransferRequestParamsDto();
    params.setS3BucketName(MockS3OperationsImpl.MOCK_S3_BUCKET_NAME_INTERNAL_ERROR);

    // Create an S3 version summary.
    S3VersionSummary s3VersionSummary = new S3VersionSummary();
    s3VersionSummary.setKey(S3_KEY);
    s3VersionSummary.setVersionId(S3_VERSION_ID);

    // Create an S3 object tag.
    Tag tag = new Tag(S3_OBJECT_TAG_KEY, S3_OBJECT_TAG_VALUE);

    try
    {
        s3Dao.tagVersions(params, new S3FileTransferRequestParamsDto(), Collections.singletonList(s3VersionSummary), tag);
        fail();
    }
    catch (IllegalStateException e)
    {
        assertEquals(String.format("Failed to tag S3 object with \"%s\" key and \"%s\" version id in \"%s\" bucket. " +
                "Reason: InternalError (Service: null; Status Code: 0; Error Code: InternalError; Request ID: null)", S3_KEY, S3_VERSION_ID,
            MockS3OperationsImpl.MOCK_S3_BUCKET_NAME_INTERNAL_ERROR), e.getMessage());
    }
}
 
Example 5
Source File: S3DaoImplTest.java    From herd with Apache License 2.0 5 votes vote down vote up
@Test
public void testTagVersionsOrphanS3DeleteMarker()
{
    // Create an S3 file transfer request parameters DTO to access S3 objects.
    S3FileTransferRequestParamsDto s3FileTransferRequestParamsDto = new S3FileTransferRequestParamsDto();
    s3FileTransferRequestParamsDto.setS3BucketName(S3_BUCKET_NAME);

    // Create an S3 file transfer request parameters DTO to tag S3 objects.
    S3FileTransferRequestParamsDto s3ObjectTaggerParamsDto = new S3FileTransferRequestParamsDto();
    s3ObjectTaggerParamsDto.setAwsAccessKeyId(AWS_ASSUMED_ROLE_ACCESS_KEY);
    s3ObjectTaggerParamsDto.setAwsSecretKey(AWS_ASSUMED_ROLE_SECRET_KEY);
    s3ObjectTaggerParamsDto.setSessionToken(AWS_ASSUMED_ROLE_SESSION_TOKEN);

    // Create an S3 version summary for an S3 delete marker.
    S3VersionSummary s3VersionSummary = new S3VersionSummary();
    s3VersionSummary.setKey(S3_KEY);
    s3VersionSummary.setVersionId(S3_VERSION_ID);
    s3VersionSummary.setIsDeleteMarker(true);

    // Create an S3 object tag.
    Tag tag = new Tag(S3_OBJECT_TAG_KEY, S3_OBJECT_TAG_VALUE);

    // Call the method under test.
    s3DaoImpl.tagVersions(s3FileTransferRequestParamsDto, s3ObjectTaggerParamsDto, Collections.singletonList(s3VersionSummary), tag);

    // Verify the external calls.
    verifyNoMoreInteractionsHelper();
}
 
Example 6
Source File: BusinessObjectDataInitiateDestroyHelperServiceImplTest.java    From herd with Apache License 2.0 4 votes vote down vote up
@Test
public void testExecuteS3SpecificSteps()
{
    // Create a business object data key.
    BusinessObjectDataKey businessObjectDataKey =
        new BusinessObjectDataKey(BDEF_NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, FORMAT_VERSION, PARTITION_VALUE,
            NO_SUBPARTITION_VALUES, DATA_VERSION);

    // Create a business object data destroy parameters DTO.
    BusinessObjectDataDestroyDto businessObjectDataDestroyDto =
        new BusinessObjectDataDestroyDto(businessObjectDataKey, STORAGE_NAME, BusinessObjectDataStatusEntity.DELETED, BusinessObjectDataStatusEntity.VALID,
            StorageUnitStatusEntity.DISABLING, StorageUnitStatusEntity.ENABLED, S3_ENDPOINT, S3_BUCKET_NAME, TEST_S3_KEY_PREFIX, S3_OBJECT_TAG_KEY,
            S3_OBJECT_TAG_VALUE, S3_OBJECT_TAGGER_ROLE_ARN, S3_OBJECT_TAGGER_ROLE_SESSION_NAME, BDATA_FINAL_DESTROY_DELAY_IN_DAYS);

    // Create an S3 file transfer parameters DTO to access the S3 bucket.
    S3FileTransferRequestParamsDto s3FileTransferRequestParamsDto = new S3FileTransferRequestParamsDto();

    // Create an S3 file transfer parameters DTO to be used for S3 object tagging operation.
    S3FileTransferRequestParamsDto s3ObjectTaggerParamsDto = new S3FileTransferRequestParamsDto();
    s3ObjectTaggerParamsDto.setAwsAccessKeyId(AWS_ASSUMED_ROLE_ACCESS_KEY);
    s3ObjectTaggerParamsDto.setAwsSecretKey(AWS_ASSUMED_ROLE_SECRET_KEY);
    s3ObjectTaggerParamsDto.setSessionToken(AWS_ASSUMED_ROLE_SESSION_TOKEN);

    // Create a list of all S3 versions matching the S3 key prefix form the S3 bucket.
    S3VersionSummary s3VersionSummary = new S3VersionSummary();
    s3VersionSummary.setKey(S3_KEY);
    s3VersionSummary.setVersionId(S3_VERSION_ID);
    List<S3VersionSummary> s3VersionSummaries = Collections.singletonList(s3VersionSummary);

    // Create an updated S3 file transfer parameters DTO to access the S3 bucket.
    S3FileTransferRequestParamsDto updatedS3FileTransferRequestParamsDto = new S3FileTransferRequestParamsDto();
    updatedS3FileTransferRequestParamsDto.setS3Endpoint(S3_ENDPOINT);
    updatedS3FileTransferRequestParamsDto.setS3BucketName(S3_BUCKET_NAME);
    updatedS3FileTransferRequestParamsDto.setS3KeyPrefix(TEST_S3_KEY_PREFIX + "/");

    // Create an updated S3 file transfer parameters DTO to be used for S3 object tagging operation.
    S3FileTransferRequestParamsDto updatedS3ObjectTaggerParamsDto = new S3FileTransferRequestParamsDto();
    updatedS3ObjectTaggerParamsDto.setAwsAccessKeyId(AWS_ASSUMED_ROLE_ACCESS_KEY);
    updatedS3ObjectTaggerParamsDto.setAwsSecretKey(AWS_ASSUMED_ROLE_SECRET_KEY);
    updatedS3ObjectTaggerParamsDto.setSessionToken(AWS_ASSUMED_ROLE_SESSION_TOKEN);
    updatedS3ObjectTaggerParamsDto.setS3Endpoint(S3_ENDPOINT);

    // Mock the external calls.
    when(storageHelper.getS3FileTransferRequestParamsDto()).thenReturn(s3FileTransferRequestParamsDto);
    when(storageHelper.getS3FileTransferRequestParamsDtoByRole(S3_OBJECT_TAGGER_ROLE_ARN, S3_OBJECT_TAGGER_ROLE_SESSION_NAME))
        .thenReturn(s3ObjectTaggerParamsDto);
    when(s3Service.listVersions(s3FileTransferRequestParamsDto)).thenReturn(s3VersionSummaries);

    // Call the method under test.
    businessObjectDataInitiateDestroyHelperServiceImpl.executeS3SpecificSteps(businessObjectDataDestroyDto);

    // Validate the results.
    assertEquals(
        new BusinessObjectDataDestroyDto(businessObjectDataKey, STORAGE_NAME, BusinessObjectDataStatusEntity.DELETED, BusinessObjectDataStatusEntity.VALID,
            StorageUnitStatusEntity.DISABLING, StorageUnitStatusEntity.ENABLED, S3_ENDPOINT, S3_BUCKET_NAME, TEST_S3_KEY_PREFIX, S3_OBJECT_TAG_KEY,
            S3_OBJECT_TAG_VALUE, S3_OBJECT_TAGGER_ROLE_ARN, S3_OBJECT_TAGGER_ROLE_SESSION_NAME, BDATA_FINAL_DESTROY_DELAY_IN_DAYS),
        businessObjectDataDestroyDto);

    // Verify the external calls.
    verify(storageHelper).getS3FileTransferRequestParamsDto();
    verify(storageHelper).getS3FileTransferRequestParamsDtoByRole(S3_OBJECT_TAGGER_ROLE_ARN, S3_OBJECT_TAGGER_ROLE_SESSION_NAME);
    verify(s3Service).listVersions(s3FileTransferRequestParamsDto);
    verify(s3Service).tagVersions(updatedS3FileTransferRequestParamsDto, updatedS3ObjectTaggerParamsDto, s3VersionSummaries,
        new Tag(S3_OBJECT_TAG_KEY, S3_OBJECT_TAG_VALUE));
    verifyNoMoreInteractionsHelper();
}
 
Example 7
Source File: MockS3OperationsImpl.java    From herd with Apache License 2.0 4 votes vote down vote up
/**
 * {@inheritDoc}
 * <p/>
 * If the bucket does not exist, returns a listing with an empty list. If a prefix is specified in listVersionsRequest, only versions starting with the
 * prefix will be returned.
 */
@Override
public VersionListing listVersions(ListVersionsRequest listVersionsRequest, AmazonS3 s3Client)
{
    LOGGER.debug("listVersions(): listVersionsRequest.getBucketName() = " + listVersionsRequest.getBucketName());

    String bucketName = listVersionsRequest.getBucketName();

    if (MOCK_S3_BUCKET_NAME_NO_SUCH_BUCKET_EXCEPTION.equals(bucketName))
    {
        AmazonS3Exception amazonS3Exception = new AmazonS3Exception(MOCK_S3_BUCKET_NAME_NO_SUCH_BUCKET_EXCEPTION);
        amazonS3Exception.setErrorCode("NoSuchBucket");
        throw amazonS3Exception;
    }
    else if (MOCK_S3_BUCKET_NAME_INTERNAL_ERROR.equals(bucketName))
    {
        throw new AmazonServiceException(S3Operations.ERROR_CODE_INTERNAL_ERROR);
    }

    VersionListing versionListing = new VersionListing();
    versionListing.setBucketName(bucketName);

    MockS3Bucket mockS3Bucket = mockS3Buckets.get(bucketName);
    if (mockS3Bucket != null)
    {
        for (MockS3Object mockS3Object : mockS3Bucket.getVersions().values())
        {
            String s3ObjectKey = mockS3Object.getKey();
            if (listVersionsRequest.getPrefix() == null || s3ObjectKey.startsWith(listVersionsRequest.getPrefix()))
            {
                S3VersionSummary s3VersionSummary = new S3VersionSummary();
                s3VersionSummary.setBucketName(bucketName);
                s3VersionSummary.setKey(s3ObjectKey);
                s3VersionSummary.setVersionId(mockS3Object.getVersion());
                s3VersionSummary.setSize(mockS3Object.getData().length);
                s3VersionSummary.setStorageClass(mockS3Object.getObjectMetadata() != null ? mockS3Object.getObjectMetadata().getStorageClass() : null);

                versionListing.getVersionSummaries().add(s3VersionSummary);
            }
        }
    }

    return versionListing;
}
 
Example 8
Source File: S3DaoImplTest.java    From herd with Apache License 2.0 4 votes vote down vote up
@Test
public void testDeleteDirectoryMultiObjectDeleteException()
{
    // Create an S3 file transfer request parameters DTO to access S3 objects.
    S3FileTransferRequestParamsDto s3FileTransferRequestParamsDto = new S3FileTransferRequestParamsDto();
    s3FileTransferRequestParamsDto.setS3BucketName(S3_BUCKET_NAME);
    s3FileTransferRequestParamsDto.setS3KeyPrefix(S3_KEY_PREFIX);

    // Create a retry policy.
    RetryPolicy retryPolicy =
        new RetryPolicy(PredefinedRetryPolicies.DEFAULT_RETRY_CONDITION, PredefinedRetryPolicies.DEFAULT_BACKOFF_STRATEGY, INTEGER_VALUE, true);

    // Create an S3 version summary.
    S3VersionSummary s3VersionSummary = new S3VersionSummary();
    s3VersionSummary.setKey(S3_KEY);
    s3VersionSummary.setVersionId(S3_VERSION_ID);

    // Create a version listing.
    VersionListing versionListing = new VersionListing();
    versionListing.setVersionSummaries(Collections.singletonList(s3VersionSummary));

    // Create a delete error.
    MultiObjectDeleteException.DeleteError deleteError = new MultiObjectDeleteException.DeleteError();
    deleteError.setKey(S3_KEY);
    deleteError.setVersionId(S3_VERSION_ID);
    deleteError.setCode(ERROR_CODE);
    deleteError.setMessage(ERROR_MESSAGE);

    // Create a multi object delete exception.
    MultiObjectDeleteException multiObjectDeleteException = new MultiObjectDeleteException(Collections.singletonList(deleteError), new ArrayList<>());

    // Mock the external calls.
    when(retryPolicyFactory.getRetryPolicy()).thenReturn(retryPolicy);
    when(s3Operations.listVersions(any(ListVersionsRequest.class), any(AmazonS3Client.class))).thenReturn(versionListing);
    when(s3Operations.deleteObjects(any(DeleteObjectsRequest.class), any(AmazonS3Client.class))).thenThrow(multiObjectDeleteException);

    // Try to call the method under test.
    try
    {
        s3DaoImpl.deleteDirectory(s3FileTransferRequestParamsDto);
    }
    catch (IllegalStateException e)
    {
        assertEquals(String.format(
            "Failed to delete keys/key versions with prefix \"%s\" from bucket \"%s\". Reason: One or more objects could not be deleted " +
                "(Service: null; Status Code: 0; Error Code: null; Request ID: null; S3 Extended Request ID: null)", S3_KEY_PREFIX, S3_BUCKET_NAME),
            e.getMessage());
    }

    // Verify the external calls.
    verify(retryPolicyFactory, times(2)).getRetryPolicy();
    verify(s3Operations).listVersions(any(ListVersionsRequest.class), any(AmazonS3Client.class));
    verify(s3Operations).deleteObjects(any(DeleteObjectsRequest.class), any(AmazonS3Client.class));
    verifyNoMoreInteractionsHelper();
}
 
Example 9
Source File: S3DaoImplTest.java    From herd with Apache License 2.0 4 votes vote down vote up
private void runTagVersionsTest()
{
    // Create an S3 file transfer request parameters DTO to access S3 objects.
    S3FileTransferRequestParamsDto s3FileTransferRequestParamsDto = new S3FileTransferRequestParamsDto();
    s3FileTransferRequestParamsDto.setS3BucketName(S3_BUCKET_NAME);

    // Create an S3 file transfer request parameters DTO to tag S3 objects.
    S3FileTransferRequestParamsDto s3ObjectTaggerParamsDto = new S3FileTransferRequestParamsDto();
    s3ObjectTaggerParamsDto.setAwsAccessKeyId(AWS_ASSUMED_ROLE_ACCESS_KEY);
    s3ObjectTaggerParamsDto.setAwsSecretKey(AWS_ASSUMED_ROLE_SECRET_KEY);
    s3ObjectTaggerParamsDto.setSessionToken(AWS_ASSUMED_ROLE_SESSION_TOKEN);

    // Create an S3 version summary.
    S3VersionSummary s3VersionSummary = new S3VersionSummary();
    s3VersionSummary.setKey(S3_KEY);
    s3VersionSummary.setVersionId(S3_VERSION_ID);

    // Create an S3 object tag.
    Tag tag = new Tag(S3_OBJECT_TAG_KEY, S3_OBJECT_TAG_VALUE);

    // Create a retry policy.
    RetryPolicy retryPolicy =
        new RetryPolicy(PredefinedRetryPolicies.DEFAULT_RETRY_CONDITION, PredefinedRetryPolicies.DEFAULT_BACKOFF_STRATEGY, INTEGER_VALUE, true);

    // Create a get object tagging result.
    GetObjectTaggingResult getObjectTaggingResult = new GetObjectTaggingResult(null);

    // Create a set object tagging result.
    SetObjectTaggingResult setObjectTaggingResult = new SetObjectTaggingResult();

    // Mock the external calls.
    when(retryPolicyFactory.getRetryPolicy()).thenReturn(retryPolicy);
    when(s3Operations.getObjectTagging(any(GetObjectTaggingRequest.class), any(AmazonS3Client.class))).thenReturn(getObjectTaggingResult);
    when(s3Operations.setObjectTagging(any(SetObjectTaggingRequest.class), any(AmazonS3Client.class))).thenReturn(setObjectTaggingResult);

    // Call the method under test.
    s3DaoImpl.tagVersions(s3FileTransferRequestParamsDto, s3ObjectTaggerParamsDto, Collections.singletonList(s3VersionSummary), tag);

    // Verify the external calls.
    verify(retryPolicyFactory, times(2)).getRetryPolicy();
    verify(s3Operations).getObjectTagging(any(GetObjectTaggingRequest.class), any(AmazonS3Client.class));
    verify(s3Operations).setObjectTagging(any(SetObjectTaggingRequest.class), any(AmazonS3Client.class));
    verifyNoMoreInteractionsHelper();
}
 
Example 10
Source File: TestListS3.java    From nifi with Apache License 2.0 4 votes vote down vote up
@Test
public void testListVersions() {
    runner.setProperty(ListS3.REGION, "eu-west-1");
    runner.setProperty(ListS3.BUCKET, "test-bucket");
    runner.setProperty(ListS3.USE_VERSIONS, "true");

    Date lastModified = new Date();
    VersionListing versionListing = new VersionListing();
    S3VersionSummary versionSummary1 = new S3VersionSummary();
    versionSummary1.setBucketName("test-bucket");
    versionSummary1.setKey("test-key");
    versionSummary1.setVersionId("1");
    versionSummary1.setLastModified(lastModified);
    versionListing.getVersionSummaries().add(versionSummary1);
    S3VersionSummary versionSummary2 = new S3VersionSummary();
    versionSummary2.setBucketName("test-bucket");
    versionSummary2.setKey("test-key");
    versionSummary2.setVersionId("2");
    versionSummary2.setLastModified(lastModified);
    versionListing.getVersionSummaries().add(versionSummary2);
    Mockito.when(mockS3Client.listVersions(Mockito.any(ListVersionsRequest.class))).thenReturn(versionListing);

    runner.run();

    ArgumentCaptor<ListVersionsRequest> captureRequest = ArgumentCaptor.forClass(ListVersionsRequest.class);
    Mockito.verify(mockS3Client, Mockito.times(1)).listVersions(captureRequest.capture());
    ListVersionsRequest request = captureRequest.getValue();
    assertEquals("test-bucket", request.getBucketName());
    Mockito.verify(mockS3Client, Mockito.never()).listObjects(Mockito.any(ListObjectsRequest.class));

    runner.assertAllFlowFilesTransferred(ListS3.REL_SUCCESS, 2);
    List<MockFlowFile> flowFiles = runner.getFlowFilesForRelationship(ListS3.REL_SUCCESS);
    MockFlowFile ff0 = flowFiles.get(0);
    ff0.assertAttributeEquals("filename", "test-key");
    ff0.assertAttributeEquals("s3.bucket", "test-bucket");
    ff0.assertAttributeEquals("s3.lastModified", String.valueOf(lastModified.getTime()));
    ff0.assertAttributeEquals("s3.version", "1");
    MockFlowFile ff1 = flowFiles.get(1);
    ff1.assertAttributeEquals("filename", "test-key");
    ff1.assertAttributeEquals("s3.bucket", "test-bucket");
    ff1.assertAttributeEquals("s3.lastModified", String.valueOf(lastModified.getTime()));
    ff1.assertAttributeEquals("s3.version", "2");
}