Java Code Examples for com.amazonaws.services.s3.AmazonS3Client#shutdown()

@Override
public ObjectMetadata getObjectMetadata(final S3FileTransferRequestParamsDto params)
{
    AmazonS3Client s3Client = getAmazonS3(params);

    try
    {
        return s3Operations.getObjectMetadata(params.getS3BucketName(), params.getS3KeyPrefix(), s3Client);
    }
    catch (AmazonServiceException e)
    {
        if (e.getStatusCode() == HttpStatus.SC_NOT_FOUND)
        {
            return null;
        }

        throw new IllegalStateException(String
            .format("Failed to get S3 metadata for object key \"%s\" from bucket \"%s\". Reason: %s", params.getS3KeyPrefix(), params.getS3BucketName(),
                e.getMessage()), e);
    }
    finally
    {
        // Shutdown the AmazonS3Client instance to release resources.
        s3Client.shutdown();
    }
}
 

@Override
public Properties getProperties(String bucketName, String key, S3FileTransferRequestParamsDto s3FileTransferRequestParamsDto)
{
    AmazonS3Client s3Client = getAmazonS3(s3FileTransferRequestParamsDto);

    try
    {
        S3Object s3Object = getS3Object(s3Client, bucketName, key, true);
        return javaPropertiesHelper.getProperties(s3Object.getObjectContent());
    }
    catch (IllegalArgumentException e)
    {
        throw new IllegalArgumentException("The properties file in S3 bucket '" + bucketName + "' and key '" + key + "' is invalid.", e);
    }
    finally
    {
        s3Client.shutdown();
    }
}
 

@Override
public void createDirectory(final S3FileTransferRequestParamsDto params)
{
    // Create metadata for the directory marker and set content-length to 0 bytes.
    ObjectMetadata metadata = new ObjectMetadata();
    metadata.setContentLength(0);
    prepareMetadata(params, metadata);

    // Create empty content.
    InputStream emptyContent = new ByteArrayInputStream(new byte[0]);

    // Create a PutObjectRequest passing the folder name suffixed by '/'.
    String directoryName = StringUtils.appendIfMissing(params.getS3KeyPrefix(), "/");
    PutObjectRequest putObjectRequest = new PutObjectRequest(params.getS3BucketName(), directoryName, emptyContent, metadata);
    // KMS key ID is being set through prepareMetadata()

    AmazonS3Client s3Client = getAmazonS3(params);

    try
    {
        s3Operations.putObject(putObjectRequest, s3Client);
    }
    catch (AmazonServiceException e)
    {
        throw new IllegalStateException(String
            .format("Failed to create 0 byte S3 object with \"%s\" key in bucket \"%s\". Reason: %s", directoryName, params.getS3BucketName(),
                e.getMessage()), e);
    }
    finally
    {
        // Shutdown the AmazonS3Client instance to release resources.
        s3Client.shutdown();
    }
}
 

@Override
public void deleteFileList(final S3FileTransferRequestParamsDto params)
{
    LOGGER.info("Deleting a list of objects from S3... s3BucketName=\"{}\" s3KeyCount={}", params.getS3BucketName(), params.getFiles().size());

    try
    {
        // In order to avoid a MalformedXML AWS exception, we send delete request only when we have any keys to delete.
        if (!params.getFiles().isEmpty())
        {
            // Create an S3 client.
            AmazonS3Client s3Client = getAmazonS3(params);

            try
            {
                // Build a list of keys to be deleted.
                List<DeleteObjectsRequest.KeyVersion> keys = new ArrayList<>();
                for (File file : params.getFiles())
                {
                    keys.add(new DeleteObjectsRequest.KeyVersion(file.getPath().replaceAll("\\\\", "/")));
                }

                // Delete the keys.
                deleteKeyVersions(s3Client, params.getS3BucketName(), keys);
            }
            finally
            {
                s3Client.shutdown();
            }
        }
    }
    catch (Exception e)
    {
        throw new IllegalStateException(
            String.format("Failed to delete a list of keys from bucket \"%s\". Reason: %s", params.getS3BucketName(), e.getMessage()), e);
    }
}
 

@Override
public String generateGetObjectPresignedUrl(String bucketName, String key, Date expiration, S3FileTransferRequestParamsDto s3FileTransferRequestParamsDto)
{
    GeneratePresignedUrlRequest generatePresignedUrlRequest = new GeneratePresignedUrlRequest(bucketName, key, HttpMethod.GET);
    generatePresignedUrlRequest.setExpiration(expiration);
    AmazonS3Client s3 = getAmazonS3(s3FileTransferRequestParamsDto);
    try
    {
        return s3Operations.generatePresignedUrl(generatePresignedUrlRequest, s3).toString();
    }
    finally
    {
        s3.shutdown();
    }
}
 

@Override
public boolean s3FileExists(S3FileTransferRequestParamsDto params) throws RuntimeException
{
    AmazonS3Client s3Client = getAmazonS3(params);

    try
    {
        S3Object s3Object = getS3Object(s3Client, params.getS3BucketName(), params.getS3KeyPrefix(), false);
        return (s3Object != null);
    }
    finally
    {
        s3Client.shutdown();
    }
}
 

@Override
public int abortMultipartUploads(S3FileTransferRequestParamsDto params, Date thresholdDate)
{
    // Create an Amazon S3 client.
    AmazonS3Client s3Client = getAmazonS3(params);
    int abortedMultipartUploadsCount = 0;

    try
    {
        // List upload markers. Null implies initial list request.
        String uploadIdMarker = null;
        String keyMarker = null;

        boolean truncated;
        do
        {
            // Create the list multipart request, optionally using the last markers.
            ListMultipartUploadsRequest request = new ListMultipartUploadsRequest(params.getS3BucketName());
            request.setUploadIdMarker(uploadIdMarker);
            request.setKeyMarker(keyMarker);

            // Request the multipart upload listing.
            MultipartUploadListing uploadListing = s3Operations.listMultipartUploads(TransferManager.appendSingleObjectUserAgent(request), s3Client);

            for (MultipartUpload upload : uploadListing.getMultipartUploads())
            {
                if (upload.getInitiated().compareTo(thresholdDate) < 0)
                {
                    // Abort the upload.
                    s3Operations.abortMultipartUpload(TransferManager
                            .appendSingleObjectUserAgent(new AbortMultipartUploadRequest(params.getS3BucketName(), upload.getKey(), upload.getUploadId())),
                        s3Client);

                    // Log the information about the aborted multipart upload.
                    LOGGER.info("Aborted S3 multipart upload. s3Key=\"{}\" s3BucketName=\"{}\" s3MultipartUploadInitiatedDate=\"{}\"", upload.getKey(),
                        params.getS3BucketName(), upload.getInitiated());

                    // Increment the counter.
                    abortedMultipartUploadsCount++;
                }
            }

            // Determine whether there are more uploads to list.
            truncated = uploadListing.isTruncated();
            if (truncated)
            {
                // Record the list markers.
                uploadIdMarker = uploadListing.getNextUploadIdMarker();
                keyMarker = uploadListing.getNextKeyMarker();
            }
        }
        while (truncated);
    }
    finally
    {
        // Shutdown the Amazon S3 client instance to release resources.
        s3Client.shutdown();
    }

    return abortedMultipartUploadsCount;
}
 

@Override
public void deleteDirectory(final S3FileTransferRequestParamsDto params)
{
    LOGGER.info("Deleting keys/key versions from S3... s3KeyPrefix=\"{}\" s3BucketName=\"{}\"", params.getS3KeyPrefix(), params.getS3BucketName());

    Assert.isTrue(!isRootKeyPrefix(params.getS3KeyPrefix()), "Deleting from root directory is not allowed.");

    try
    {
        // List S3 versions.
        List<S3VersionSummary> s3VersionSummaries = listVersions(params);
        LOGGER.info("Found keys/key versions in S3 for deletion. s3KeyCount={} s3KeyPrefix=\"{}\" s3BucketName=\"{}\"", s3VersionSummaries.size(),
            params.getS3KeyPrefix(), params.getS3BucketName());

        // In order to avoid a MalformedXML AWS exception, we send delete request only when we have any key versions to delete.
        if (CollectionUtils.isNotEmpty(s3VersionSummaries))
        {
            // Create an S3 client.
            AmazonS3Client s3Client = getAmazonS3(params);

            // Build a list of objects to be deleted.
            List<DeleteObjectsRequest.KeyVersion> keyVersions = new ArrayList<>();
            for (S3VersionSummary s3VersionSummary : s3VersionSummaries)
            {
                keyVersions.add(new DeleteObjectsRequest.KeyVersion(s3VersionSummary.getKey(), s3VersionSummary.getVersionId()));
            }

            try
            {
                // Delete the key versions.
                deleteKeyVersions(s3Client, params.getS3BucketName(), keyVersions);
            }
            finally
            {
                s3Client.shutdown();
            }
        }
    }
    catch (AmazonClientException e)
    {
        throw new IllegalStateException(String
            .format("Failed to delete keys/key versions with prefix \"%s\" from bucket \"%s\". Reason: %s", params.getS3KeyPrefix(),
                params.getS3BucketName(), e.getMessage()), e);
    }
}
 

@Override
public List<S3ObjectSummary> listDirectory(final S3FileTransferRequestParamsDto params, boolean ignoreZeroByteDirectoryMarkers)
{
    Assert.isTrue(!isRootKeyPrefix(params.getS3KeyPrefix()), "Listing of S3 objects from root directory is not allowed.");

    AmazonS3Client s3Client = getAmazonS3(params);
    List<S3ObjectSummary> s3ObjectSummaries = new ArrayList<>();

    try
    {
        ListObjectsRequest listObjectsRequest = new ListObjectsRequest().withBucketName(params.getS3BucketName()).withPrefix(params.getS3KeyPrefix());
        ObjectListing objectListing;

        do
        {
            objectListing = s3Operations.listObjects(listObjectsRequest, s3Client);

            for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries())
            {
                // Ignore 0 byte objects that represent S3 directories.
                if (!(ignoreZeroByteDirectoryMarkers && objectSummary.getKey().endsWith("/") && objectSummary.getSize() == 0L))
                {
                    s3ObjectSummaries.add(objectSummary);
                }
            }

            listObjectsRequest.setMarker(objectListing.getNextMarker());
        }
        while (objectListing.isTruncated());
    }
    catch (AmazonS3Exception amazonS3Exception)
    {
        if (S3Operations.ERROR_CODE_NO_SUCH_BUCKET.equals(amazonS3Exception.getErrorCode()))
        {
            throw new IllegalArgumentException("The specified bucket '" + params.getS3BucketName() + "' does not exist.", amazonS3Exception);
        }
        throw new IllegalStateException("Error accessing S3", amazonS3Exception);
    }
    catch (AmazonClientException e)
    {
        throw new IllegalStateException(String
            .format("Failed to list keys with prefix \"%s\" from bucket \"%s\". Reason: %s", params.getS3KeyPrefix(), params.getS3BucketName(),
                e.getMessage()), e);
    }
    finally
    {
        // Shutdown the AmazonS3Client instance to release resources.
        s3Client.shutdown();
    }

    return s3ObjectSummaries;
}
 

@Override
public List<S3VersionSummary> listVersions(final S3FileTransferRequestParamsDto params)
{
    Assert.isTrue(!isRootKeyPrefix(params.getS3KeyPrefix()), "Listing of S3 versions from root directory is not allowed.");

    AmazonS3Client s3Client = getAmazonS3(params);
    List<S3VersionSummary> s3VersionSummaries = new ArrayList<>();

    try
    {
        ListVersionsRequest listVersionsRequest = new ListVersionsRequest().withBucketName(params.getS3BucketName()).withPrefix(params.getS3KeyPrefix());
        VersionListing versionListing;

        do
        {
            versionListing = s3Operations.listVersions(listVersionsRequest, s3Client);
            s3VersionSummaries.addAll(versionListing.getVersionSummaries());
            listVersionsRequest.setKeyMarker(versionListing.getNextKeyMarker());
            listVersionsRequest.setVersionIdMarker(versionListing.getNextVersionIdMarker());
        }
        while (versionListing.isTruncated());
    }
    catch (AmazonS3Exception amazonS3Exception)
    {
        if (S3Operations.ERROR_CODE_NO_SUCH_BUCKET.equals(amazonS3Exception.getErrorCode()))
        {
            throw new IllegalArgumentException("The specified bucket '" + params.getS3BucketName() + "' does not exist.", amazonS3Exception);
        }
        throw new IllegalStateException("Error accessing S3", amazonS3Exception);
    }
    catch (AmazonClientException e)
    {
        throw new IllegalStateException(String
            .format("Failed to list S3 versions with prefix \"%s\" from bucket \"%s\". Reason: %s", params.getS3KeyPrefix(), params.getS3BucketName(),
                e.getMessage()), e);
    }
    finally
    {
        // Shutdown the AmazonS3Client instance to release resources.
        s3Client.shutdown();
    }

    return s3VersionSummaries;
}
 

@Override
public void restoreObjects(final S3FileTransferRequestParamsDto params, int expirationInDays, String archiveRetrievalOption)
{
    LOGGER.info("Restoring a list of objects in S3... s3KeyPrefix=\"{}\" s3BucketName=\"{}\" s3KeyCount={}", params.getS3KeyPrefix(),
        params.getS3BucketName(), params.getFiles().size());

    if (!CollectionUtils.isEmpty(params.getFiles()))
    {
        // Initialize a key value pair for the error message in the catch block.
        String key = params.getFiles().get(0).getPath().replaceAll("\\\\", "/");

        try
        {
            // Create an S3 client.
            AmazonS3Client s3Client = getAmazonS3(params);

            // Create a restore object request.
            RestoreObjectRequest requestRestore = new RestoreObjectRequest(params.getS3BucketName(), null, expirationInDays);
            // Make Bulk the default archive retrieval option if the option is not provided
            requestRestore.setGlacierJobParameters(new GlacierJobParameters().withTier(
                StringUtils.isNotEmpty(archiveRetrievalOption) ? archiveRetrievalOption : Tier.Bulk.toString()));

            try
            {
                for (File file : params.getFiles())
                {
                    key = file.getPath().replaceAll("\\\\", "/");
                    ObjectMetadata objectMetadata = s3Operations.getObjectMetadata(params.getS3BucketName(), key, s3Client);

                    // Request a restore for objects that are not already being restored.
                    if (BooleanUtils.isNotTrue(objectMetadata.getOngoingRestore()))
                    {
                        requestRestore.setKey(key);

                        try
                        {
                            // Try the S3 restore operation on this file.
                            s3Operations.restoreObject(requestRestore, s3Client);
                        }
                        catch (AmazonS3Exception amazonS3Exception)
                        {
                            // If this exception has a status code of 409, log the information and continue to the next file.
                            if (amazonS3Exception.getStatusCode() == HttpStatus.SC_CONFLICT)
                            {
                                LOGGER.info("Restore already in progress for file with s3Key=\"{}\".", key);
                            }
                            // Else, we need to propagate the exception to the next level of try/catch block.
                            else
                            {
                                throw new Exception(amazonS3Exception);
                            }
                        }
                    }
                }
            }
            finally
            {
                s3Client.shutdown();
            }
        }
        catch (Exception e)
        {
            if (StringUtils.contains(e.getMessage(), "Retrieval option is not supported by this storage class"))
            {
                throw new IllegalArgumentException(String
                    .format("Failed to initiate a restore request for \"%s\" key in \"%s\" bucket. Reason: %s", key, params.getS3BucketName(),
                        e.getMessage()), e);
            }
            else
            {
                throw new IllegalStateException(String
                    .format("Failed to initiate a restore request for \"%s\" key in \"%s\" bucket. Reason: %s", key, params.getS3BucketName(),
                        e.getMessage()), e);
            }
        }
    }
}
 

@Override
public void validateGlacierS3FilesRestored(S3FileTransferRequestParamsDto params) throws RuntimeException
{
    LOGGER.info("Checking for already restored Glacier or DeepArchive storage class objects... s3KeyPrefix=\"{}\" s3BucketName=\"{}\" s3KeyCount={}",
        params.getS3KeyPrefix(), params.getS3BucketName(), params.getFiles().size());

    if (!CollectionUtils.isEmpty(params.getFiles()))
    {
        // Initialize a key value pair for the error message in the catch block.
        String key = params.getFiles().get(0).getPath().replaceAll("\\\\", "/");

        try
        {
            // Create an S3 client.
            AmazonS3Client s3Client = getAmazonS3(params);

            try
            {
                for (File file : params.getFiles())
                {
                    key = file.getPath().replaceAll("\\\\", "/");
                    ObjectMetadata objectMetadata = s3Operations.getObjectMetadata(params.getS3BucketName(), key, s3Client);

                    // Fail if a not already restored object is detected.
                    if (BooleanUtils.isNotFalse(objectMetadata.getOngoingRestore()))
                    {
                        throw new IllegalArgumentException(String
                            .format("Archived S3 file \"%s\" is not restored. StorageClass {%s}, OngoingRestore flag {%s}, S3 bucket name {%s}",
                                key, objectMetadata.getStorageClass(), objectMetadata.getOngoingRestore(), params.getS3BucketName()));
                    }
                }
            }
            finally
            {
                s3Client.shutdown();
            }
        }
        catch (AmazonServiceException e)
        {
            throw new IllegalStateException(
                String.format("Fail to check restore status for \"%s\" key in \"%s\" bucket. Reason: %s", key, params.getS3BucketName(), e.getMessage()),
                e);
        }
    }
}
 

private void tagVersionsHelper(final S3FileTransferRequestParamsDto s3FileTransferRequestParamsDto,
    final S3FileTransferRequestParamsDto s3ObjectTaggerParamsDto, final List<S3VersionSummary> s3VersionSummaries, final Tag tag)
{
    // Initialize an S3 version for the error message in the catch block.
    S3VersionSummary currentS3VersionSummary = s3VersionSummaries.get(0);

    // Amazon S3 client to access S3 objects.
    AmazonS3Client s3Client = null;

    // Amazon S3 client for S3 object tagging.
    AmazonS3Client s3ObjectTaggerClient = null;

    try
    {
        // Create an S3 client to access S3 objects.
        s3Client = getAmazonS3(s3FileTransferRequestParamsDto);

        // Create an S3 client for S3 object tagging.
        s3ObjectTaggerClient = getAmazonS3(s3ObjectTaggerParamsDto);

        // Create a get object tagging request.
        GetObjectTaggingRequest getObjectTaggingRequest = new GetObjectTaggingRequest(s3FileTransferRequestParamsDto.getS3BucketName(), null, null);

        // Create a set object tagging request.
        SetObjectTaggingRequest setObjectTaggingRequest = new SetObjectTaggingRequest(s3FileTransferRequestParamsDto.getS3BucketName(), null, null, null);

        for (S3VersionSummary s3VersionSummary : s3VersionSummaries)
        {
            // Set the current S3 version summary.
            currentS3VersionSummary = s3VersionSummary;

            // Retrieve the current tagging information for the S3 version.
            getObjectTaggingRequest.setKey(s3VersionSummary.getKey());
            getObjectTaggingRequest.setVersionId(s3VersionSummary.getVersionId());
            GetObjectTaggingResult getObjectTaggingResult = s3Operations.getObjectTagging(getObjectTaggingRequest, s3Client);

            // Update the list of tags to include the specified S3 object tag.
            List<Tag> updatedTags = new ArrayList<>();
            updatedTags.add(tag);
            if (CollectionUtils.isNotEmpty(getObjectTaggingResult.getTagSet()))
            {
                for (Tag currentTag : getObjectTaggingResult.getTagSet())
                {
                    if (!StringUtils.equals(tag.getKey(), currentTag.getKey()))
                    {
                        updatedTags.add(currentTag);
                    }
                }
            }

            // Update tagging information for the S3 version.
            setObjectTaggingRequest.setKey(s3VersionSummary.getKey());
            setObjectTaggingRequest.setVersionId(s3VersionSummary.getVersionId());
            setObjectTaggingRequest.setTagging(new ObjectTagging(updatedTags));
            s3Operations.setObjectTagging(setObjectTaggingRequest, s3ObjectTaggerClient);
        }
    }
    catch (Exception e)
    {
        throw new IllegalStateException(String
            .format("Failed to tag S3 object with \"%s\" key and \"%s\" version id in \"%s\" bucket. Reason: %s", currentS3VersionSummary.getKey(),
                currentS3VersionSummary.getVersionId(), s3FileTransferRequestParamsDto.getS3BucketName(), e.getMessage()), e);
    }
    finally
    {
        if (s3Client != null)
        {
            s3Client.shutdown();
        }

        if (s3ObjectTaggerClient != null)
        {
            s3ObjectTaggerClient.shutdown();
        }
    }
}