Java Code Examples for com.amazonaws.services.s3.transfer.Copy#addProgressListener()
The following examples show how to use
com.amazonaws.services.s3.transfer.Copy#addProgressListener() .
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: S3AFileSystem.java From hadoop with Apache License 2.0 | 5 votes |
private void copyFile(String srcKey, String dstKey) throws IOException { if (LOG.isDebugEnabled()) { LOG.debug("copyFile " + srcKey + " -> " + dstKey); } ObjectMetadata srcom = s3.getObjectMetadata(bucket, srcKey); final ObjectMetadata dstom = srcom.clone(); if (StringUtils.isNotBlank(serverSideEncryptionAlgorithm)) { dstom.setServerSideEncryption(serverSideEncryptionAlgorithm); } CopyObjectRequest copyObjectRequest = new CopyObjectRequest(bucket, srcKey, bucket, dstKey); copyObjectRequest.setCannedAccessControlList(cannedACL); copyObjectRequest.setNewObjectMetadata(dstom); ProgressListener progressListener = new ProgressListener() { public void progressChanged(ProgressEvent progressEvent) { switch (progressEvent.getEventCode()) { case ProgressEvent.PART_COMPLETED_EVENT_CODE: statistics.incrementWriteOps(1); break; default: break; } } }; Copy copy = transfers.copy(copyObjectRequest); copy.addProgressListener(progressListener); try { copy.waitForCopyResult(); statistics.incrementWriteOps(1); } catch (InterruptedException e) { throw new IOException("Got interrupted, cancelling"); } }
Example 2
Source File: S3AFileSystem.java From big-c with Apache License 2.0 | 5 votes |
private void copyFile(String srcKey, String dstKey) throws IOException { if (LOG.isDebugEnabled()) { LOG.debug("copyFile " + srcKey + " -> " + dstKey); } ObjectMetadata srcom = s3.getObjectMetadata(bucket, srcKey); final ObjectMetadata dstom = srcom.clone(); if (StringUtils.isNotBlank(serverSideEncryptionAlgorithm)) { dstom.setServerSideEncryption(serverSideEncryptionAlgorithm); } CopyObjectRequest copyObjectRequest = new CopyObjectRequest(bucket, srcKey, bucket, dstKey); copyObjectRequest.setCannedAccessControlList(cannedACL); copyObjectRequest.setNewObjectMetadata(dstom); ProgressListener progressListener = new ProgressListener() { public void progressChanged(ProgressEvent progressEvent) { switch (progressEvent.getEventCode()) { case ProgressEvent.PART_COMPLETED_EVENT_CODE: statistics.incrementWriteOps(1); break; default: break; } } }; Copy copy = transfers.copy(copyObjectRequest); copy.addProgressListener(progressListener); try { copy.waitForCopyResult(); statistics.incrementWriteOps(1); } catch (InterruptedException e) { throw new IOException("Got interrupted, cancelling"); } }
Example 3
Source File: S3CopyStep.java From pipeline-aws-plugin with Apache License 2.0 | 4 votes |
@Override public String run() throws Exception { final String fromBucket = this.step.getFromBucket(); final String toBucket = this.step.getToBucket(); final String fromPath = this.step.getFromPath(); final String toPath = this.step.getToPath(); final String kmsId = this.step.getKmsId(); final Map<String, String> metadatas = new HashMap<>(); final CannedAccessControlList acl = this.step.getAcl(); final String cacheControl = this.step.getCacheControl(); final String contentType = this.step.getContentType(); final String sseAlgorithm = this.step.getSseAlgorithm(); final S3ClientOptions s3ClientOptions = this.step.createS3ClientOptions(); final EnvVars envVars = this.getContext().get(EnvVars.class); if (this.step.getMetadatas() != null && this.step.getMetadatas().length != 0) { for (String metadata : this.step.getMetadatas()) { if (metadata.split(":").length == 2) { metadatas.put(metadata.split(":")[0], metadata.split(":")[1]); } } } Preconditions.checkArgument(fromBucket != null && !fromBucket.isEmpty(), "From bucket must not be null or empty"); Preconditions.checkArgument(fromPath != null && !fromPath.isEmpty(), "From path must not be null or empty"); Preconditions.checkArgument(toBucket != null && !toBucket.isEmpty(), "To bucket must not be null or empty"); Preconditions.checkArgument(toPath != null && !toPath.isEmpty(), "To path must not be null or empty"); TaskListener listener = Execution.this.getContext().get(TaskListener.class); listener.getLogger().format("Copying s3://%s/%s to s3://%s/%s%n", fromBucket, fromPath, toBucket, toPath); CopyObjectRequest request = new CopyObjectRequest(fromBucket, fromPath, toBucket, toPath); // Add metadata if (metadatas.size() > 0 || (cacheControl != null && !cacheControl.isEmpty()) || (contentType != null && !contentType.isEmpty()) || (sseAlgorithm != null && !sseAlgorithm.isEmpty())) { ObjectMetadata metas = new ObjectMetadata(); if (metadatas.size() > 0) { metas.setUserMetadata(metadatas); } if (cacheControl != null && !cacheControl.isEmpty()) { metas.setCacheControl(cacheControl); } if (contentType != null && !contentType.isEmpty()) { metas.setContentType(contentType); } if (sseAlgorithm != null && !sseAlgorithm.isEmpty()) { metas.setSSEAlgorithm(sseAlgorithm); } request.withNewObjectMetadata(metas); } // Add acl if (acl != null) { request.withCannedAccessControlList(acl); } // Add kms if (kmsId != null && !kmsId.isEmpty()) { listener.getLogger().format("Using KMS: %s%n", kmsId); request.withSSEAwsKeyManagementParams(new SSEAwsKeyManagementParams(kmsId)); } TransferManager mgr = TransferManagerBuilder.standard() .withS3Client(AWSClientFactory.create(s3ClientOptions.createAmazonS3ClientBuilder(), envVars)) .build(); try { final Copy copy = mgr.copy(request); copy.addProgressListener((ProgressListener) progressEvent -> { if (progressEvent.getEventType() == ProgressEventType.TRANSFER_COMPLETED_EVENT) { listener.getLogger().println("Finished: " + copy.getDescription()); } }); copy.waitForCompletion(); } finally{ mgr.shutdownNow(); } listener.getLogger().println("Copy complete"); return String.format("s3://%s/%s", toBucket, toPath); }