com.amazonaws.services.s3.model.SSEAwsKeyManagementParams Java Examples

private void writeBufferToS3(byte[] toWrite, int limit) {
    try (EtlProfilingScope scope = new EtlProfilingScope(parentMetrics, "S3FastLoader.writeToS3")) {
        InputStream inputStream = new ByteArrayInputStream(toWrite, 0, limit);
        String s3Key = s3PartFileKeyGenerator.apply(++fileSequenceNumber);
        ObjectMetadata metadata = new ObjectMetadata();
        metadata.setContentLength(limit);
        PutObjectRequest putObjectRequest = new PutObjectRequest(s3Bucket, s3Key, inputStream, metadata);

        if (sseKmsArn != null && !sseKmsArn.isEmpty()) {
            putObjectRequest.setSSEAwsKeyManagementParams(new SSEAwsKeyManagementParams(sseKmsArn));
        }

        try {
            amazonS3.putObject(putObjectRequest);
            emitSuccessAndFailureMetrics(scope, true);
        } catch (AmazonClientException e) {
                logger.error(e);
                scope.addCounter(e.getClass().getSimpleName(), 1);
                emitSuccessAndFailureMetrics(scope, false);
                throw new UnrecoverableStreamFailureException("Exception caught trying to write object to S3: ", e);
        }
    }
}
 

@Test
public void writeCallsS3WithKmsSseEncryption() {
    testS3LoaderWithStrings("123456789ABCDEF");

    verify(s3Client, times(1)).putObject(putObjectRequestArgumentCaptor.capture());

    final PutObjectRequest request = putObjectRequestArgumentCaptor.getValue();

    SSEAwsKeyManagementParams SSEParams = request.getSSEAwsKeyManagementParams();
    assertThat(SSEParams.getEncryption(), equalTo(KMS_SSE_ENCRYPTION_TYPE));
    assertThat(SSEParams.getAwsKmsKeyId(), equalTo(KMS_KEY));
}
 

private void enableKmsEncryption(PutObjectRequest uploadRequest) {
    String keyId = mConfig.getAwsSseKmsKey();
    if (!keyId.isEmpty()) {
        uploadRequest.withSSEAwsKeyManagementParams(new SSEAwsKeyManagementParams(keyId));
    } else {
        uploadRequest.withSSEAwsKeyManagementParams(new SSEAwsKeyManagementParams());
    }
}
 

/**
 * not sure the method is called one or two times, it depend on the platform
 */
@Override
public Result close() throws IOException {
    if (closed) {
        return result;
    }

    closed = true;

    try {
        if (writer != null) {
            writer.flush();
            writer.close();
        }

        S3DatasetProperties data_set = properties.getDatasetProperties();
        PutObjectRequest request = new PutObjectRequest(data_set.bucket.getValue(), data_set.object.getValue(), data_file);

        Boolean serverSideEnc = data_set.encryptDataAtRest.getValue();
        if (serverSideEnc != null && serverSideEnc) {
            request.withSSEAwsKeyManagementParams(new SSEAwsKeyManagementParams(data_set.kmsForDataAtRest.getValue()));
        }

        s3_client.putObject(request);
    } finally {
        writer = null;
        data_file.delete();

        if (s3_client != null) {
            s3_client.shutdown();
            s3_client = null;
        }
    }

    result.successCount = result.totalCount;
    return result;
}
 

@Test
public void uploadWithUnknownEncryptionKeyType() throws IOException {
    TestUtils.initializeTestingFolders();

    when(mockEncryptionKey.getId()).thenReturn("KMS-KEY-ARN");
    when(mockEncryptionKey.getType()).thenReturn("Custom");

    final File compressedFile = CompressionTools.compressFile(
            "ZipProject",
            PATH_TO_COMPRESS,
            CompressionType.Zip,
            null);

    PublisherTools.uploadFile(
            compressedFile,
            mockArtifact,
            CompressionType.Zip,
            mockEncryptionKey,
            mockS3Client,
            null); // Listener

    verify(mockS3Client).initiateMultipartUpload(initiateCaptor.capture());

    assertContainsIgnoreCase("[AWS CodePipeline Plugin] Upload successful", outContent.toString());

    final InitiateMultipartUploadRequest request = initiateCaptor.getValue();
    final SSEAwsKeyManagementParams encryptionParams = request.getSSEAwsKeyManagementParams();
    assertNotNull(encryptionParams);
    assertNull(encryptionParams.getAwsKmsKeyId());
    assertEquals("aws:kms", encryptionParams.getEncryption());

    compressedFile.delete();
    TestUtils.cleanUpTestingFolders();
}
 

@Test
public void uploadWithCustomKmsEncryptionKey() throws IOException {
    TestUtils.initializeTestingFolders();

    when(mockEncryptionKey.getId()).thenReturn("KMS-KEY-ARN");
    when(mockEncryptionKey.getType()).thenReturn(EncryptionKeyType.KMS.toString());

    final File compressedFile = CompressionTools.compressFile(
            "ZipProject",
            PATH_TO_COMPRESS,
            CompressionType.Zip,
            null);

    PublisherTools.uploadFile(
            compressedFile,
            mockArtifact,
            CompressionType.Zip,
            mockEncryptionKey,
            mockS3Client,
            null); // Listener

    verify(mockS3Client).initiateMultipartUpload(initiateCaptor.capture());

    assertContainsIgnoreCase("[AWS CodePipeline Plugin] Upload successful", outContent.toString());

    final InitiateMultipartUploadRequest request = initiateCaptor.getValue();
    final SSEAwsKeyManagementParams encryptionParams = request.getSSEAwsKeyManagementParams();
    assertNotNull(encryptionParams);
    assertEquals("KMS-KEY-ARN", encryptionParams.getAwsKmsKeyId());
    assertEquals("aws:kms", encryptionParams.getEncryption());

    compressedFile.delete();
    TestUtils.cleanUpTestingFolders();
}
 

@Test
public void uploadFileSuccess() throws IOException {
    TestUtils.initializeTestingFolders();

    final File compressedFile = CompressionTools.compressFile(
            "ZipProject",
            PATH_TO_COMPRESS,
            CompressionType.Zip,
            null);

    PublisherTools.uploadFile(
            compressedFile,
            mockArtifact,
            CompressionType.Zip,
            null, // No custom encryption key
            mockS3Client,
            null); // Listener

    final InOrder inOrder = inOrder(mockS3Client);
    inOrder.verify(mockS3Client, times(1)).initiateMultipartUpload(initiateCaptor.capture());
    // Total size is less than 5MB, should only be one upload
    inOrder.verify(mockS3Client, times(1)).uploadPart(any(UploadPartRequest.class));
    inOrder.verify(mockS3Client, times(1)).completeMultipartUpload(any(CompleteMultipartUploadRequest.class));

    assertContainsIgnoreCase("[AWS CodePipeline Plugin] Uploading artifact:", outContent.toString());
    assertContainsIgnoreCase("[AWS CodePipeline Plugin] Upload successful", outContent.toString());

    final InitiateMultipartUploadRequest request = initiateCaptor.getValue();
    final SSEAwsKeyManagementParams encryptionParams = request.getSSEAwsKeyManagementParams();
    assertNotNull(encryptionParams);
    assertNull(encryptionParams.getAwsKmsKeyId());
    assertEquals("aws:kms", encryptionParams.getEncryption());

    compressedFile.delete();
    TestUtils.cleanUpTestingFolders();
}
 

private static SSEAwsKeyManagementParams toSSEAwsKeyManagementParams(final EncryptionKey encryptionKey) {
    if (encryptionKey != null
            && encryptionKey.getId() != null
            && EncryptionKeyType.KMS.toString().equals(encryptionKey.getType())) {
        return new SSEAwsKeyManagementParams(encryptionKey.getId());
    }

    return new SSEAwsKeyManagementParams();
}
 

public KMSEncrypter(final Optional<String> kmsId) {
  this.kmsParameters = checkNotNull(kmsId)
      .map(String::trim)
      .filter(id -> !id.isEmpty())
      .map(SSEAwsKeyManagementParams::new)
      .orElse(new SSEAwsKeyManagementParams());
}
 

static S3Options s3OptionsWithSSEAwsKeyManagementParams() {
  S3Options options = s3Options();
  String awsKmsKeyId =
      "arn:aws:kms:eu-west-1:123456789012:key/dc123456-7890-ABCD-EF01-234567890ABC";
  SSEAwsKeyManagementParams sseAwsKeyManagementParams =
      new SSEAwsKeyManagementParams(awsKmsKeyId);
  options.setSSEAwsKeyManagementParams(sseAwsKeyManagementParams);
  return options;
}
 

@Test
public void testSSEAwsKeyManagementParamsSerializationDeserialization() throws Exception {
  final String awsKmsKeyId =
      "arn:aws:kms:eu-west-1:123456789012:key/dc123456-7890-ABCD-EF01-234567890ABC";
  final String encryption = "aws:kms";
  SSEAwsKeyManagementParams value = new SSEAwsKeyManagementParams(awsKmsKeyId);

  String valueAsJson = objectMapper.writeValueAsString(value);
  SSEAwsKeyManagementParams valueDes =
      objectMapper.readValue(valueAsJson, SSEAwsKeyManagementParams.class);
  assertEquals(awsKmsKeyId, valueDes.getAwsKmsKeyId());
  assertEquals(encryption, valueDes.getEncryption());
}
 

@Override
public SSEAwsKeyManagementParams deserialize(JsonParser parser, DeserializationContext context)
    throws IOException {
  Map<String, String> asMap = parser.readValueAs(new TypeReference<Map<String, String>>() {});
  final String awsKmsKeyId = asMap.getOrDefault("awsKmsKeyId", null);
  return new SSEAwsKeyManagementParams(awsKmsKeyId);
}
 

public AwsModule() {
  super("AwsModule");
  setMixInAnnotation(AWSCredentialsProvider.class, AWSCredentialsProviderMixin.class);
  setMixInAnnotation(SSECustomerKey.class, SSECustomerKeyMixin.class);
  setMixInAnnotation(SSEAwsKeyManagementParams.class, SSEAwsKeyManagementParamsMixin.class);
  setMixInAnnotation(ClientConfiguration.class, AwsHttpClientConfigurationMixin.class);
}
 

@Override
public S3FileTransferResultsDto copyFile(final S3FileCopyRequestParamsDto params) throws InterruptedException
{
    LOGGER
        .info("Copying S3 object... sourceS3Key=\"{}\" sourceS3BucketName=\"{}\" targetS3Key=\"{}\" targetS3BucketName=\"{}\"", params.getSourceObjectKey(),
            params.getSourceBucketName(), params.getTargetObjectKey(), params.getTargetBucketName());

    // Perform the copy.
    S3FileTransferResultsDto results = performTransfer(params, new Transferer()
    {
        @Override
        public Transfer performTransfer(TransferManager transferManager)
        {
            // Create a copy request.
            CopyObjectRequest copyObjectRequest =
                new CopyObjectRequest(params.getSourceBucketName(), params.getSourceObjectKey(), params.getTargetBucketName(), params.getTargetObjectKey());

            // If KMS Key ID is specified, set the AWS Key Management System parameters to be used to encrypt the object.
            if (StringUtils.isNotBlank(params.getKmsKeyId()))
            {
                copyObjectRequest.withSSEAwsKeyManagementParams(new SSEAwsKeyManagementParams(params.getKmsKeyId()));
            }
            // Otherwise, specify the server-side encryption algorithm for encrypting the object using AWS-managed keys.
            else
            {
                ObjectMetadata metadata = new ObjectMetadata();
                metadata.setSSEAlgorithm(ObjectMetadata.AES_256_SERVER_SIDE_ENCRYPTION);
                copyObjectRequest.setNewObjectMetadata(metadata);
            }

            return s3Operations.copyFile(copyObjectRequest, transferManager);
        }
    });

    LOGGER.info("Copied S3 object. sourceS3Key=\"{}\" sourceS3BucketName=\"{}\" targetS3Key=\"{}\" targetS3BucketName=\"{}\" " +
            "totalBytesTransferred={} transferDuration=\"{}\"", params.getSourceObjectKey(), params.getSourceBucketName(), params.getTargetObjectKey(),
        params.getTargetBucketName(), results.getTotalBytesTransferred(), HerdDateUtils.formatDuration(results.getDurationMillis()));

    logOverallTransferRate(results);

    return results;
}
 

@Description(
    "KMS key id for SSE-KMS encryption, e.g. \"arn:aws:kms:...\"."
        + "To specify on the command-line, represent the value as a JSON object. For example:"
        + " --SSEAwsKeyManagementParams={\"awsKmsKeyId\": \"arn:aws:kms:...\"}")
@Nullable
SSEAwsKeyManagementParams getSSEAwsKeyManagementParams();
 

@Override
public Void invoke(File localFile, VirtualChannel channel) throws IOException, InterruptedException {
	TransferManager mgr = TransferManagerBuilder.standard()
			.withS3Client(AWSClientFactory.create(this.amazonS3ClientOptions.createAmazonS3ClientBuilder(), this.envVars))
			.build();
	final MultipleFileUpload fileUpload;
	ObjectMetadataProvider metadatasProvider = (file, meta) -> {
		if (meta != null) {
			if (RemoteListUploader.this.metadatas != null && RemoteListUploader.this.metadatas.size() > 0) {
				meta.setUserMetadata(RemoteListUploader.this.metadatas);
			}
			if (RemoteListUploader.this.acl != null) {
				meta.setHeader(Headers.S3_CANNED_ACL, RemoteListUploader.this.acl);
			}
			if (RemoteListUploader.this.cacheControl != null && !RemoteListUploader.this.cacheControl.isEmpty()) {
				meta.setCacheControl(RemoteListUploader.this.cacheControl);
			}
			if (RemoteListUploader.this.contentEncoding != null && !RemoteListUploader.this.contentEncoding.isEmpty()) {
				meta.setContentEncoding(RemoteListUploader.this.contentEncoding);
			}
			if (RemoteListUploader.this.contentType != null && !RemoteListUploader.this.contentType.isEmpty()) {
				meta.setContentType(RemoteListUploader.this.contentType);
			}
			if (RemoteListUploader.this.sseAlgorithm != null && !RemoteListUploader.this.sseAlgorithm.isEmpty()) {
				meta.setSSEAlgorithm(RemoteListUploader.this.sseAlgorithm);
			}
			if (RemoteListUploader.this.kmsId != null && !RemoteListUploader.this.kmsId.isEmpty()) {
				final SSEAwsKeyManagementParams sseAwsKeyManagementParams = new SSEAwsKeyManagementParams(RemoteListUploader.this.kmsId);
				meta.setSSEAlgorithm(sseAwsKeyManagementParams.getAwsKmsKeyId());
				meta.setHeader(
						Headers.SERVER_SIDE_ENCRYPTION_AWS_KMS_KEYID,
						sseAwsKeyManagementParams.getAwsKmsKeyId()
				);
			}

		}
	};

	ObjectTaggingProvider objectTaggingProvider =(uploadContext) -> {
		List<Tag> tagList = new ArrayList<Tag>();

		//add tags
		if(tags != null){
			for (Map.Entry<String, String> entry : tags.entrySet()) {
				Tag tag = new Tag(entry.getKey(), entry.getValue());
				tagList.add(tag);
			}
		}
		return new ObjectTagging(tagList);
	};

	try {
		fileUpload = mgr.uploadFileList(this.bucket, this.path, localFile, this.fileList, metadatasProvider, objectTaggingProvider);
		for (final Upload upload : fileUpload.getSubTransfers()) {
			upload.addProgressListener((ProgressListener) progressEvent -> {
				if (progressEvent.getEventType() == ProgressEventType.TRANSFER_COMPLETED_EVENT) {
					RemoteListUploader.this.taskListener.getLogger().println("Finished: " + upload.getDescription());
				}
			});
		}
		fileUpload.waitForCompletion();
	}
	finally {
		mgr.shutdownNow();
	}
	return null;
}
 

@Override
public String run() throws Exception {
	final String fromBucket = this.step.getFromBucket();
	final String toBucket = this.step.getToBucket();
	final String fromPath = this.step.getFromPath();
	final String toPath = this.step.getToPath();
	final String kmsId = this.step.getKmsId();
	final Map<String, String> metadatas = new HashMap<>();
	final CannedAccessControlList acl = this.step.getAcl();
	final String cacheControl = this.step.getCacheControl();
	final String contentType = this.step.getContentType();
	final String sseAlgorithm = this.step.getSseAlgorithm();
	final S3ClientOptions s3ClientOptions = this.step.createS3ClientOptions();
	final EnvVars envVars = this.getContext().get(EnvVars.class);

	if (this.step.getMetadatas() != null && this.step.getMetadatas().length != 0) {
		for (String metadata : this.step.getMetadatas()) {
			if (metadata.split(":").length == 2) {
				metadatas.put(metadata.split(":")[0], metadata.split(":")[1]);
			}
		}
	}

	Preconditions.checkArgument(fromBucket != null && !fromBucket.isEmpty(), "From bucket must not be null or empty");
	Preconditions.checkArgument(fromPath != null && !fromPath.isEmpty(), "From path must not be null or empty");
	Preconditions.checkArgument(toBucket != null && !toBucket.isEmpty(), "To bucket must not be null or empty");
	Preconditions.checkArgument(toPath != null && !toPath.isEmpty(), "To path must not be null or empty");

	TaskListener listener = Execution.this.getContext().get(TaskListener.class);
	listener.getLogger().format("Copying s3://%s/%s to s3://%s/%s%n", fromBucket, fromPath, toBucket, toPath);

	CopyObjectRequest request = new CopyObjectRequest(fromBucket, fromPath, toBucket, toPath);

	// Add metadata
	if (metadatas.size() > 0 || (cacheControl != null && !cacheControl.isEmpty()) || (contentType != null && !contentType.isEmpty()) || (sseAlgorithm != null && !sseAlgorithm.isEmpty())) {
		ObjectMetadata metas = new ObjectMetadata();
		if (metadatas.size() > 0) {
			metas.setUserMetadata(metadatas);
		}
		if (cacheControl != null && !cacheControl.isEmpty()) {
			metas.setCacheControl(cacheControl);
		}
		if (contentType != null && !contentType.isEmpty()) {
			metas.setContentType(contentType);
		}
		if (sseAlgorithm != null && !sseAlgorithm.isEmpty()) {
			metas.setSSEAlgorithm(sseAlgorithm);
		}
		request.withNewObjectMetadata(metas);
	}

	// Add acl
	if (acl != null) {
		request.withCannedAccessControlList(acl);
	}

	// Add kms
	if (kmsId != null && !kmsId.isEmpty()) {
		listener.getLogger().format("Using KMS: %s%n", kmsId);
		request.withSSEAwsKeyManagementParams(new SSEAwsKeyManagementParams(kmsId));
	}

	TransferManager mgr = TransferManagerBuilder.standard()
			.withS3Client(AWSClientFactory.create(s3ClientOptions.createAmazonS3ClientBuilder(), envVars))
			.build();
	try {
		final Copy copy = mgr.copy(request);
		copy.addProgressListener((ProgressListener) progressEvent -> {
			if (progressEvent.getEventType() == ProgressEventType.TRANSFER_COMPLETED_EVENT) {
				listener.getLogger().println("Finished: " + copy.getDescription());
			}
		});
		copy.waitForCompletion();
	}
	finally{
		mgr.shutdownNow();
	}

	listener.getLogger().println("Copy complete");
	return String.format("s3://%s/%s", toBucket, toPath);
}
 

@Override
public void configurePutObjectRequest(PutObjectRequest request, ObjectMetadata objectMetadata, String keyValue) {
    SSEAwsKeyManagementParams keyParams = new SSEAwsKeyManagementParams(keyValue);
    request.setSSEAwsKeyManagementParams(keyParams);
}
 

@Override
public void configureInitiateMultipartUploadRequest(InitiateMultipartUploadRequest request, ObjectMetadata objectMetadata, String keyValue) {
    SSEAwsKeyManagementParams keyParams = new SSEAwsKeyManagementParams(keyValue);
    request.setSSEAwsKeyManagementParams(keyParams);
}
 

void setSSEAwsKeyManagementParams(SSEAwsKeyManagementParams value);