Java Code Examples for com.amazonaws.services.s3.AmazonS3#putObject()

/**
 * Uses AES/GCM with AESWrap key wrapping to encrypt the key. Uses v2 metadata schema. Note that authenticated
 * encryption requires the bouncy castle provider to be on the classpath. Also, for authenticated encryption the size
 * of the data can be no longer than 64 GB.
 */
// snippet-start:[s3.java1.s3_encrypt.authenticated_encryption]
public void authenticatedEncryption_CustomerManagedKey() throws NoSuchAlgorithmException {
    // snippet-start:[s3.java1.s3_encrypt.authenticated_encryption_build]
    SecretKey secretKey = KeyGenerator.getInstance("AES").generateKey();
    AmazonS3Encryption s3Encryption = AmazonS3EncryptionClientBuilder
            .standard()
            .withRegion(Regions.US_WEST_2)
            .withCryptoConfiguration(new CryptoConfiguration(CryptoMode.AuthenticatedEncryption))
            .withEncryptionMaterials(new StaticEncryptionMaterialsProvider(new EncryptionMaterials(secretKey)))
            .build();

    AmazonS3 s3NonEncrypt = AmazonS3ClientBuilder.standard().withRegion(Regions.DEFAULT_REGION).build();
    // snippet-end:[s3.java1.s3_encrypt.authenticated_encryption_build]

    s3Encryption.putObject(BUCKET_NAME, ENCRYPTED_KEY, "some contents");
    s3NonEncrypt.putObject(BUCKET_NAME, NON_ENCRYPTED_KEY, "some other contents");
    System.out.println(s3Encryption.getObjectAsString(BUCKET_NAME, ENCRYPTED_KEY));
    System.out.println(s3Encryption.getObjectAsString(BUCKET_NAME, NON_ENCRYPTED_KEY));
}
 

@Override
public void upload(final AmazonS3 s3, final String bucket, final String key, final InputStream contents) {
  try (InputStream input = new BufferedInputStream(contents, chunkSize)) {
    log.debug("Starting upload to key {} in bucket {}", key, bucket);

    input.mark(chunkSize);
    ChunkReader chunkReader = new ChunkReader(input);
    Chunk chunk = chunkReader.readChunk(chunkSize).orElse(EMPTY_CHUNK);
    input.reset();

    if (chunk.dataLength < chunkSize) {
      ObjectMetadata metadata = new ObjectMetadata();
      metadata.setContentLength(chunk.dataLength);
      s3.putObject(bucket, key, new ByteArrayInputStream(chunk.data, 0, chunk.dataLength), metadata);
    }
    else {
      ChunkReader parallelReader = new ChunkReader(input);
      parallelRequests(s3, bucket, key,
          () -> (uploadId -> uploadChunks(s3, bucket, key, uploadId, parallelReader)));
    }
    log.debug("Finished upload to key {} in bucket {}", key, bucket);
  }
  catch (IOException | SdkClientException e) { // NOSONAR
    throw new BlobStoreException(format("Error uploading blob to bucket:%s key:%s", bucket, key), e, null);
  }
}
 

protected final void updateStack(final String stackName, final String template, final Parameter... parameters) {
    UpdateStackRequest req = new UpdateStackRequest()
            .withStackName(stackName)
            .withParameters(parameters)
            .withCapabilities(Capability.CAPABILITY_IAM);
    if (Config.has(Config.Key.TEMPLATE_DIR)) {
        final String dir = Config.get(Config.Key.TEMPLATE_DIR);
        if (Config.has(Config.Key.BUCKET_NAME)) {
            final String bucketName = Config.get(Config.Key.BUCKET_NAME);
            final String bucketRegion = Config.get(Config.Key.BUCKET_REGION);
            final AmazonS3 s3local = AmazonS3ClientBuilder.standard().withCredentials(this.credentialsProvider).withRegion(bucketRegion).build();
            s3local.putObject(bucketName, stackName, new File(dir + template));
            req = req.withTemplateURL("https://s3-" + bucketRegion + ".amazonaws.com/" + bucketName + "/" + stackName);
        } else {
            final String body = readFile(dir + template, Charset.forName("UTF-8"));
            req = req.withTemplateBody(body);
        }
    } else {
        req = req.withTemplateURL("https://s3-eu-west-1.amazonaws.com/widdix-aws-cf-templates/" + template);
    }
    this.cf.updateStack(req);
    this.waitForStack(stackName, FinalStatus.UPDATE_COMPLETE);
}
 

/**
 * This uses the V2 metadata schema with a key wrap algorithm of 'kms' and a CEK algorithm of AES/CBC/PKCS5Padding.
 */
// snippet-start:[s3.java1.s3_encrypt.kms_encryption_only]
public void encryptionOnly_KmsManagedKey() throws NoSuchAlgorithmException {
    // snippet-start:[s3.java1.s3_encrypt.kms_encryption_only_build]
    AmazonS3Encryption s3Encryption = AmazonS3EncryptionClientBuilder
            .standard()
            .withRegion(Regions.US_WEST_2)
            .withCryptoConfiguration(new CryptoConfiguration(CryptoMode.EncryptionOnly).withAwsKmsRegion(Region.getRegion(Regions.US_WEST_2)))
            // Can either be Key ID or alias (prefixed with 'alias/')
            .withEncryptionMaterials(new KMSEncryptionMaterialsProvider("alias/s3-kms-key"))
            .build();

    AmazonS3 s3NonEncrypt = AmazonS3ClientBuilder.standard().withRegion(Regions.DEFAULT_REGION).build();
    // snippet-end:[s3.java1.s3_encrypt.kms_encryption_only_build]

    // snippet-start:[s3.java1.s3_encrypt.kms_encryption_only_put_object]
    s3Encryption.putObject(BUCKET_NAME, ENCRYPTED_KEY, "some contents");
    s3NonEncrypt.putObject(BUCKET_NAME, NON_ENCRYPTED_KEY, "some other contents");
    // snippet-end:[s3.java1.s3_encrypt.kms_encryption_only_put_object]
    // snippet-start:[s3.java1.s3_encrypt.kms_encryption_only_retrieve]
    System.out.println(s3Encryption.getObjectAsString(BUCKET_NAME, ENCRYPTED_KEY));
    System.out.println(s3Encryption.getObjectAsString(BUCKET_NAME, NON_ENCRYPTED_KEY));
    // snippet-end:[s3.java1.s3_encrypt.kms_encryption_only_retrieve]
}
 

/**
 * Uses AES/CBC algorithm, no key wrapping.
 */
public void encryptionOnly_CustomerManagedKey() throws NoSuchAlgorithmException {
    SecretKey secretKey = KeyGenerator.getInstance("AES").generateKey();
    AmazonS3Encryption s3Encryption = AmazonS3EncryptionClientBuilder
            .standard()
            .withRegion(Regions.US_WEST_2)
            .withCryptoConfiguration(new CryptoConfiguration(CryptoMode.EncryptionOnly))
            .withEncryptionMaterials(new StaticEncryptionMaterialsProvider(new EncryptionMaterials(secretKey)))
            .build();

    AmazonS3 s3NonEncrypt = AmazonS3ClientBuilder.standard().withRegion(Regions.DEFAULT_REGION).build();

    s3Encryption.putObject(BUCKET_NAME, ENCRYPTED_KEY, "some contents");
    s3NonEncrypt.putObject(BUCKET_NAME, NON_ENCRYPTED_KEY, "some other contents");
    System.out.println(s3Encryption.getObjectAsString(BUCKET_NAME, ENCRYPTED_KEY));
    System.out.println(s3Encryption.getObjectAsString(BUCKET_NAME, NON_ENCRYPTED_KEY));
}
 

/**
 * Uses AES/GCM with AESWrap key wrapping to encrypt the key. Uses v2 metadata schema. The only difference between this and
 * {@link #authenticatedEncryption_CustomerManagedKey()} is that attempting to retrieve an object non
 * encrypted with AES/GCM will thrown an exception instead of falling back to encryption only or plaintext GET.
 */
// snippet-start:[s3.java1.s3_encrypt.strict_authenticated_encryption]
public void strictAuthenticatedEncryption_CustomerManagedKey() throws NoSuchAlgorithmException {
    // snippet-start:[s3.java1.s3_encrypt.strict_authenticated_encryption_build]
    SecretKey secretKey = KeyGenerator.getInstance("AES").generateKey();
    AmazonS3Encryption s3Encryption = AmazonS3EncryptionClientBuilder
            .standard()
            .withRegion(Regions.US_WEST_2)
            .withCryptoConfiguration(new CryptoConfiguration(CryptoMode.StrictAuthenticatedEncryption))
            .withEncryptionMaterials(new StaticEncryptionMaterialsProvider(new EncryptionMaterials(secretKey)))
            .build();

    AmazonS3 s3NonEncrypt = AmazonS3ClientBuilder.standard().withRegion(Regions.DEFAULT_REGION).build();
    // snippet-end:[s3.java1.s3_encrypt.strict_authenticated_encryption_build]

    s3Encryption.putObject(BUCKET_NAME, ENCRYPTED_KEY, "some contents");
    s3NonEncrypt.putObject(BUCKET_NAME, NON_ENCRYPTED_KEY, "some other contents");
    System.out.println(s3Encryption.getObjectAsString(BUCKET_NAME, ENCRYPTED_KEY));
    try {
        s3Encryption.getObjectAsString(BUCKET_NAME, NON_ENCRYPTED_KEY);
    } catch (SecurityException e) {
        // Strict authenticated encryption will throw an exception if an object is not encrypted with AES/GCM
        System.err.println(NON_ENCRYPTED_KEY + " was not encrypted with AES/GCM");
    }
}
 

@Test
public void testS3PutObjectSubsegmentContainsBucketName() {
    // Setup test
    AmazonS3 s3 = AmazonS3ClientBuilder
        .standard()
        .withRequestHandlers(new TracingHandler())
        .withRegion(Regions.US_EAST_1)
        .withCredentials(new AWSStaticCredentialsProvider(new BasicAWSCredentials("fake", "fake")))
        .build();
    mockHttpClient(s3, null);

    String bucket = "test-bucket";
    String key = "test-key";
    // Test logic 
    Segment segment = AWSXRay.beginSegment("test");
    s3.putObject(bucket, key, "This is a test from java");
    Assert.assertEquals(1, segment.getSubsegments().size());
    Assert.assertEquals("PutObject", segment.getSubsegments().get(0).getAws().get("operation"));
    Assert.assertEquals(bucket, segment.getSubsegments().get(0).getAws().get("bucket_name"));
    Assert.assertEquals(key, segment.getSubsegments().get(0).getAws().get("key"));
}
 

@BeforeClass
public static void setupResources() throws IOException {

    bucketName = UUID.randomUUID().toString();
    testFileToDownload = "test-file-download.txt";
    testFileToUpload = "test-file-upload.txt";

    filesWithSimilarName = new String[] { "foo/hello-apple.txt", "foo/hello-orange.txt", "bar/hello-grapes.txt", };

    similarNameFiles = new ArrayList<>();
    for (String name : filesWithSimilarName) {
        similarNameFiles.add(new File(name.substring(0, name.lastIndexOf("/") + 1)));
    }

    Files.write(Paths.get(testFileToUpload), "Hello World Uploaded!".getBytes());

    AmazonS3 amazonS3 = SpringCloudAwsTestUtil.amazonS3();
    amazonS3.createBucket(bucketName);

    amazonS3.putObject(bucketName, testFileToDownload, "Hello World");

    for (String s3Key : filesWithSimilarName) {
        amazonS3.putObject(bucketName, s3Key, "Hello World");
    }
}
 

@Override
public void addNewStateToDatastore(String state) throws Exception {
    AmazonS3 s3 = this.createS3Client();
    String value = "" + (System.currentTimeMillis() + getExpirationInSeconds() * 1000);
    PutObjectResult putObjectResult = s3.putObject(bucketName, getKey(state), value);
    if (log.isDebugEnabled()) {
        log.debug("AWS S3 putObject result of state data - {}", JsonOps.toJsonString(putObjectResult));
    }
}
 

public static void putMirthS3(String bucketName, String key, String fileLocation, String fileContents) {
AmazonS3 s3 = new AmazonS3Client();
Region usWest2 = Region.getRegion(Regions.US_WEST_2);
s3.setRegion(usWest2);
try {
s3.putObject(new PutObjectRequest(bucketName, key, createTmpFile(fileContents)));
} catch (Exception e) { System.out.println("ERROR ON TEXT FILE"); }
   }
 

/**
 * Checking S3 connection with a put request.
 * This function throws a runtime exception if something goes wrong.
 *
 * @param s3
 * @param bucket
 * @param key
 */
public void checkS3Put(AmazonS3 s3, String bucket, String key) {
  try {
    byte[] source = { 0x0 };
    ByteArrayInputStream is = new ByteArrayInputStream(source);

    final PutObjectRequest object = new PutObjectRequest(bucket, key, is, new ObjectMetadata());
    s3.putObject(object);
  } catch (Exception e) {
    log.error("Unable to write an object to AWS S3: bucket={}, key={}.", bucket, key);
    throw e;
  }
}
 

private void putObjectToS3(ResultSet resultSet, int taskId, AmazonS3 s3Client, String bucketName) throws SQLException {

        ResultSetMetaData rsmd = resultSet.getMetaData();
        int columnCount = rsmd.getColumnCount();

        // Get content column index
        int rowContentColumnIndex = 0;
        for (int i = 1; i <= columnCount; i++) {
            if (rsmd.getColumnName(i).toUpperCase().equals(rowContentColumnName.toUpperCase())) {
                rowContentColumnIndex = i;
            }
        }

        ObjectMetadata binMetadata = new ObjectMetadata();

        while (resultSet.next()) {

            switch (rsmd.getColumnType(rowContentColumnIndex)) {
                case Types.BINARY:
                case Types.BLOB:
                case Types.CLOB:
                    s3Client.putObject(bucketName, resultSet.getString(rowKeyColumnName), resultSet.getBinaryStream(rowContentColumnName), binMetadata);
                    break;
                case Types.SQLXML:
                    throw new IllegalArgumentException("SQLXML Data Type is not supported. You should convert it to BLOB or CLOB");
                default:
                    s3Client.putObject(bucketName, resultSet.getString(rowKeyColumnName), resultSet.getString(rowContentColumnName));
                    break;
            }
        }
    }
 

/**
 * Same as authenticatedEncryption_KmsManagedKey except throws an exception when trying to get objects not encrypted with
 * AES/GCM.
 */
// snippet-start:[s3.java1.s3_encrypt.kms_authenticated_encryption_strict]
public void strictAuthenticatedEncryption_KmsManagedKey() throws NoSuchAlgorithmException {
    // snippet-start:[s3.java1.s3_encrypt.kms_authenticated_encryption_strict_builder]
    AmazonS3Encryption s3Encryption = AmazonS3EncryptionClientBuilder
            .standard()
            .withRegion(Regions.US_WEST_2)
            .withCryptoConfiguration(new CryptoConfiguration(CryptoMode.StrictAuthenticatedEncryption).withAwsKmsRegion(Region.getRegion(Regions.US_WEST_2)))
            // Can either be Key ID or alias (prefixed with 'alias/')
            .withEncryptionMaterials(new KMSEncryptionMaterialsProvider("alias/s3-kms-key"))
            .build();

    AmazonS3 s3NonEncrypt = AmazonS3ClientBuilder.standard().withRegion(Regions.DEFAULT_REGION).build();
    // snippet-end:[s3.java1.s3_encrypt.kms_authenticated_encryption_strict_builder]

    // snippet-start:[s3.java1.s3_encrypt.kms_authenticated_encryption_strict_put_object]
    s3Encryption.putObject(BUCKET_NAME, ENCRYPTED_KEY, "some contents");
    s3NonEncrypt.putObject(BUCKET_NAME, NON_ENCRYPTED_KEY, "some other contents");
    // snippet-end:[s3.java1.s3_encrypt.kms_authenticated_encryption_strict_put_object]
    // snippet-start:[s3.java1.s3_encrypt.kms_authenticated_encryption_strict_exception]
    try {
        s3Encryption.getObjectAsString(BUCKET_NAME, NON_ENCRYPTED_KEY);
    } catch (SecurityException e) {
        // Strict authenticated encryption will throw an exception if an object is not encrypted with AES/GCM
        System.err.println(NON_ENCRYPTED_KEY + " was not encrypted with AES/GCM");
    }

    // snippet-end:[s3.java1.s3_encrypt.kms_authenticated_encryption_strict_exception]
}
 

/**
 * This function creates empty folder in a s3 bucket
 * @param bucketName
 * @param folderName
 * @param client
 */
public static void createFolder(String bucketName, String folderName, AmazonS3 client) {
	final String SUFFIX = "/";
	
	// create meta-data for your folder and set content-length to 0
	ObjectMetadata metadata = new ObjectMetadata();
	metadata.setContentLength(0);
	// create empty content
	InputStream emptyContent = new ByteArrayInputStream(new byte[0]);
	// create a PutObjectRequest passing the folder name suffixed by /
	PutObjectRequest putObjectRequest = new PutObjectRequest(bucketName,
				folderName + SUFFIX, emptyContent, metadata);
	// send request to S3 to create folder
	client.putObject(putObjectRequest);
}
 

@Test
@Ignore
public void testDownloadOfRemoteManifest() throws Exception {
    S3BucketService s3BucketService = new S3BucketService(getTransferManagerFactory(), getBackupOperationRequest());

    try {
        s3BucketService.create(BUCKET_NAME);

        AmazonS3 amazonS3Client = getTransferManagerFactory().build(getBackupOperationRequest()).getAmazonS3Client();

        amazonS3Client.putObject(BUCKET_NAME, "cluster/dc/node/manifests/snapshot-name-" + BUCKET_NAME, "hello");

        Thread.sleep(5000);

        ObjectListing objectListing = amazonS3Client.listObjects(BUCKET_NAME);

        objectListing.getObjectSummaries().forEach(summary -> System.out.println(summary.getKey()));

        RestoreOperationRequest restoreOperationRequest = new RestoreOperationRequest();
        restoreOperationRequest.storageLocation = new StorageLocation("s3://" + BUCKET_NAME + "/cluster/dc/node");

        S3Restorer s3Restorer = new S3Restorer(getTransferManagerFactory(), new FixedTasksExecutorSupplier(), restoreOperationRequest);

        final Path downloadedFile = s3Restorer.downloadFileToDir(Paths.get("/tmp"), Paths.get("manifests"), new Predicate<String>() {
            @Override
            public boolean test(final String s) {
                return s.contains("manifests/snapshot-name");
            }
        });

        assertTrue(Files.exists(downloadedFile));
    } finally {
        s3BucketService.delete(BUCKET_NAME);
        deleteDirectory(Paths.get(target("commitlog_download_dir")));
    }
}
 

public static void s3Put(AmazonS3 s3, String bucket, String key, String resource)
        throws IOException
{
    logger.info("put {} -> s3://{}/{}", resource, bucket, key);
    URL resourceUrl = Resources.getResource(resource);
    byte[] bytes = Resources.toByteArray(resourceUrl);
    ObjectMetadata metadata = new ObjectMetadata();
    metadata.setContentLength(bytes.length);
    s3.putObject(bucket, key, new ByteArrayInputStream(bytes), metadata);
}
 

private void save(AmazonS3 s3, String s3Key, String json, String logMessage) {
    PutObjectResult botPutResult = s3.putObject(bucketName, s3Key, json);
    if (log.isDebugEnabled()) {
        log.debug(logMessage, s3Key, JsonOps.toJsonString(botPutResult));
    }
}
 

public String handleRequest(S3Event s3event, Context context) {
    _logger = context.getLogger();
    _logger.log("Received S3 Event: " + s3event.toJson());

    try {
        S3EventNotificationRecord record = s3event.getRecords().get(0);

        String bucket = record.getS3().getBucket().getName();
        String extractBucket = "extracts." + bucket;

        // Object key may have spaces or unicode non-ASCII characters.
        String key = URLDecoder.decode(record.getS3().getObject().getKey().replace('+', ' '), "UTF-8");

        // Short-circuit ignore .extract files because they have already been extracted, this prevents an endless loop
        if (key.toLowerCase().endsWith(".extract")) {
          _logger.log("Ignoring extract file " + key);
          return "Ignored";
        }

        AmazonS3 s3Client = new AmazonS3Client();
        S3Object s3Object = s3Client.getObject(new GetObjectRequest(bucket, key));

        try (InputStream objectData = s3Object.getObjectContent()) {
            String extractJson = doTikaStuff(bucket, key, objectData);

            byte[] extractBytes = extractJson.getBytes(Charset.forName("UTF-8"));
            int extractLength = extractBytes.length;

            ObjectMetadata metaData = new ObjectMetadata();
            metaData.setContentLength(extractLength);

            _logger.log("Saving extract file to S3");
            InputStream inputStream = new ByteArrayInputStream(extractBytes);
            s3Client.putObject(extractBucket, key + ".extract", inputStream, metaData);
        }
    } catch (IOException | TransformerConfigurationException | SAXException e) {
        _logger.log("Exception: " + e.getLocalizedMessage());
        throw new RuntimeException(e);
    }
    return "Success";
}
 

@Override
public PutObjectResult putObject(PutObjectRequest putObjectRequest, AmazonS3 s3Client)
{
    return s3Client.putObject(putObjectRequest);
}
 

/**
 * CLO-8589: set content length to 0 to prevent a warning being logged.
 * 
 * @param service
 * @param bucketName
 * @param key
 */
public static void createEmptyObject(AmazonS3 service, String bucketName, String key) {
	ObjectMetadata metadata = createPutObjectMetadata();
	metadata.setContentLength(0);
	service.putObject(bucketName, key, new ByteArrayInputStream(new byte[0]), metadata);
}