Java Code Examples for com.amazonaws.services.s3.model.ObjectMetadata#setContentLength()
The following examples show how to use
com.amazonaws.services.s3.model.ObjectMetadata#setContentLength() .
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: AwsSdkTest.java From s3proxy with Apache License 2.0 | 6 votes |
@Test public void testDeleteMultipleObjects() throws Exception { String blobName = "foo"; ObjectMetadata metadata = new ObjectMetadata(); metadata.setContentLength(BYTE_SOURCE.size()); DeleteObjectsRequest request = new DeleteObjectsRequest(containerName) .withKeys(blobName); // without quiet client.putObject(containerName, blobName, BYTE_SOURCE.openStream(), metadata); DeleteObjectsResult result = client.deleteObjects(request); assertThat(result.getDeletedObjects()).hasSize(1); assertThat(result.getDeletedObjects().iterator().next().getKey()) .isEqualTo(blobName); // with quiet client.putObject(containerName, blobName, BYTE_SOURCE.openStream(), metadata); result = client.deleteObjects(request.withQuiet(true)); assertThat(result.getDeletedObjects()).isEmpty(); }
Example 2
Source File: AwsSdkTest.java From s3proxy with Apache License 2.0 | 6 votes |
@Test public void testUnicodeObject() throws Exception { String blobName = "ŪņЇЌœđЗ/☺ unicode € rocks ™"; ObjectMetadata metadata = new ObjectMetadata(); metadata.setContentLength(BYTE_SOURCE.size()); client.putObject(containerName, blobName, BYTE_SOURCE.openStream(), metadata); metadata = client.getObjectMetadata(containerName, blobName); assertThat(metadata).isNotNull(); ObjectListing listing = client.listObjects(containerName); List<S3ObjectSummary> summaries = listing.getObjectSummaries(); assertThat(summaries).hasSize(1); S3ObjectSummary summary = summaries.iterator().next(); assertThat(summary.getKey()).isEqualTo(blobName); }
Example 3
Source File: AwsSdkTest.java From s3proxy with Apache License 2.0 | 6 votes |
@Test public void testHttpClient() throws Exception { String blobName = "blob-name"; ObjectMetadata metadata = new ObjectMetadata(); metadata.setContentLength(BYTE_SOURCE.size()); client.putObject(containerName, blobName, BYTE_SOURCE.openStream(), metadata); if (Quirks.NO_BLOB_ACCESS_CONTROL.contains(blobStoreType)) { client.setBucketAcl(containerName, CannedAccessControlList.PublicRead); } else { client.setObjectAcl(containerName, blobName, CannedAccessControlList.PublicRead); } HttpClient httpClient = context.utils().http(); URI uri = new URI(s3Endpoint.getScheme(), s3Endpoint.getUserInfo(), s3Endpoint.getHost(), s3Proxy.getSecurePort(), servicePath + "/" + containerName + "/" + blobName, /*query=*/ null, /*fragment=*/ null); try (InputStream actual = httpClient.get(uri); InputStream expected = BYTE_SOURCE.openStream()) { assertThat(actual).hasContentEqualTo(expected); } }
Example 4
Source File: AwsSdkAnonymousTest.java From s3proxy with Apache License 2.0 | 6 votes |
@Test public void testAwsV4SignatureChunkedAnonymous() throws Exception { client = AmazonS3ClientBuilder.standard() .withChunkedEncodingDisabled(false) .withEndpointConfiguration(s3EndpointConfig) .build(); ObjectMetadata metadata = new ObjectMetadata(); metadata.setContentLength(BYTE_SOURCE.size()); client.putObject(containerName, "foo", BYTE_SOURCE.openStream(), metadata); S3Object object = client.getObject(containerName, "foo"); assertThat(object.getObjectMetadata().getContentLength()).isEqualTo( BYTE_SOURCE.size()); try (InputStream actual = object.getObjectContent(); InputStream expected = BYTE_SOURCE.openStream()) { assertThat(actual).hasContentEqualTo(expected); } }
Example 5
Source File: S3FastLoader.java From pocket-etl with Apache License 2.0 | 6 votes |
private void writeBufferToS3(byte[] toWrite, int limit) { try (EtlProfilingScope scope = new EtlProfilingScope(parentMetrics, "S3FastLoader.writeToS3")) { InputStream inputStream = new ByteArrayInputStream(toWrite, 0, limit); String s3Key = s3PartFileKeyGenerator.apply(++fileSequenceNumber); ObjectMetadata metadata = new ObjectMetadata(); metadata.setContentLength(limit); PutObjectRequest putObjectRequest = new PutObjectRequest(s3Bucket, s3Key, inputStream, metadata); if (sseKmsArn != null && !sseKmsArn.isEmpty()) { putObjectRequest.setSSEAwsKeyManagementParams(new SSEAwsKeyManagementParams(sseKmsArn)); } try { amazonS3.putObject(putObjectRequest); emitSuccessAndFailureMetrics(scope, true); } catch (AmazonClientException e) { logger.error(e); scope.addCounter(e.getClass().getSimpleName(), 1); emitSuccessAndFailureMetrics(scope, false); throw new UnrecoverableStreamFailureException("Exception caught trying to write object to S3: ", e); } } }
Example 6
Source File: S3DataGenerator.java From aws-big-data-blog with Apache License 2.0 | 6 votes |
private void putObject(final String key, final byte[] content, Map<String, String> userMetadata) { final ObjectMetadata metadata = new ObjectMetadata(); metadata.setContentLength(content.length); metadata.setContentMD5(md5b64(content)); metadata.setUserMetadata(userMetadata); final PutObjectRequest request = new PutObjectRequest(bucket, key, new ByteArrayInputStream(content), metadata); S3.putObject(request); }
Example 7
Source File: AwsSdkTest.java From s3proxy with Apache License 2.0 | 6 votes |
@Test public void testBlobRemove() throws Exception { String blobName = "blob"; ObjectMetadata metadata = new ObjectMetadata(); metadata.setContentLength(BYTE_SOURCE.size()); client.putObject(containerName, blobName, BYTE_SOURCE.openStream(), metadata); assertThat(client.getObjectMetadata(containerName, blobName)) .isNotNull(); client.deleteObject(containerName, blobName); try { client.getObjectMetadata(containerName, blobName); Fail.failBecauseExceptionWasNotThrown(AmazonS3Exception.class); } catch (AmazonS3Exception e) { assertThat(e.getErrorCode()).isEqualTo("404 Not Found"); } client.deleteObject(containerName, blobName); }
Example 8
Source File: AwsS3BuildCacheService.java From gradle-s3-build-cache with Apache License 2.0 | 6 votes |
@Override public void store(BuildCacheKey key, BuildCacheEntryWriter writer) { final String bucketPath = getBucketPath(key); logger.info("Start storing cache entry '{}' in S3 bucket", bucketPath); ObjectMetadata meta = new ObjectMetadata(); meta.setContentType(BUILD_CACHE_CONTENT_TYPE); try (ByteArrayOutputStream os = new ByteArrayOutputStream()) { writer.writeTo(os); meta.setContentLength(os.size()); try (InputStream is = new ByteArrayInputStream(os.toByteArray())) { PutObjectRequest request = getPutObjectRequest(bucketPath, meta, is); if(this.reducedRedundancy) { request.withStorageClass(StorageClass.ReducedRedundancy); } s3.putObject(request); } } catch (IOException e) { throw new BuildCacheException("Error while storing cache object in S3 bucket", e); } }
Example 9
Source File: S3ObjectWriter.java From micro-server with Apache License 2.0 | 5 votes |
/** * Metadata object creation * @param length * * @return Metadata with AES_256 encryption if enabled */ private ObjectMetadata createMetadata(int length) { ObjectMetadata metadata = new ObjectMetadata(); metadata.setContentLength(length); if (aes256Encryption) metadata.setSSEAlgorithm(ObjectMetadata.AES_256_SERVER_SIDE_ENCRYPTION); return metadata; }
Example 10
Source File: S3FileSystemTest.java From beam with Apache License 2.0 | 5 votes |
@Test public void matchNonGlobNotReadSeekEfficient() { S3FileSystem s3FileSystem = buildMockedS3FileSystem(s3Options()); S3ResourceId path = S3ResourceId.fromUri("s3://testbucket/testdirectory/filethatexists"); long lastModifiedMillis = 1540000000000L; ObjectMetadata s3ObjectMetadata = new ObjectMetadata(); s3ObjectMetadata.setContentLength(100); s3ObjectMetadata.setLastModified(new Date(lastModifiedMillis)); s3ObjectMetadata.setContentEncoding("gzip"); when(s3FileSystem .getAmazonS3Client() .getObjectMetadata( argThat( new GetObjectMetadataRequestMatcher( new GetObjectMetadataRequest(path.getBucket(), path.getKey()))))) .thenReturn(s3ObjectMetadata); MatchResult result = s3FileSystem.matchNonGlobPath(path); assertThat( result, MatchResultMatcher.create( ImmutableList.of( MatchResult.Metadata.builder() .setSizeBytes(100) .setLastModifiedMillis(lastModifiedMillis) .setResourceId(path) .setIsReadSeekEfficient(false) .build()))); }
Example 11
Source File: UploadThread.java From aws-photosharing-example with Apache License 2.0 | 5 votes |
@Override public void run() { ObjectMetadata meta_data = new ObjectMetadata(); if (p_content_type != null) meta_data.setContentType(p_content_type); meta_data.setContentLength(p_size); PutObjectRequest putObjectRequest = new PutObjectRequest(p_bucket_name, p_s3_key, p_file_stream, meta_data); putObjectRequest.setCannedAcl(CannedAccessControlList.PublicRead); PutObjectResult res = s3Client.putObject(putObjectRequest); }
Example 12
Source File: S3MultiReportOutput.java From java-almanac with Creative Commons Attribution Share Alike 4.0 International | 5 votes |
private void upload(String path, byte[] content) { ObjectMetadata metadata = new ObjectMetadata(); metadata.setContentLength(content.length); metadata.setContentType(getContentType(path)); PutObjectRequest request = new PutObjectRequest(bucketName, basePath + path, new ByteArrayInputStream(content), metadata).withCannedAcl(CannedAccessControlList.PublicRead); s3Client.putObject(request); }
Example 13
Source File: AwsSdkTest.java From s3proxy with Apache License 2.0 | 5 votes |
@Test public void testAwsV2SignatureWithOverrideParameters() throws Exception { client = AmazonS3ClientBuilder.standard() .withClientConfiguration(V2_SIGNER_CONFIG) .withCredentials(new AWSStaticCredentialsProvider(awsCreds)) .withEndpointConfiguration(s3EndpointConfig).build(); ObjectMetadata metadata = new ObjectMetadata(); metadata.setContentLength(BYTE_SOURCE.size()); client.putObject(containerName, "foo", BYTE_SOURCE.openStream(), metadata); String blobName = "foo"; ResponseHeaderOverrides headerOverride = new ResponseHeaderOverrides(); String expectedContentDisposition = "attachment; " + blobName; headerOverride.setContentDisposition(expectedContentDisposition); String expectedContentType = "text/plain"; headerOverride.setContentType(expectedContentType); GetObjectRequest request = new GetObjectRequest(containerName, blobName); request.setResponseHeaders(headerOverride); S3Object object = client.getObject(request); assertThat(object.getObjectMetadata().getContentLength()).isEqualTo( BYTE_SOURCE.size()); assertThat(object.getObjectMetadata().getContentDisposition()) .isEqualTo(expectedContentDisposition); assertThat(object.getObjectMetadata().getContentType()).isEqualTo( expectedContentType); try (InputStream actual = object.getObjectContent(); InputStream expected = BYTE_SOURCE.openStream()) { assertThat(actual).hasContentEqualTo(expected); } }
Example 14
Source File: S3BlobContainer.java From crate with Apache License 2.0 | 5 votes |
/** * Uploads a blob using a single upload request */ void executeSingleUpload(final S3BlobStore blobStore, final String blobName, final InputStream input, final long blobSize) throws IOException { // Extra safety checks if (blobSize > MAX_FILE_SIZE.getBytes()) { throw new IllegalArgumentException("Upload request size [" + blobSize + "] can't be larger than " + MAX_FILE_SIZE); } if (blobSize > blobStore.bufferSizeInBytes()) { throw new IllegalArgumentException("Upload request size [" + blobSize + "] can't be larger than buffer size"); } final ObjectMetadata md = new ObjectMetadata(); md.setContentLength(blobSize); if (blobStore.serverSideEncryption()) { md.setSSEAlgorithm(ObjectMetadata.AES_256_SERVER_SIDE_ENCRYPTION); } final PutObjectRequest putRequest = new PutObjectRequest(blobStore.bucket(), blobName, input, md); putRequest.setStorageClass(blobStore.getStorageClass()); putRequest.setCannedAcl(blobStore.getCannedACL()); try (AmazonS3Reference clientReference = blobStore.clientReference()) { clientReference.client().putObject(putRequest); } catch (final AmazonClientException e) { throw new IOException("Unable to upload object [" + blobName + "] using a single upload", e); } }
Example 15
Source File: AwsSdkTest.java From s3proxy with Apache License 2.0 | 5 votes |
@Test public void testBlobMetadata() throws Exception { String blobName = "blob"; ObjectMetadata metadata = new ObjectMetadata(); metadata.setContentLength(BYTE_SOURCE.size()); client.putObject(containerName, blobName, BYTE_SOURCE.openStream(), metadata); ObjectMetadata newMetadata = client.getObjectMetadata(containerName, blobName); assertThat(newMetadata.getContentLength()) .isEqualTo(BYTE_SOURCE.size()); }
Example 16
Source File: AwsSdkTest.java From s3proxy with Apache License 2.0 | 4 votes |
@Test public void testAwsV2UrlSigningWithOverrideParameters() throws Exception { client = AmazonS3ClientBuilder.standard() .withClientConfiguration(V2_SIGNER_CONFIG) .withCredentials(new AWSStaticCredentialsProvider(awsCreds)) .withEndpointConfiguration(s3EndpointConfig).build(); String blobName = "foo"; ObjectMetadata metadata = new ObjectMetadata(); metadata.setContentLength(BYTE_SOURCE.size()); client.putObject(containerName, blobName, BYTE_SOURCE.openStream(), metadata); GeneratePresignedUrlRequest generatePresignedUrlRequest = new GeneratePresignedUrlRequest(containerName, blobName); generatePresignedUrlRequest.setMethod(HttpMethod.GET); ResponseHeaderOverrides headerOverride = new ResponseHeaderOverrides(); headerOverride.setContentDisposition("attachment; " + blobName); headerOverride.setContentType("text/plain"); generatePresignedUrlRequest.setResponseHeaders(headerOverride); Date expiration = new Date(System.currentTimeMillis() + TimeUnit.HOURS.toMillis(1)); generatePresignedUrlRequest.setExpiration(expiration); URL url = client.generatePresignedUrl(generatePresignedUrlRequest); URLConnection connection = url.openConnection(); try (InputStream actual = connection.getInputStream(); InputStream expected = BYTE_SOURCE.openStream()) { String value = connection.getHeaderField("Content-Disposition"); assertThat(value).isEqualTo(headerOverride.getContentDisposition()); value = connection.getHeaderField("Content-Type"); assertThat(value).isEqualTo(headerOverride.getContentType()); assertThat(actual).hasContentEqualTo(expected); } }
Example 17
Source File: S3FileSystem.java From stratosphere with Apache License 2.0 | 4 votes |
private void createEmptyObject(final String bucketName, final String objectName) { final InputStream im = new InputStream() { @Override public int read() throws IOException { return -1; } }; final ObjectMetadata om = new ObjectMetadata(); om.setContentLength(0L); this.s3Client.putObject(bucketName, objectName, im, om); }
Example 18
Source File: S3FileSystemTest.java From beam with Apache License 2.0 | 4 votes |
@Test public void matchVariousInvokeThreadPool() throws IOException { S3FileSystem s3FileSystem = buildMockedS3FileSystem(s3Options()); AmazonS3Exception notFoundException = new AmazonS3Exception("mock exception"); notFoundException.setStatusCode(404); S3ResourceId pathNotExist = S3ResourceId.fromUri("s3://testbucket/testdirectory/nonexistentfile"); when(s3FileSystem .getAmazonS3Client() .getObjectMetadata( argThat( new GetObjectMetadataRequestMatcher( new GetObjectMetadataRequest( pathNotExist.getBucket(), pathNotExist.getKey()))))) .thenThrow(notFoundException); AmazonS3Exception forbiddenException = new AmazonS3Exception("mock exception"); forbiddenException.setStatusCode(403); S3ResourceId pathForbidden = S3ResourceId.fromUri("s3://testbucket/testdirectory/forbiddenfile"); when(s3FileSystem .getAmazonS3Client() .getObjectMetadata( argThat( new GetObjectMetadataRequestMatcher( new GetObjectMetadataRequest( pathForbidden.getBucket(), pathForbidden.getKey()))))) .thenThrow(forbiddenException); S3ResourceId pathExist = S3ResourceId.fromUri("s3://testbucket/testdirectory/filethatexists"); ObjectMetadata s3ObjectMetadata = new ObjectMetadata(); s3ObjectMetadata.setContentLength(100); s3ObjectMetadata.setLastModified(new Date(1540000000000L)); s3ObjectMetadata.setContentEncoding("not-gzip"); when(s3FileSystem .getAmazonS3Client() .getObjectMetadata( argThat( new GetObjectMetadataRequestMatcher( new GetObjectMetadataRequest(pathExist.getBucket(), pathExist.getKey()))))) .thenReturn(s3ObjectMetadata); S3ResourceId pathGlob = S3ResourceId.fromUri("s3://testbucket/path/part*"); S3ObjectSummary foundListObject = new S3ObjectSummary(); foundListObject.setBucketName(pathGlob.getBucket()); foundListObject.setKey("path/part-0"); foundListObject.setSize(200); foundListObject.setLastModified(new Date(1541000000000L)); ListObjectsV2Result listObjectsResult = new ListObjectsV2Result(); listObjectsResult.setNextContinuationToken(null); listObjectsResult.getObjectSummaries().add(foundListObject); when(s3FileSystem.getAmazonS3Client().listObjectsV2(notNull(ListObjectsV2Request.class))) .thenReturn(listObjectsResult); ObjectMetadata metadata = new ObjectMetadata(); metadata.setContentEncoding(""); when(s3FileSystem .getAmazonS3Client() .getObjectMetadata( argThat( new GetObjectMetadataRequestMatcher( new GetObjectMetadataRequest(pathGlob.getBucket(), "path/part-0"))))) .thenReturn(metadata); assertThat( s3FileSystem.match( ImmutableList.of( pathNotExist.toString(), pathForbidden.toString(), pathExist.toString(), pathGlob.toString())), contains( MatchResultMatcher.create(MatchResult.Status.NOT_FOUND, new FileNotFoundException()), MatchResultMatcher.create( MatchResult.Status.ERROR, new IOException(forbiddenException)), MatchResultMatcher.create(100, 1540000000000L, pathExist, true), MatchResultMatcher.create( 200, 1541000000000L, S3ResourceId.fromComponents(pathGlob.getBucket(), foundListObject.getKey()), true))); }
Example 19
Source File: AwsSdkTest.java From s3proxy with Apache License 2.0 | 4 votes |
@Test public void testMultipartCopy() throws Exception { // B2 requires two parts to issue an MPU assumeTrue(!blobStoreType.equals("b2")); String sourceBlobName = "testMultipartCopy-source"; String targetBlobName = "testMultipartCopy-target"; ObjectMetadata metadata = new ObjectMetadata(); metadata.setContentLength(BYTE_SOURCE.size()); client.putObject(containerName, sourceBlobName, BYTE_SOURCE.openStream(), metadata); InitiateMultipartUploadRequest initiateRequest = new InitiateMultipartUploadRequest(containerName, targetBlobName); InitiateMultipartUploadResult initResult = client.initiateMultipartUpload(initiateRequest); String uploadId = initResult.getUploadId(); CopyPartRequest copyRequest = new CopyPartRequest() .withDestinationBucketName(containerName) .withDestinationKey(targetBlobName) .withSourceBucketName(containerName) .withSourceKey(sourceBlobName) .withUploadId(uploadId) .withFirstByte(0L) .withLastByte(BYTE_SOURCE.size() - 1) .withPartNumber(1); CopyPartResult copyPartResult = client.copyPart(copyRequest); CompleteMultipartUploadRequest completeRequest = new CompleteMultipartUploadRequest( containerName, targetBlobName, uploadId, ImmutableList.of(copyPartResult.getPartETag())); client.completeMultipartUpload(completeRequest); S3Object object = client.getObject(containerName, targetBlobName); assertThat(object.getObjectMetadata().getContentLength()).isEqualTo( BYTE_SOURCE.size()); try (InputStream actual = object.getObjectContent(); InputStream expected = BYTE_SOURCE.openStream()) { assertThat(actual).hasContentEqualTo(expected); } }
Example 20
Source File: AwsSdkTest.java From s3proxy with Apache License 2.0 | 4 votes |
@Test public void testCopyObjectReplaceMetadata() throws Exception { String fromName = "from-name"; String toName = "to-name"; ObjectMetadata metadata = new ObjectMetadata(); metadata.setContentLength(BYTE_SOURCE.size()); if (!Quirks.NO_CACHE_CONTROL_SUPPORT.contains(blobStoreType)) { metadata.setCacheControl("max-age=3600"); } if (!Quirks.NO_CONTENT_DISPOSITION.contains(blobStoreType)) { metadata.setContentDisposition("attachment; filename=old.jpg"); } if (!Quirks.NO_CONTENT_ENCODING.contains(blobStoreType)) { metadata.setContentEncoding("compress"); } if (!Quirks.NO_CONTENT_LANGUAGE.contains(blobStoreType)) { metadata.setContentLanguage("en"); } metadata.setContentType("audio/ogg"); // TODO: expires metadata.setUserMetadata(ImmutableMap.of( "key1", "value1", "key2", "value2")); client.putObject(containerName, fromName, BYTE_SOURCE.openStream(), metadata); String cacheControl = "max-age=1800"; String contentDisposition = "attachment; filename=new.jpg"; String contentEncoding = "gzip"; String contentLanguage = "fr"; String contentType = "audio/mp4"; ObjectMetadata contentMetadata = new ObjectMetadata(); if (!Quirks.NO_CACHE_CONTROL_SUPPORT.contains(blobStoreType)) { contentMetadata.setCacheControl(cacheControl); } if (!Quirks.NO_CONTENT_DISPOSITION.contains(blobStoreType)) { contentMetadata.setContentDisposition(contentDisposition); } if (!Quirks.NO_CONTENT_ENCODING.contains(blobStoreType)) { contentMetadata.setContentEncoding(contentEncoding); } if (!Quirks.NO_CONTENT_LANGUAGE.contains(blobStoreType)) { contentMetadata.setContentLanguage(contentLanguage); } contentMetadata.setContentType(contentType); // TODO: expires Map<String, String> userMetadata = ImmutableMap.of( "key3", "value3", "key4", "value4"); contentMetadata.setUserMetadata(userMetadata); client.copyObject(new CopyObjectRequest( containerName, fromName, containerName, toName) .withNewObjectMetadata(contentMetadata)); S3Object object = client.getObject(containerName, toName); try (InputStream actual = object.getObjectContent(); InputStream expected = BYTE_SOURCE.openStream()) { assertThat(actual).hasContentEqualTo(expected); } ObjectMetadata toContentMetadata = object.getObjectMetadata(); if (!Quirks.NO_CACHE_CONTROL_SUPPORT.contains(blobStoreType)) { assertThat(contentMetadata.getCacheControl()).isEqualTo( cacheControl); } if (!Quirks.NO_CONTENT_DISPOSITION.contains(blobStoreType)) { assertThat(toContentMetadata.getContentDisposition()).isEqualTo( contentDisposition); } if (!Quirks.NO_CONTENT_ENCODING.contains(blobStoreType)) { assertThat(toContentMetadata.getContentEncoding()).isEqualTo( contentEncoding); } if (!Quirks.NO_CONTENT_LANGUAGE.contains(blobStoreType)) { assertThat(toContentMetadata.getContentLanguage()).isEqualTo( contentLanguage); } assertThat(toContentMetadata.getContentType()).isEqualTo( contentType); // TODO: expires assertThat(toContentMetadata.getUserMetadata()).isEqualTo( userMetadata); }