com.amazonaws.services.s3.model.S3Object#setObjectMetadata

Source File: AwsS3EnvironmentRepositoryTests.java From spring-cloud-config with Apache License 2.0

6 votes

private void setupS3(String fileName, String version, String propertyContent)
		throws UnsupportedEncodingException {
	final S3ObjectId s3ObjectId = new S3ObjectId("bucket1", fileName);
	final GetObjectRequest request = new GetObjectRequest(s3ObjectId);

	final S3Object s3Object = new S3Object();
	s3Object.setObjectContent(new StringInputStream(propertyContent));

	if (version != null) {
		final ObjectMetadata metadata = new ObjectMetadata();
		metadata.setHeader("x-amz-version-id", version);
		s3Object.setObjectMetadata(metadata);
	}

	when(s3Client.getObject(argThat(new GetObjectRequestMatcher(request))))
			.thenReturn(s3Object);
}

Source File: SimpleStorageResourceTest.java From spring-cloud-aws with Apache License 2.0

6 votes

@Test
void getInputStream_existingObject_returnsInputStreamWithContent() throws Exception {
	// Arrange
	AmazonS3 amazonS3 = mock(AmazonS3.class);
	ObjectMetadata objectMetadata = mock(ObjectMetadata.class);
	when(amazonS3.getObjectMetadata(any(GetObjectMetadataRequest.class)))
			.thenReturn(objectMetadata);

	S3Object s3Object = new S3Object();
	s3Object.setObjectMetadata(objectMetadata);
	s3Object.setObjectContent(new ByteArrayInputStream(new byte[] { 42 }));
	when(amazonS3.getObject(any(GetObjectRequest.class))).thenReturn(s3Object);

	// Act
	SimpleStorageResource simpleStorageResource = new SimpleStorageResource(amazonS3,
			"bucket", "object", new SyncTaskExecutor());

	// Assert
	assertThat(simpleStorageResource.exists()).isTrue();
	assertThat(simpleStorageResource.getInputStream().read()).isEqualTo(42);
}

Source File: MockAmazonS3.java From crate with Apache License 2.0

6 votes

@Override
public S3Object getObject(final GetObjectRequest request) throws AmazonClientException {
    assertThat(request.getBucketName(), equalTo(bucket));

    final String blobName = request.getKey();
    final byte[] content = blobs.get(blobName);
    if (content == null) {
        AmazonS3Exception exception = new AmazonS3Exception("[" + blobName + "] does not exist.");
        exception.setStatusCode(404);
        throw exception;
    }

    ObjectMetadata metadata = new ObjectMetadata();
    metadata.setContentLength(content.length);

    S3Object s3Object = new S3Object();
    s3Object.setObjectContent(new S3ObjectInputStream(new ByteArrayInputStream(content), null, false));
    s3Object.setKey(blobName);
    s3Object.setObjectMetadata(metadata);

    return s3Object;
}

Source File: MockS3OperationsImpl.java From herd with Apache License 2.0

5 votes

@Override
public S3Object getS3Object(GetObjectRequest getObjectRequest, AmazonS3 s3)
{
    MockS3Object mockS3Object = getMockS3Object(getObjectRequest.getBucketName(), getObjectRequest.getKey());

    S3Object s3Object = new S3Object();
    s3Object.setBucketName(getObjectRequest.getBucketName());
    s3Object.setKey(getObjectRequest.getKey());
    s3Object.setObjectContent(new ByteArrayInputStream(mockS3Object.getData()));
    s3Object.setObjectMetadata(mockS3Object.getObjectMetadata());

    return s3Object;
}

Source File: TestFetchS3Object.java From nifi with Apache License 2.0

5 votes

@Test
public void testGetObjectVersion() throws IOException {
    runner.setProperty(FetchS3Object.REGION, "us-east-1");
    runner.setProperty(FetchS3Object.BUCKET, "request-bucket");
    runner.setProperty(FetchS3Object.VERSION_ID, "${s3.version}");
    final Map<String, String> attrs = new HashMap<>();
    attrs.put("filename", "request-key");
    attrs.put("s3.version", "request-version");
    runner.enqueue(new byte[0], attrs);

    S3Object s3ObjectResponse = new S3Object();
    s3ObjectResponse.setBucketName("response-bucket-name");
    s3ObjectResponse.setObjectContent(new StringInputStream("Some Content"));
    ObjectMetadata metadata = Mockito.spy(ObjectMetadata.class);
    metadata.setContentDisposition("key/path/to/file.txt");
    Mockito.when(metadata.getVersionId()).thenReturn("response-version");
    s3ObjectResponse.setObjectMetadata(metadata);
    Mockito.when(mockS3Client.getObject(Mockito.any())).thenReturn(s3ObjectResponse);

    runner.run(1);

    ArgumentCaptor<GetObjectRequest> captureRequest = ArgumentCaptor.forClass(GetObjectRequest.class);
    Mockito.verify(mockS3Client, Mockito.times(1)).getObject(captureRequest.capture());
    GetObjectRequest request = captureRequest.getValue();
    assertEquals("request-bucket", request.getBucketName());
    assertEquals("request-key", request.getKey());
    assertEquals("request-version", request.getVersionId());

    runner.assertAllFlowFilesTransferred(FetchS3Object.REL_SUCCESS, 1);
    final List<MockFlowFile> ffs = runner.getFlowFilesForRelationship(FetchS3Object.REL_SUCCESS);
    MockFlowFile ff = ffs.get(0);
    ff.assertAttributeEquals("s3.bucket", "response-bucket-name");
    ff.assertAttributeEquals(CoreAttributes.FILENAME.key(), "file.txt");
    ff.assertAttributeEquals(CoreAttributes.PATH.key(), "key/path/to");
    ff.assertAttributeEquals(CoreAttributes.ABSOLUTE_PATH.key(), "key/path/to/file.txt");
    ff.assertAttributeEquals("s3.version", "response-version");
    ff.assertContentEquals("Some Content");
}

Source File: AmazonS3FileSystem.java From iaf with Apache License 2.0

5 votes

@Override
public Iterator<S3Object> listFiles(String folder) throws FileSystemException {
	List<S3ObjectSummary> summaries = null;
	String prefix = folder != null ? folder + "/" : "";
	try {
		ObjectListing listing = s3Client.listObjects(bucketName, prefix);
		summaries = listing.getObjectSummaries();
		while (listing.isTruncated()) {
			listing = s3Client.listNextBatchOfObjects(listing);
			summaries.addAll(listing.getObjectSummaries());
		}
	} catch (AmazonServiceException e) {
		throw new FileSystemException("Cannot process requested action", e);
	}

	List<S3Object> list = new ArrayList<S3Object>();
	for (S3ObjectSummary summary : summaries) {
		S3Object object = new S3Object();
		ObjectMetadata metadata = new ObjectMetadata();
		metadata.setContentLength(summary.getSize());

		object.setBucketName(summary.getBucketName());
		object.setKey(summary.getKey());
		object.setObjectMetadata(metadata);
		if(!object.getKey().endsWith("/") && !(prefix.isEmpty() && object.getKey().contains("/"))) {
			list.add(object);
		} 
	}

	return list.iterator();
}

Source File: StashReaderTest.java From emodb with Apache License 2.0

4 votes

@Test
public void getSplit() throws Exception {
    // Intentionally create the gzip file as concatenated gzip files to verify the Java bug involving reading
    // concatenated gzip files is resolved.

    List<Map<String, Object>> expected = Lists.newArrayListWithCapacity(4);
    ByteArrayOutputStream splitOut = new ByteArrayOutputStream();

    for (int p=0; p < 2; p++) {
        for (int i=0; i < 2; i++) {
            ByteArrayOutputStream partialOut = new ByteArrayOutputStream();
            try (BufferedWriter out = new BufferedWriter(new OutputStreamWriter(new GZIPOutputStream(partialOut)))) {
                Map<String, Object> value = ImmutableMap.<String, Object>builder()
                        .put("~id", "row" + (p * 2 + i))
                        .put("~table", "test:table")
                        .put("~version", 1)
                        .put("~signature", "3a0da59fabf298d389b7b0b59728e887")
                        .put("~lastUpdateAt", "2014-08-28T21:24:36.440Z")
                        .put("~firstUpdateAt", "2014-06-07T09:51:40.077Z")
                        .build();

                String json = JsonHelper.asJson(value);
                out.write(json);
                out.write("\n");

                expected.add(value);
            }
            partialOut.writeTo(splitOut);
        }
    }

    splitOut.close();

    S3Object s3Object = new S3Object();
    s3Object.setObjectContent(new ByteArrayInputStream(splitOut.toByteArray(), 0, splitOut.size()));
    ObjectMetadata objectMetadata = new ObjectMetadata();
    objectMetadata.setContentLength(splitOut.size());
    s3Object.setObjectMetadata(objectMetadata);

    AmazonS3 s3 = mock(AmazonS3.class);
    when(s3.getObject(argThat(getsObject("stash-bucket", "stash/test/2015-01-01-00-00-00/test-table/split0.gz"))))
            .thenReturn(s3Object);

    StashSplit stashSplit = new StashSplit("test:table", "2015-01-01-00-00-00/test-table/split0.gz", splitOut.size());

    StandardStashReader reader = new StandardStashReader(URI.create("s3://stash-bucket/stash/test"), s3, 0);
    StashRowIterator contentIter = reader.getSplit(stashSplit);

    List<Map<String, Object>> content = ImmutableList.copyOf(contentIter);
    assertEquals(content, expected);
}

Source File: StashReaderTest.java From emodb with Apache License 2.0

4 votes

/**
 * Note:  This test depends on the default character set to be US-ASCII
 */
@Test
public void testUTF8ClientDecoding() throws Exception {
    if (!Charset.defaultCharset().equals(Charsets.US_ASCII)) {
        throw new SkipException("testUTF8ClientDecoding() requires default charset to be US-ASCII");
    }

    String utf8Text = "ชิงรางวัลกันจ้า";
    ByteArrayOutputStream splitOut = new ByteArrayOutputStream();

    try (BufferedWriter out = new BufferedWriter(new OutputStreamWriter(new GZIPOutputStream(splitOut), Charsets.UTF_8))) {
        Map<String, Object> value = ImmutableMap.<String, Object>builder()
                .put("~id", "1")
                .put("~table", "test:table")
                .put("~version", 1)
                .put("~signature", "3a0da59fabf298d389b7b0b59728e887")
                .put("~lastUpdateAt", "2014-08-28T21:24:36.440Z")
                .put("~firstUpdateAt", "2014-06-07T09:51:40.077Z")
                .put("text", utf8Text)
                .build();

        String json = JsonHelper.asJson(value);
        out.write(json);
        out.write("\n");
    }

    splitOut.close();

    S3Object s3Object = new S3Object();
    s3Object.setObjectContent(new ByteArrayInputStream(splitOut.toByteArray(), 0, splitOut.size()));
    ObjectMetadata objectMetadata = new ObjectMetadata();
    objectMetadata.setContentLength(splitOut.size());
    s3Object.setObjectMetadata(objectMetadata);

    AmazonS3 s3 = mock(AmazonS3.class);
    when(s3.getObject(argThat(getsObject("stash-bucket", "stash/test/2015-01-01-00-00-00/test-table/split0.gz"))))
            .thenReturn(s3Object);

    StashSplit stashSplit = new StashSplit("test:table", "2015-01-01-00-00-00/test-table/split0.gz", splitOut.size());

    StandardStashReader reader = new StandardStashReader(URI.create("s3://stash-bucket/stash/test"), s3, 0);
    try (StashRowIterator rowIter = reader.getSplit(stashSplit)) {
        assertTrue(rowIter.hasNext());
        Map<String, Object> row = rowIter.next();
        assertEquals(row.get("text"), utf8Text);
    }
}

Source File: TestFetchS3Object.java From nifi with Apache License 2.0

4 votes

@Test
public void testGetObject() throws IOException {
    runner.setProperty(FetchS3Object.REGION, "us-east-1");
    runner.setProperty(FetchS3Object.BUCKET, "request-bucket");
    final Map<String, String> attrs = new HashMap<>();
    attrs.put("filename", "request-key");
    runner.enqueue(new byte[0], attrs);

    S3Object s3ObjectResponse = new S3Object();
    s3ObjectResponse.setBucketName("response-bucket-name");
    s3ObjectResponse.setKey("response-key");
    s3ObjectResponse.setObjectContent(new StringInputStream("Some Content"));
    ObjectMetadata metadata = Mockito.spy(ObjectMetadata.class);
    metadata.setContentDisposition("key/path/to/file.txt");
    metadata.setContentType("text/plain");
    metadata.setContentMD5("testMD5hash");
    Date expiration = new Date();
    metadata.setExpirationTime(expiration);
    metadata.setExpirationTimeRuleId("testExpirationRuleId");
    Map<String, String> userMetadata = new HashMap<>();
    userMetadata.put("userKey1", "userValue1");
    userMetadata.put("userKey2", "userValue2");
    metadata.setUserMetadata(userMetadata);
    metadata.setSSEAlgorithm("testAlgorithm");
    Mockito.when(metadata.getETag()).thenReturn("test-etag");
    s3ObjectResponse.setObjectMetadata(metadata);
    Mockito.when(mockS3Client.getObject(Mockito.any())).thenReturn(s3ObjectResponse);

    runner.run(1);

    ArgumentCaptor<GetObjectRequest> captureRequest = ArgumentCaptor.forClass(GetObjectRequest.class);
    Mockito.verify(mockS3Client, Mockito.times(1)).getObject(captureRequest.capture());
    GetObjectRequest request = captureRequest.getValue();
    assertEquals("request-bucket", request.getBucketName());
    assertEquals("request-key", request.getKey());
    assertFalse(request.isRequesterPays());
    assertNull(request.getVersionId());

    runner.assertAllFlowFilesTransferred(FetchS3Object.REL_SUCCESS, 1);
    final List<MockFlowFile> ffs = runner.getFlowFilesForRelationship(FetchS3Object.REL_SUCCESS);
    MockFlowFile ff = ffs.get(0);
    ff.assertAttributeEquals("s3.bucket", "response-bucket-name");
    ff.assertAttributeEquals(CoreAttributes.FILENAME.key(), "file.txt");
    ff.assertAttributeEquals(CoreAttributes.PATH.key(), "key/path/to");
    ff.assertAttributeEquals(CoreAttributes.ABSOLUTE_PATH.key(), "key/path/to/file.txt");
    ff.assertAttributeEquals(CoreAttributes.MIME_TYPE.key(), "text/plain");
    ff.assertAttributeEquals("hash.value", "testMD5hash");
    ff.assertAttributeEquals("hash.algorithm", "MD5");
    ff.assertAttributeEquals("s3.etag", "test-etag");
    ff.assertAttributeEquals("s3.expirationTime", String.valueOf(expiration.getTime()));
    ff.assertAttributeEquals("s3.expirationTimeRuleId", "testExpirationRuleId");
    ff.assertAttributeEquals("userKey1", "userValue1");
    ff.assertAttributeEquals("userKey2", "userValue2");
    ff.assertAttributeEquals("s3.sseAlgorithm", "testAlgorithm");
    ff.assertContentEquals("Some Content");
}

Source File: TestFetchS3Object.java From nifi with Apache License 2.0

4 votes

@Test
public void testGetObjectWithRequesterPays() throws IOException {
    runner.setProperty(FetchS3Object.REGION, "us-east-1");
    runner.setProperty(FetchS3Object.BUCKET, "request-bucket");
    runner.setProperty(FetchS3Object.REQUESTER_PAYS, "true");
    final Map<String, String> attrs = new HashMap<>();
    attrs.put("filename", "request-key");
    runner.enqueue(new byte[0], attrs);

    S3Object s3ObjectResponse = new S3Object();
    s3ObjectResponse.setBucketName("response-bucket-name");
    s3ObjectResponse.setKey("response-key");
    s3ObjectResponse.setObjectContent(new StringInputStream("Some Content"));
    ObjectMetadata metadata = Mockito.spy(ObjectMetadata.class);
    metadata.setContentDisposition("key/path/to/file.txt");
    metadata.setContentType("text/plain");
    metadata.setContentMD5("testMD5hash");
    Date expiration = new Date();
    metadata.setExpirationTime(expiration);
    metadata.setExpirationTimeRuleId("testExpirationRuleId");
    Map<String, String> userMetadata = new HashMap<>();
    userMetadata.put("userKey1", "userValue1");
    userMetadata.put("userKey2", "userValue2");
    metadata.setUserMetadata(userMetadata);
    metadata.setSSEAlgorithm("testAlgorithm");
    Mockito.when(metadata.getETag()).thenReturn("test-etag");
    s3ObjectResponse.setObjectMetadata(metadata);
    Mockito.when(mockS3Client.getObject(Mockito.any())).thenReturn(s3ObjectResponse);

    runner.run(1);

    ArgumentCaptor<GetObjectRequest> captureRequest = ArgumentCaptor.forClass(GetObjectRequest.class);
    Mockito.verify(mockS3Client, Mockito.times(1)).getObject(captureRequest.capture());
    GetObjectRequest request = captureRequest.getValue();
    assertEquals("request-bucket", request.getBucketName());
    assertEquals("request-key", request.getKey());
    assertTrue(request.isRequesterPays());
    assertNull(request.getVersionId());

    runner.assertAllFlowFilesTransferred(FetchS3Object.REL_SUCCESS, 1);
    final List<MockFlowFile> ffs = runner.getFlowFilesForRelationship(FetchS3Object.REL_SUCCESS);
    MockFlowFile ff = ffs.get(0);
    ff.assertAttributeEquals("s3.bucket", "response-bucket-name");
    ff.assertAttributeEquals(CoreAttributes.FILENAME.key(), "file.txt");
    ff.assertAttributeEquals(CoreAttributes.PATH.key(), "key/path/to");
    ff.assertAttributeEquals(CoreAttributes.ABSOLUTE_PATH.key(), "key/path/to/file.txt");
    ff.assertAttributeEquals(CoreAttributes.MIME_TYPE.key(), "text/plain");
    ff.assertAttributeEquals("hash.value", "testMD5hash");
    ff.assertAttributeEquals("hash.algorithm", "MD5");
    ff.assertAttributeEquals("s3.etag", "test-etag");
    ff.assertAttributeEquals("s3.expirationTime", String.valueOf(expiration.getTime()));
    ff.assertAttributeEquals("s3.expirationTimeRuleId", "testExpirationRuleId");
    ff.assertAttributeEquals("userKey1", "userValue1");
    ff.assertAttributeEquals("userKey2", "userValue2");
    ff.assertAttributeEquals("s3.sseAlgorithm", "testAlgorithm");
    ff.assertContentEquals("Some Content");
}

Java Code Examples for com.amazonaws.services.s3.model.S3Object#setObjectMetadata()