Java Code Examples for com.amazonaws.services.s3.model.S3ObjectSummary#setKey()

@Test
public void testTagObjects()
{
    // Create an S3 file transfer request parameters DTO to access S3 objects.
    S3FileTransferRequestParamsDto s3FileTransferRequestParamsDto = new S3FileTransferRequestParamsDto();

    // Create an S3 file transfer request parameters DTO to tag S3 objects.
    S3FileTransferRequestParamsDto s3ObjectTaggerParamsDto = new S3FileTransferRequestParamsDto();
    s3ObjectTaggerParamsDto.setAwsAccessKeyId(AWS_ASSUMED_ROLE_ACCESS_KEY);
    s3ObjectTaggerParamsDto.setAwsSecretKey(AWS_ASSUMED_ROLE_SECRET_KEY);
    s3ObjectTaggerParamsDto.setSessionToken(AWS_ASSUMED_ROLE_SESSION_TOKEN);

    // Create an S3 object summary.
    S3ObjectSummary s3ObjectSummary = new S3ObjectSummary();
    s3ObjectSummary.setKey(S3_KEY);

    // Create an S3 object tag.
    Tag tag = new Tag(S3_OBJECT_TAG_KEY, S3_OBJECT_TAG_VALUE);

    // Call the method under test.
    s3Service.tagObjects(s3FileTransferRequestParamsDto, s3ObjectTaggerParamsDto, Collections.singletonList(s3ObjectSummary), tag);

    // Verify the external calls.
    verify(s3Dao).tagObjects(s3FileTransferRequestParamsDto, s3ObjectTaggerParamsDto, Collections.singletonList(s3ObjectSummary), tag);
    verifyNoMoreInteractions(s3Dao);
}
 

private List<S3ObjectSummary> createSummaries() {
    List<S3ObjectSummary> summaries = new ArrayList<>();

    for (int i = 0; i < 1000; i++) {
        S3ObjectSummary summary = new S3ObjectSummary();
        summary.setKey("" + i);
        summaries.add(summary);
    }

    return summaries;
}
 

@Test
@PrepareForTest(TerrapinUtil.class)
public void testGetS3FileList() throws Exception {
  AmazonS3Client s3Client = mock(AmazonS3Client.class);
  ObjectListing objectListing = mock(ObjectListing.class);
  S3ObjectSummary summary1 = new S3ObjectSummary();
  S3ObjectSummary summary2 = new S3ObjectSummary();
  S3ObjectSummary summary3 = new S3ObjectSummary();
  summary1.setKey("/abc/123");
  summary2.setKey("/abc/456");
  summary3.setKey("/def/123");
  summary1.setSize(32432);
  summary2.setSize(213423);
  summary3.setSize(2334);
  List<S3ObjectSummary> summaries = ImmutableList.of(summary1, summary2, summary3);
  whenNew(AmazonS3Client.class).withAnyArguments().thenReturn(s3Client);
  when(s3Client.listObjects(any(ListObjectsRequest.class))).thenReturn(objectListing);
  when(objectListing.getObjectSummaries()).thenReturn(summaries);

  List<Pair<Path, Long>> results = TerrapinUtil.getS3FileList(new AnonymousAWSCredentials(),
      "bucket", "/abc");

  assertEquals(2, results.size());
  assertTrue(results.get(0).getLeft().toString().endsWith(summary1.getKey()));
  assertEquals(new Long(summary1.getSize()), results.get(0).getRight());
  assertTrue(results.get(1).getLeft().toString().endsWith(summary2.getKey()));
  assertEquals(new Long(summary2.getSize()), results.get(1).getRight());
}
 

private List<S3ObjectSummary> objectSummaries() {
    listObjectSummaries = new LinkedList<>();

    S3ObjectSummary firstObj = new S3ObjectSummary();
    S3ObjectSummary secondObj = new S3ObjectSummary();
    firstObj.setBucketName(BUCKET_NAME);
    secondObj.setBucketName(BUCKET_NAME);
    firstObj.setKey("prefix/test1.json.gz");
    secondObj.setKey("prefix/test2.json.gz");
    listObjectSummaries.add(firstObj);
    listObjectSummaries.add(secondObj);
    return listObjectSummaries;
}
 

/**
 * {@inheritDoc}
 * <p/>
 * If the bucket does not exist, returns a listing with an empty list. If a prefix is specified in listObjectsRequest, only keys starting with the prefix
 * will be returned.
 */
@Override
public ObjectListing listObjects(ListObjectsRequest listObjectsRequest, AmazonS3 s3Client)
{
    LOGGER.debug("listObjects(): listObjectsRequest.getBucketName() = " + listObjectsRequest.getBucketName());

    String bucketName = listObjectsRequest.getBucketName();

    if (MOCK_S3_BUCKET_NAME_NO_SUCH_BUCKET_EXCEPTION.equals(bucketName))
    {
        AmazonS3Exception amazonS3Exception = new AmazonS3Exception(MOCK_S3_BUCKET_NAME_NO_SUCH_BUCKET_EXCEPTION);
        amazonS3Exception.setErrorCode("NoSuchBucket");
        throw amazonS3Exception;
    }

    ObjectListing objectListing = new ObjectListing();
    objectListing.setBucketName(bucketName);

    MockS3Bucket mockS3Bucket = mockS3Buckets.get(bucketName);
    if (mockS3Bucket != null)
    {
        for (MockS3Object mockS3Object : mockS3Bucket.getObjects().values())
        {
            String s3ObjectKey = mockS3Object.getKey();
            if (listObjectsRequest.getPrefix() == null || s3ObjectKey.startsWith(listObjectsRequest.getPrefix()))
            {
                S3ObjectSummary s3ObjectSummary = new S3ObjectSummary();
                s3ObjectSummary.setBucketName(bucketName);
                s3ObjectSummary.setKey(s3ObjectKey);
                s3ObjectSummary.setSize(mockS3Object.getData().length);
                s3ObjectSummary.setStorageClass(mockS3Object.getObjectMetadata() != null ? mockS3Object.getObjectMetadata().getStorageClass() : null);

                objectListing.getObjectSummaries().add(s3ObjectSummary);
            }
        }
    }

    return objectListing;
}
 

@Test
public void testTagObjectsOtherTagKeyAlreadyExists()
{
    // Create two S3 object tags having different tag keys.
    List<Tag> tags = Arrays.asList(new Tag(S3_OBJECT_TAG_KEY, S3_OBJECT_TAG_VALUE), new Tag(S3_OBJECT_TAG_KEY_2, S3_OBJECT_TAG_VALUE_2));

    // Put a file in S3 that is already tagged with the first S3 object tag.
    PutObjectRequest putObjectRequest = new PutObjectRequest(S3_BUCKET_NAME, TARGET_S3_KEY, new ByteArrayInputStream(new byte[1]), new ObjectMetadata());
    putObjectRequest.setTagging(new ObjectTagging(Collections.singletonList(tags.get(0))));
    s3Operations.putObject(putObjectRequest, null);

    // Validate that the S3 object is tagged with the first tag only.
    GetObjectTaggingResult getObjectTaggingResult = s3Operations.getObjectTagging(new GetObjectTaggingRequest(S3_BUCKET_NAME, TARGET_S3_KEY), null);
    assertEquals(Collections.singletonList(tags.get(0)), getObjectTaggingResult.getTagSet());

    // Create an S3 file transfer request parameters DTO to access S3 objects.
    S3FileTransferRequestParamsDto params = new S3FileTransferRequestParamsDto();
    params.setS3BucketName(S3_BUCKET_NAME);

    // Create an S3 object summary.
    S3ObjectSummary s3ObjectSummary = new S3ObjectSummary();
    s3ObjectSummary.setKey(TARGET_S3_KEY);

    // Tag the S3 file with the second S3 object tag.
    s3Dao.tagObjects(params, new S3FileTransferRequestParamsDto(), Collections.singletonList(s3ObjectSummary), tags.get(1));

    // Validate that the S3 object is now tagged with both tags.
    getObjectTaggingResult = s3Operations.getObjectTagging(new GetObjectTaggingRequest(S3_BUCKET_NAME, TARGET_S3_KEY), null);
    assertEquals(tags.size(), getObjectTaggingResult.getTagSet().size());
    assertTrue(getObjectTaggingResult.getTagSet().containsAll(tags));
}
 

@Test
public void testListObjectsNothingNew() throws IOException {
    runner.setProperty(ListS3.REGION, "eu-west-1");
    runner.setProperty(ListS3.BUCKET, "test-bucket");

    Calendar calendar = Calendar.getInstance();
    calendar.set(2017, 5, 2);
    Date objectLastModified = calendar.getTime();
    long stateCurrentTimestamp = objectLastModified.getTime();

    Map<String, String> state = new HashMap<>();
    state.put(ListS3.CURRENT_TIMESTAMP, String.valueOf(stateCurrentTimestamp));
    state.put(ListS3.CURRENT_KEY_PREFIX+"0", "test-key");
    MockStateManager mockStateManager = runner.getStateManager();
    mockStateManager.setState(state, Scope.CLUSTER);

    ObjectListing objectListing = new ObjectListing();
    S3ObjectSummary objectSummary1 = new S3ObjectSummary();
    objectSummary1.setBucketName("test-bucket");
    objectSummary1.setKey("test-key");
    objectSummary1.setLastModified(objectLastModified);
    objectListing.getObjectSummaries().add(objectSummary1);
    Mockito.when(mockS3Client.listObjects(Mockito.any(ListObjectsRequest.class))).thenReturn(objectListing);

    runner.run();

    ArgumentCaptor<ListObjectsRequest> captureRequest = ArgumentCaptor.forClass(ListObjectsRequest.class);
    Mockito.verify(mockS3Client, Mockito.times(1)).listObjects(captureRequest.capture());
    ListObjectsRequest request = captureRequest.getValue();
    assertEquals("test-bucket", request.getBucketName());
    Mockito.verify(mockS3Client, Mockito.never()).listVersions(Mockito.any());

    runner.assertAllFlowFilesTransferred(ListS3.REL_SUCCESS, 0);
}
 

@Test
public void createFileWrapperFromFile() throws Exception {
	FileWrapper file;
	S3ObjectSummary s3ObjectSummary = new S3ObjectSummary();
	s3ObjectSummary.setBucketName("my-bucket");
	s3ObjectSummary.setKey("path/to/my/file.ext");
	s3ObjectSummary.setLastModified(new Date(9000));
	s3ObjectSummary.setSize(12);

	file = S3FindFilesStep.Execution.createFileWrapperFromFile(0, Paths.get(s3ObjectSummary.getKey()), s3ObjectSummary);
	Assert.assertEquals("file.ext", file.getName());
	Assert.assertEquals("path/to/my/file.ext", file.getPath());
	Assert.assertFalse(file.isDirectory());
	Assert.assertEquals(12, file.getLength());
	Assert.assertEquals(9000, file.getLastModified());

	file = S3FindFilesStep.Execution.createFileWrapperFromFile(1, Paths.get(s3ObjectSummary.getKey()), s3ObjectSummary);
	Assert.assertEquals("file.ext", file.getName());
	Assert.assertEquals("to/my/file.ext", file.getPath());
	Assert.assertFalse(file.isDirectory());
	Assert.assertEquals(12, file.getLength());
	Assert.assertEquals(9000, file.getLastModified());

	file = S3FindFilesStep.Execution.createFileWrapperFromFile(2, Paths.get(s3ObjectSummary.getKey()), s3ObjectSummary);
	Assert.assertEquals("file.ext", file.getName());
	Assert.assertEquals("my/file.ext", file.getPath());
	Assert.assertFalse(file.isDirectory());
	Assert.assertEquals(12, file.getLength());
	Assert.assertEquals(9000, file.getLastModified());
}
 

@Test
public void testWriteUserMetadata() {
    runner.setProperty(ListS3.REGION, "eu-west-1");
    runner.setProperty(ListS3.BUCKET, "test-bucket");
    runner.setProperty(ListS3.WRITE_USER_METADATA, "true");

    Date lastModified = new Date();
    ObjectListing objectListing = new ObjectListing();
    S3ObjectSummary objectSummary1 = new S3ObjectSummary();
    objectSummary1.setBucketName("test-bucket");
    objectSummary1.setKey("a");
    objectSummary1.setLastModified(lastModified);
    objectListing.getObjectSummaries().add(objectSummary1);

    Mockito.when(mockS3Client.listObjects(Mockito.any(ListObjectsRequest.class))).thenReturn(objectListing);

    runner.run();

    ArgumentCaptor<GetObjectMetadataRequest> captureRequest = ArgumentCaptor.forClass(GetObjectMetadataRequest.class);
    Mockito.verify(mockS3Client, Mockito.times(1)).getObjectMetadata(captureRequest.capture());
    GetObjectMetadataRequest request = captureRequest.getValue();

    assertEquals("test-bucket", request.getBucketName());
    assertEquals("a", request.getKey());

    Mockito.verify(mockS3Client, Mockito.never()).listVersions(Mockito.any());
}
 

@Test
public void testListVersion2WithRequesterPays() {
    runner.setProperty(ListS3.REGION, "eu-west-1");
    runner.setProperty(ListS3.BUCKET, "test-bucket");
    runner.setProperty(ListS3.REQUESTER_PAYS, "true");
    runner.setProperty(ListS3.LIST_TYPE, "2");

    Date lastModified = new Date();
    ListObjectsV2Result objectListing = new ListObjectsV2Result();
    S3ObjectSummary objectSummary1 = new S3ObjectSummary();
    objectSummary1.setBucketName("test-bucket");
    objectSummary1.setKey("a");
    objectSummary1.setLastModified(lastModified);
    objectListing.getObjectSummaries().add(objectSummary1);
    S3ObjectSummary objectSummary2 = new S3ObjectSummary();
    objectSummary2.setBucketName("test-bucket");
    objectSummary2.setKey("b/c");
    objectSummary2.setLastModified(lastModified);
    objectListing.getObjectSummaries().add(objectSummary2);
    S3ObjectSummary objectSummary3 = new S3ObjectSummary();
    objectSummary3.setBucketName("test-bucket");
    objectSummary3.setKey("d/e");
    objectSummary3.setLastModified(lastModified);
    objectListing.getObjectSummaries().add(objectSummary3);
    Mockito.when(mockS3Client.listObjectsV2(Mockito.any(ListObjectsV2Request.class))).thenReturn(objectListing);

    runner.run();

    ArgumentCaptor<ListObjectsV2Request> captureRequest = ArgumentCaptor.forClass(ListObjectsV2Request.class);
    Mockito.verify(mockS3Client, Mockito.times(1)).listObjectsV2(captureRequest.capture());
    ListObjectsV2Request request = captureRequest.getValue();
    assertEquals("test-bucket", request.getBucketName());
    assertTrue(request.isRequesterPays());
    Mockito.verify(mockS3Client, Mockito.never()).listVersions(Mockito.any());

    runner.assertAllFlowFilesTransferred(ListS3.REL_SUCCESS, 3);
    List<MockFlowFile> flowFiles = runner.getFlowFilesForRelationship(ListS3.REL_SUCCESS);
    MockFlowFile ff0 = flowFiles.get(0);
    ff0.assertAttributeEquals("filename", "a");
    ff0.assertAttributeEquals("s3.bucket", "test-bucket");
    String lastModifiedTimestamp = String.valueOf(lastModified.getTime());
    ff0.assertAttributeEquals("s3.lastModified", lastModifiedTimestamp);
    flowFiles.get(1).assertAttributeEquals("filename", "b/c");
    flowFiles.get(2).assertAttributeEquals("filename", "d/e");
    runner.getStateManager().assertStateEquals(ListS3.CURRENT_TIMESTAMP, lastModifiedTimestamp, Scope.CLUSTER);
}
 

@Test
public void matchVariousInvokeThreadPool() throws IOException {
  S3FileSystem s3FileSystem = buildMockedS3FileSystem(s3Options());

  AmazonS3Exception notFoundException = new AmazonS3Exception("mock exception");
  notFoundException.setStatusCode(404);
  S3ResourceId pathNotExist =
      S3ResourceId.fromUri("s3://testbucket/testdirectory/nonexistentfile");
  when(s3FileSystem
          .getAmazonS3Client()
          .getObjectMetadata(
              argThat(
                  new GetObjectMetadataRequestMatcher(
                      new GetObjectMetadataRequest(
                          pathNotExist.getBucket(), pathNotExist.getKey())))))
      .thenThrow(notFoundException);

  AmazonS3Exception forbiddenException = new AmazonS3Exception("mock exception");
  forbiddenException.setStatusCode(403);
  S3ResourceId pathForbidden =
      S3ResourceId.fromUri("s3://testbucket/testdirectory/forbiddenfile");
  when(s3FileSystem
          .getAmazonS3Client()
          .getObjectMetadata(
              argThat(
                  new GetObjectMetadataRequestMatcher(
                      new GetObjectMetadataRequest(
                          pathForbidden.getBucket(), pathForbidden.getKey())))))
      .thenThrow(forbiddenException);

  S3ResourceId pathExist = S3ResourceId.fromUri("s3://testbucket/testdirectory/filethatexists");
  ObjectMetadata s3ObjectMetadata = new ObjectMetadata();
  s3ObjectMetadata.setContentLength(100);
  s3ObjectMetadata.setLastModified(new Date(1540000000000L));
  s3ObjectMetadata.setContentEncoding("not-gzip");
  when(s3FileSystem
          .getAmazonS3Client()
          .getObjectMetadata(
              argThat(
                  new GetObjectMetadataRequestMatcher(
                      new GetObjectMetadataRequest(pathExist.getBucket(), pathExist.getKey())))))
      .thenReturn(s3ObjectMetadata);

  S3ResourceId pathGlob = S3ResourceId.fromUri("s3://testbucket/path/part*");

  S3ObjectSummary foundListObject = new S3ObjectSummary();
  foundListObject.setBucketName(pathGlob.getBucket());
  foundListObject.setKey("path/part-0");
  foundListObject.setSize(200);
  foundListObject.setLastModified(new Date(1541000000000L));

  ListObjectsV2Result listObjectsResult = new ListObjectsV2Result();
  listObjectsResult.setNextContinuationToken(null);
  listObjectsResult.getObjectSummaries().add(foundListObject);
  when(s3FileSystem.getAmazonS3Client().listObjectsV2(notNull(ListObjectsV2Request.class)))
      .thenReturn(listObjectsResult);

  ObjectMetadata metadata = new ObjectMetadata();
  metadata.setContentEncoding("");
  when(s3FileSystem
          .getAmazonS3Client()
          .getObjectMetadata(
              argThat(
                  new GetObjectMetadataRequestMatcher(
                      new GetObjectMetadataRequest(pathGlob.getBucket(), "path/part-0")))))
      .thenReturn(metadata);

  assertThat(
      s3FileSystem.match(
          ImmutableList.of(
              pathNotExist.toString(),
              pathForbidden.toString(),
              pathExist.toString(),
              pathGlob.toString())),
      contains(
          MatchResultMatcher.create(MatchResult.Status.NOT_FOUND, new FileNotFoundException()),
          MatchResultMatcher.create(
              MatchResult.Status.ERROR, new IOException(forbiddenException)),
          MatchResultMatcher.create(100, 1540000000000L, pathExist, true),
          MatchResultMatcher.create(
              200,
              1541000000000L,
              S3ResourceId.fromComponents(pathGlob.getBucket(), foundListObject.getKey()),
              true)));
}
 

@Test
public void testValidateAccessNoStorageFiles() throws Exception
{
    // Create business object data without any registered storage files, but with storage unit directory path and an actual non zero-byte S3 file.
    BusinessObjectData businessObjectData = createBusinessObjectData();
    businessObjectData.getStorageUnits().get(0).setStorageFiles(null);
    StorageDirectory storageDirectory = new StorageDirectory();
    storageDirectory.setDirectoryPath(STORAGE_DIRECTORY_PATH);
    businessObjectData.getStorageUnits().get(0).setStorageDirectory(storageDirectory);

    // Create AWS list objects response with a non zero-byte S3 file.
    S3ObjectSummary s3ObjectSummary = new S3ObjectSummary();
    s3ObjectSummary.setKey(S3_KEY);
    s3ObjectSummary.setSize(FILE_SIZE_1_KB);
    objectListing = Mockito.mock(ObjectListing.class);
    when(objectListing.getObjectSummaries()).thenReturn(Lists.newArrayList(s3ObjectSummary));

    // Create AWS get object request.
    GetObjectRequest getObjectRequest = new GetObjectRequest(S3_BUCKET_NAME, S3_KEY).withRange(0, MAX_BYTE_DOWNLOAD);

    // Create S3 object.
    S3Object s3Object = new S3Object();
    s3Object.setObjectContent(new ByteArrayInputStream(RandomStringUtils.randomAlphabetic(MAX_BYTE_DOWNLOAD).getBytes()));

    // Create S3 object metadata.
    ObjectMetadata objectMetadata = new ObjectMetadata();
    objectMetadata.setContentLength(MAX_BYTE_DOWNLOAD);

    // Mock the external calls.
    when(propertiesHelper.getProperty(HERD_BASE_URL_PROPERTY)).thenReturn(HERD_BASE_URL);
    when(propertiesHelper.getProperty(HERD_USERNAME_PROPERTY)).thenReturn(HERD_USERNAME);
    when(propertiesHelper.getProperty(HERD_PASSWORD_PROPERTY)).thenReturn(HERD_PASSWORD);
    when(propertiesHelper.getProperty(AWS_REGION_PROPERTY)).thenReturn(AWS_REGION_NAME_US_EAST_1);
    when(propertiesHelper.getProperty(AWS_ROLE_ARN_PROPERTY)).thenReturn(AWS_ROLE_ARN);
    when(propertiesHelper.getProperty(BUSINESS_OBJECT_FORMAT_VERSION_PROPERTY)).thenReturn(BUSINESS_OBJECT_FORMAT_VERSION.toString());
    when(propertiesHelper.getProperty(BUSINESS_OBJECT_DATA_VERSION_PROPERTY)).thenReturn(BUSINESS_OBJECT_DATA_VERSION.toString());
    when(propertiesHelper.getProperty(NAMESPACE_PROPERTY)).thenReturn(NAMESPACE);
    when(propertiesHelper.getProperty(BUSINESS_OBJECT_DEFINITION_NAME_PROPERTY)).thenReturn(BUSINESS_OBJECT_DEFINITION_NAME);
    when(propertiesHelper.getProperty(BUSINESS_OBJECT_FORMAT_USAGE_PROPERTY)).thenReturn(BUSINESS_OBJECT_FORMAT_USAGE);
    when(propertiesHelper.getProperty(BUSINESS_OBJECT_FORMAT_FILE_TYPE_PROPERTY)).thenReturn(BUSINESS_OBJECT_FORMAT_FILE_TYPE);
    when(propertiesHelper.getProperty(PRIMARY_PARTITION_VALUE_PROPERTY)).thenReturn(PRIMARY_PARTITION_VALUE);
    when(propertiesHelper.getProperty(SUB_PARTITION_VALUES_PROPERTY)).thenReturn(SUB_PARTITION_VALUES);
    when(herdApiClientOperations
        .businessObjectDataGetBusinessObjectData(any(BusinessObjectDataApi.class), eq(NAMESPACE), eq(BUSINESS_OBJECT_DEFINITION_NAME),
            eq(BUSINESS_OBJECT_FORMAT_USAGE), eq(BUSINESS_OBJECT_FORMAT_FILE_TYPE), eq(null), eq(PRIMARY_PARTITION_VALUE), eq(SUB_PARTITION_VALUES),
            eq(BUSINESS_OBJECT_FORMAT_VERSION), eq(BUSINESS_OBJECT_DATA_VERSION), eq(null), eq(false), eq(false))).thenReturn(businessObjectData);
    when(s3Operations.listObjects(any(ListObjectsRequest.class), any(AmazonS3.class))).thenReturn(objectListing);
    when(s3Operations.getS3Object(eq(getObjectRequest), any(AmazonS3.class))).thenReturn(s3Object);

    // Call the method under test with message flag set to "false".
    accessValidatorController.validateAccess(new File(PROPERTIES_FILE_PATH), false);

    // Verify the external calls.
    verify(herdApiClientOperations).checkPropertiesFile(propertiesHelper, false);
    verify(propertiesHelper).loadProperties(new File(PROPERTIES_FILE_PATH));
    verify(propertiesHelper).getProperty(HERD_BASE_URL_PROPERTY);
    verify(propertiesHelper).getProperty(HERD_USERNAME_PROPERTY);
    verify(propertiesHelper).getProperty(HERD_PASSWORD_PROPERTY);
    verify(herdApiClientOperations).applicationGetBuildInfo(any(ApplicationApi.class));
    verify(herdApiClientOperations).currentUserGetCurrentUser(any(CurrentUserApi.class));
    verify(propertiesHelper).getProperty(AWS_REGION_PROPERTY);
    verify(propertiesHelper).getProperty(AWS_ROLE_ARN_PROPERTY);
    verify(propertiesHelper).getProperty(BUSINESS_OBJECT_FORMAT_VERSION_PROPERTY);
    verify(propertiesHelper).getProperty(BUSINESS_OBJECT_DATA_VERSION_PROPERTY);
    verify(propertiesHelper).getProperty(NAMESPACE_PROPERTY);
    verify(propertiesHelper).getProperty(BUSINESS_OBJECT_DEFINITION_NAME_PROPERTY);
    verify(propertiesHelper).getProperty(BUSINESS_OBJECT_FORMAT_USAGE_PROPERTY);
    verify(propertiesHelper).getProperty(BUSINESS_OBJECT_FORMAT_FILE_TYPE_PROPERTY);
    verify(propertiesHelper).getProperty(PRIMARY_PARTITION_VALUE_PROPERTY);
    verify(propertiesHelper).getProperty(SUB_PARTITION_VALUES_PROPERTY);
    verify(herdApiClientOperations)
        .businessObjectDataGetBusinessObjectData(any(BusinessObjectDataApi.class), eq(NAMESPACE), eq(BUSINESS_OBJECT_DEFINITION_NAME),
            eq(BUSINESS_OBJECT_FORMAT_USAGE), eq(BUSINESS_OBJECT_FORMAT_FILE_TYPE), eq(null), eq(PRIMARY_PARTITION_VALUE), eq(SUB_PARTITION_VALUES),
            eq(BUSINESS_OBJECT_FORMAT_VERSION), eq(BUSINESS_OBJECT_DATA_VERSION), eq(null), eq(false), eq(false));
    verify(s3Operations).listObjects(any(ListObjectsRequest.class), any(AmazonS3.class));
    verify(s3Operations).getS3Object(eq(getObjectRequest), any(AmazonS3.class));
    verifyNoMoreInteractionsHelper();
}
 

@Test
public void testValidateAccessNoStorageFilesZeroByteS3File() throws Exception
{
    // Create business object data without any registered storage files, but with storage unit directory path and a zero-byte S3 file.
    BusinessObjectData businessObjectData = createBusinessObjectData();
    businessObjectData.getStorageUnits().get(0).setStorageFiles(null);
    StorageDirectory storageDirectory = new StorageDirectory();
    storageDirectory.setDirectoryPath(STORAGE_DIRECTORY_PATH);
    businessObjectData.getStorageUnits().get(0).setStorageDirectory(storageDirectory);

    // Create AWS list objects response with a zero-byte S3 file.
    S3ObjectSummary s3ObjectSummary = new S3ObjectSummary();
    s3ObjectSummary.setKey(S3_KEY);
    s3ObjectSummary.setSize(FILE_SIZE_0_BYTE);
    objectListing = Mockito.mock(ObjectListing.class);
    when(objectListing.getObjectSummaries()).thenReturn(Lists.newArrayList(s3ObjectSummary));

    // Create AWS get object request.
    GetObjectRequest getObjectRequest = new GetObjectRequest(S3_BUCKET_NAME, S3_KEY).withRange(0, MAX_BYTE_DOWNLOAD);

    // Create S3 object.
    S3Object s3Object = new S3Object();
    s3Object.setObjectContent(new ByteArrayInputStream(RandomStringUtils.randomAlphabetic(MAX_BYTE_DOWNLOAD).getBytes()));

    // Create S3 object metadata.
    ObjectMetadata objectMetadata = new ObjectMetadata();
    objectMetadata.setContentLength(MAX_BYTE_DOWNLOAD);

    // Mock the external calls.
    when(propertiesHelper.getProperty(HERD_BASE_URL_PROPERTY)).thenReturn(HERD_BASE_URL);
    when(propertiesHelper.getProperty(HERD_USERNAME_PROPERTY)).thenReturn(HERD_USERNAME);
    when(propertiesHelper.getProperty(HERD_PASSWORD_PROPERTY)).thenReturn(HERD_PASSWORD);
    when(propertiesHelper.getProperty(AWS_REGION_PROPERTY)).thenReturn(AWS_REGION_NAME_US_EAST_1);
    when(propertiesHelper.getProperty(AWS_ROLE_ARN_PROPERTY)).thenReturn(AWS_ROLE_ARN);
    when(propertiesHelper.getProperty(BUSINESS_OBJECT_FORMAT_VERSION_PROPERTY)).thenReturn(BUSINESS_OBJECT_FORMAT_VERSION.toString());
    when(propertiesHelper.getProperty(BUSINESS_OBJECT_DATA_VERSION_PROPERTY)).thenReturn(BUSINESS_OBJECT_DATA_VERSION.toString());
    when(propertiesHelper.getProperty(NAMESPACE_PROPERTY)).thenReturn(NAMESPACE);
    when(propertiesHelper.getProperty(BUSINESS_OBJECT_DEFINITION_NAME_PROPERTY)).thenReturn(BUSINESS_OBJECT_DEFINITION_NAME);
    when(propertiesHelper.getProperty(BUSINESS_OBJECT_FORMAT_USAGE_PROPERTY)).thenReturn(BUSINESS_OBJECT_FORMAT_USAGE);
    when(propertiesHelper.getProperty(BUSINESS_OBJECT_FORMAT_FILE_TYPE_PROPERTY)).thenReturn(BUSINESS_OBJECT_FORMAT_FILE_TYPE);
    when(propertiesHelper.getProperty(PRIMARY_PARTITION_VALUE_PROPERTY)).thenReturn(PRIMARY_PARTITION_VALUE);
    when(propertiesHelper.getProperty(SUB_PARTITION_VALUES_PROPERTY)).thenReturn(SUB_PARTITION_VALUES);
    when(herdApiClientOperations
        .businessObjectDataGetBusinessObjectData(any(BusinessObjectDataApi.class), eq(NAMESPACE), eq(BUSINESS_OBJECT_DEFINITION_NAME),
            eq(BUSINESS_OBJECT_FORMAT_USAGE), eq(BUSINESS_OBJECT_FORMAT_FILE_TYPE), eq(null), eq(PRIMARY_PARTITION_VALUE), eq(SUB_PARTITION_VALUES),
            eq(BUSINESS_OBJECT_FORMAT_VERSION), eq(BUSINESS_OBJECT_DATA_VERSION), eq(null), eq(false), eq(false))).thenReturn(businessObjectData);
    when(s3Operations.listObjects(any(ListObjectsRequest.class), any(AmazonS3.class))).thenReturn(objectListing);

    // Call the method under test with message flag set to "false".
    accessValidatorController.validateAccess(new File(PROPERTIES_FILE_PATH), false);

    // Verify the external calls.
    verify(herdApiClientOperations).checkPropertiesFile(propertiesHelper, false);
    verify(propertiesHelper).loadProperties(new File(PROPERTIES_FILE_PATH));
    verify(propertiesHelper).getProperty(HERD_BASE_URL_PROPERTY);
    verify(propertiesHelper).getProperty(HERD_USERNAME_PROPERTY);
    verify(propertiesHelper).getProperty(HERD_PASSWORD_PROPERTY);
    verify(herdApiClientOperations).applicationGetBuildInfo(any(ApplicationApi.class));
    verify(herdApiClientOperations).currentUserGetCurrentUser(any(CurrentUserApi.class));
    verify(propertiesHelper).getProperty(AWS_REGION_PROPERTY);
    verify(propertiesHelper).getProperty(AWS_ROLE_ARN_PROPERTY);
    verify(propertiesHelper).getProperty(BUSINESS_OBJECT_FORMAT_VERSION_PROPERTY);
    verify(propertiesHelper).getProperty(BUSINESS_OBJECT_DATA_VERSION_PROPERTY);
    verify(propertiesHelper).getProperty(NAMESPACE_PROPERTY);
    verify(propertiesHelper).getProperty(BUSINESS_OBJECT_DEFINITION_NAME_PROPERTY);
    verify(propertiesHelper).getProperty(BUSINESS_OBJECT_FORMAT_USAGE_PROPERTY);
    verify(propertiesHelper).getProperty(BUSINESS_OBJECT_FORMAT_FILE_TYPE_PROPERTY);
    verify(propertiesHelper).getProperty(PRIMARY_PARTITION_VALUE_PROPERTY);
    verify(propertiesHelper).getProperty(SUB_PARTITION_VALUES_PROPERTY);
    verify(herdApiClientOperations)
        .businessObjectDataGetBusinessObjectData(any(BusinessObjectDataApi.class), eq(NAMESPACE), eq(BUSINESS_OBJECT_DEFINITION_NAME),
            eq(BUSINESS_OBJECT_FORMAT_USAGE), eq(BUSINESS_OBJECT_FORMAT_FILE_TYPE), eq(null), eq(PRIMARY_PARTITION_VALUE), eq(SUB_PARTITION_VALUES),
            eq(BUSINESS_OBJECT_FORMAT_VERSION), eq(BUSINESS_OBJECT_DATA_VERSION), eq(null), eq(false), eq(false));
    verify(s3Operations).listObjects(any(ListObjectsRequest.class), any(AmazonS3.class));
    verifyNoMoreInteractionsHelper();
}
 

@Test
public void matchGlob() throws IOException {
  S3FileSystem s3FileSystem = buildMockedS3FileSystem(s3Options());

  S3ResourceId path = S3ResourceId.fromUri("s3://testbucket/foo/bar*baz");

  ListObjectsV2Request firstRequest =
      new ListObjectsV2Request()
          .withBucketName(path.getBucket())
          .withPrefix(path.getKeyNonWildcardPrefix())
          .withContinuationToken(null);

  // Expected to be returned; prefix and wildcard/regex match
  S3ObjectSummary firstMatch = new S3ObjectSummary();
  firstMatch.setBucketName(path.getBucket());
  firstMatch.setKey("foo/bar0baz");
  firstMatch.setSize(100);
  firstMatch.setLastModified(new Date(1540000000001L));

  // Expected to not be returned; prefix matches, but substring after wildcard does not
  S3ObjectSummary secondMatch = new S3ObjectSummary();
  secondMatch.setBucketName(path.getBucket());
  secondMatch.setKey("foo/bar1qux");
  secondMatch.setSize(200);
  secondMatch.setLastModified(new Date(1540000000002L));

  // Expected first request returns continuation token
  ListObjectsV2Result firstResult = new ListObjectsV2Result();
  firstResult.setNextContinuationToken("token");
  firstResult.getObjectSummaries().add(firstMatch);
  firstResult.getObjectSummaries().add(secondMatch);
  when(s3FileSystem
          .getAmazonS3Client()
          .listObjectsV2(argThat(new ListObjectsV2RequestArgumentMatches(firstRequest))))
      .thenReturn(firstResult);

  // Expect second request with continuation token
  ListObjectsV2Request secondRequest =
      new ListObjectsV2Request()
          .withBucketName(path.getBucket())
          .withPrefix(path.getKeyNonWildcardPrefix())
          .withContinuationToken("token");

  // Expected to be returned; prefix and wildcard/regex match
  S3ObjectSummary thirdMatch = new S3ObjectSummary();
  thirdMatch.setBucketName(path.getBucket());
  thirdMatch.setKey("foo/bar2baz");
  thirdMatch.setSize(300);
  thirdMatch.setLastModified(new Date(1540000000003L));

  // Expected second request returns third prefix match and no continuation token
  ListObjectsV2Result secondResult = new ListObjectsV2Result();
  secondResult.setNextContinuationToken(null);
  secondResult.getObjectSummaries().add(thirdMatch);
  when(s3FileSystem
          .getAmazonS3Client()
          .listObjectsV2(argThat(new ListObjectsV2RequestArgumentMatches(secondRequest))))
      .thenReturn(secondResult);

  // Expect object metadata queries for content encoding
  ObjectMetadata metadata = new ObjectMetadata();
  metadata.setContentEncoding("");
  when(s3FileSystem.getAmazonS3Client().getObjectMetadata(anyObject())).thenReturn(metadata);

  assertThat(
      s3FileSystem.matchGlobPaths(ImmutableList.of(path)).get(0),
      MatchResultMatcher.create(
          ImmutableList.of(
              MatchResult.Metadata.builder()
                  .setIsReadSeekEfficient(true)
                  .setResourceId(
                      S3ResourceId.fromComponents(
                          firstMatch.getBucketName(), firstMatch.getKey()))
                  .setSizeBytes(firstMatch.getSize())
                  .setLastModifiedMillis(firstMatch.getLastModified().getTime())
                  .build(),
              MatchResult.Metadata.builder()
                  .setIsReadSeekEfficient(true)
                  .setResourceId(
                      S3ResourceId.fromComponents(
                          thirdMatch.getBucketName(), thirdMatch.getKey()))
                  .setSizeBytes(thirdMatch.getSize())
                  .setLastModifiedMillis(thirdMatch.getLastModified().getTime())
                  .build())));
}
 

@Test
public void testListIgnoreByMinAge() throws IOException {
    runner.setProperty(ListS3.REGION, "eu-west-1");
    runner.setProperty(ListS3.BUCKET, "test-bucket");
    runner.setProperty(ListS3.MIN_AGE, "30 sec");

    Date lastModifiedNow = new Date();
    Date lastModifiedMinus1Hour = DateUtils.addHours(lastModifiedNow, -1);
    Date lastModifiedMinus3Hour = DateUtils.addHours(lastModifiedNow, -3);
    ObjectListing objectListing = new ObjectListing();
    S3ObjectSummary objectSummary1 = new S3ObjectSummary();
    objectSummary1.setBucketName("test-bucket");
    objectSummary1.setKey("minus-3hour");
    objectSummary1.setLastModified(lastModifiedMinus3Hour);
    objectListing.getObjectSummaries().add(objectSummary1);
    S3ObjectSummary objectSummary2 = new S3ObjectSummary();
    objectSummary2.setBucketName("test-bucket");
    objectSummary2.setKey("minus-1hour");
    objectSummary2.setLastModified(lastModifiedMinus1Hour);
    objectListing.getObjectSummaries().add(objectSummary2);
    S3ObjectSummary objectSummary3 = new S3ObjectSummary();
    objectSummary3.setBucketName("test-bucket");
    objectSummary3.setKey("now");
    objectSummary3.setLastModified(lastModifiedNow);
    objectListing.getObjectSummaries().add(objectSummary3);
    Mockito.when(mockS3Client.listObjects(Mockito.any(ListObjectsRequest.class))).thenReturn(objectListing);

    Map<String,String> stateMap = new HashMap<>();
    String previousTimestamp = String.valueOf(lastModifiedMinus3Hour.getTime());
    stateMap.put(ListS3.CURRENT_TIMESTAMP, previousTimestamp);
    stateMap.put(ListS3.CURRENT_KEY_PREFIX + "0", "minus-3hour");
    runner.getStateManager().setState(stateMap, Scope.CLUSTER);

    runner.run();

    ArgumentCaptor<ListObjectsRequest> captureRequest = ArgumentCaptor.forClass(ListObjectsRequest.class);
    Mockito.verify(mockS3Client, Mockito.times(1)).listObjects(captureRequest.capture());
    ListObjectsRequest request = captureRequest.getValue();
    assertEquals("test-bucket", request.getBucketName());
    Mockito.verify(mockS3Client, Mockito.never()).listVersions(Mockito.any());

    runner.assertAllFlowFilesTransferred(ListS3.REL_SUCCESS, 1);
    List<MockFlowFile> flowFiles = runner.getFlowFilesForRelationship(ListS3.REL_SUCCESS);
    MockFlowFile ff0 = flowFiles.get(0);
    ff0.assertAttributeEquals("filename", "minus-1hour");
    ff0.assertAttributeEquals("s3.bucket", "test-bucket");
    String lastModifiedTimestamp = String.valueOf(lastModifiedMinus1Hour.getTime());
    ff0.assertAttributeEquals("s3.lastModified", lastModifiedTimestamp);
    runner.getStateManager().assertStateEquals(ListS3.CURRENT_TIMESTAMP, lastModifiedTimestamp, Scope.CLUSTER);
}
 

@Test
public void testListWithRecords() throws InitializationException {
    runner.setProperty(ListS3.REGION, "eu-west-1");
    runner.setProperty(ListS3.BUCKET, "test-bucket");

    final MockRecordWriter recordWriter = new MockRecordWriter(null, false);
    runner.addControllerService("record-writer", recordWriter);
    runner.enableControllerService(recordWriter);
    runner.setProperty(ListS3.RECORD_WRITER, "record-writer");

    Date lastModified = new Date();
    ObjectListing objectListing = new ObjectListing();
    S3ObjectSummary objectSummary1 = new S3ObjectSummary();
    objectSummary1.setBucketName("test-bucket");
    objectSummary1.setKey("a");
    objectSummary1.setLastModified(lastModified);
    objectListing.getObjectSummaries().add(objectSummary1);
    S3ObjectSummary objectSummary2 = new S3ObjectSummary();
    objectSummary2.setBucketName("test-bucket");
    objectSummary2.setKey("b/c");
    objectSummary2.setLastModified(lastModified);
    objectListing.getObjectSummaries().add(objectSummary2);
    S3ObjectSummary objectSummary3 = new S3ObjectSummary();
    objectSummary3.setBucketName("test-bucket");
    objectSummary3.setKey("d/e");
    objectSummary3.setLastModified(lastModified);
    objectListing.getObjectSummaries().add(objectSummary3);
    Mockito.when(mockS3Client.listObjects(Mockito.any(ListObjectsRequest.class))).thenReturn(objectListing);

    runner.run();

    ArgumentCaptor<ListObjectsRequest> captureRequest = ArgumentCaptor.forClass(ListObjectsRequest.class);
    Mockito.verify(mockS3Client, Mockito.times(1)).listObjects(captureRequest.capture());
    ListObjectsRequest request = captureRequest.getValue();
    assertEquals("test-bucket", request.getBucketName());
    assertFalse(request.isRequesterPays());
    Mockito.verify(mockS3Client, Mockito.never()).listVersions(Mockito.any());

    runner.assertAllFlowFilesTransferred(ListS3.REL_SUCCESS, 1);

    final DateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
    dateFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
    final String lastModifiedString = dateFormat.format(lastModified);

    final MockFlowFile flowFile = runner.getFlowFilesForRelationship(ListS3.REL_SUCCESS).get(0);
    flowFile.assertAttributeEquals("record.count", "3");
    flowFile.assertContentEquals("a,test-bucket,,," + lastModifiedString + ",0,,true,,,\n"
        + "b/c,test-bucket,,," + lastModifiedString + ",0,,true,,,\n"
        + "d/e,test-bucket,,," + lastModifiedString + ",0,,true,,,\n");
}
 

private static S3ObjectSummary buildObjectSummary( String bucketName, String key, String dataString ) {
  S3ObjectSummary s3ObjectSummary = new S3ObjectSummary();
  s3ObjectSummary.setKey( key );
  s3ObjectSummary.setBucketName( bucketName );
  return s3ObjectSummary;
}
 

private void runTagObjectsTest()
{
    // Create an S3 file transfer request parameters DTO to access S3 objects.
    S3FileTransferRequestParamsDto s3FileTransferRequestParamsDto = new S3FileTransferRequestParamsDto();
    s3FileTransferRequestParamsDto.setS3BucketName(S3_BUCKET_NAME);

    // Create an S3 file transfer request parameters DTO to tag S3 objects.
    S3FileTransferRequestParamsDto s3ObjectTaggerParamsDto = new S3FileTransferRequestParamsDto();
    s3ObjectTaggerParamsDto.setAwsAccessKeyId(AWS_ASSUMED_ROLE_ACCESS_KEY);
    s3ObjectTaggerParamsDto.setAwsSecretKey(AWS_ASSUMED_ROLE_SECRET_KEY);
    s3ObjectTaggerParamsDto.setSessionToken(AWS_ASSUMED_ROLE_SESSION_TOKEN);

    // Create an S3 object summary.
    S3ObjectSummary s3ObjectSummary = new S3ObjectSummary();
    s3ObjectSummary.setKey(S3_KEY);

    // Create an S3 object tag.
    Tag tag = new Tag(S3_OBJECT_TAG_KEY, S3_OBJECT_TAG_VALUE);

    // Create a retry policy.
    RetryPolicy retryPolicy =
        new RetryPolicy(PredefinedRetryPolicies.DEFAULT_RETRY_CONDITION, PredefinedRetryPolicies.DEFAULT_BACKOFF_STRATEGY, INTEGER_VALUE, true);

    // Create a get object tagging result.
    GetObjectTaggingResult getObjectTaggingResult = new GetObjectTaggingResult(null);

    // Create a set object tagging result.
    SetObjectTaggingResult setObjectTaggingResult = new SetObjectTaggingResult();

    // Mock the external calls.
    when(retryPolicyFactory.getRetryPolicy()).thenReturn(retryPolicy);
    when(s3Operations.getObjectTagging(any(GetObjectTaggingRequest.class), any(AmazonS3Client.class))).thenReturn(getObjectTaggingResult);
    when(s3Operations.setObjectTagging(any(SetObjectTaggingRequest.class), any(AmazonS3Client.class))).thenReturn(setObjectTaggingResult);

    // Call the method under test.
    s3DaoImpl.tagObjects(s3FileTransferRequestParamsDto, s3ObjectTaggerParamsDto, Collections.singletonList(s3ObjectSummary), tag);

    // Verify the external calls.
    verify(retryPolicyFactory, times(2)).getRetryPolicy();
    verify(s3Operations).getObjectTagging(any(GetObjectTaggingRequest.class), any(AmazonS3Client.class));
    verify(s3Operations).setObjectTagging(any(SetObjectTaggingRequest.class), any(AmazonS3Client.class));
    verifyNoMoreInteractionsHelper();
}
 

@Test
public void testListVersion2() {
    runner.setProperty(ListS3.REGION, "eu-west-1");
    runner.setProperty(ListS3.BUCKET, "test-bucket");
    runner.setProperty(ListS3.LIST_TYPE, "2");

    Date lastModified = new Date();
    ListObjectsV2Result objectListing = new ListObjectsV2Result();
    S3ObjectSummary objectSummary1 = new S3ObjectSummary();
    objectSummary1.setBucketName("test-bucket");
    objectSummary1.setKey("a");
    objectSummary1.setLastModified(lastModified);
    objectListing.getObjectSummaries().add(objectSummary1);
    S3ObjectSummary objectSummary2 = new S3ObjectSummary();
    objectSummary2.setBucketName("test-bucket");
    objectSummary2.setKey("b/c");
    objectSummary2.setLastModified(lastModified);
    objectListing.getObjectSummaries().add(objectSummary2);
    S3ObjectSummary objectSummary3 = new S3ObjectSummary();
    objectSummary3.setBucketName("test-bucket");
    objectSummary3.setKey("d/e");
    objectSummary3.setLastModified(lastModified);
    objectListing.getObjectSummaries().add(objectSummary3);
    Mockito.when(mockS3Client.listObjectsV2(Mockito.any(ListObjectsV2Request.class))).thenReturn(objectListing);

    runner.run();

    ArgumentCaptor<ListObjectsV2Request> captureRequest = ArgumentCaptor.forClass(ListObjectsV2Request.class);
    Mockito.verify(mockS3Client, Mockito.times(1)).listObjectsV2(captureRequest.capture());
    ListObjectsV2Request request = captureRequest.getValue();
    assertEquals("test-bucket", request.getBucketName());
    assertFalse(request.isRequesterPays());
    Mockito.verify(mockS3Client, Mockito.never()).listVersions(Mockito.any());

    runner.assertAllFlowFilesTransferred(ListS3.REL_SUCCESS, 3);
    List<MockFlowFile> flowFiles = runner.getFlowFilesForRelationship(ListS3.REL_SUCCESS);
    MockFlowFile ff0 = flowFiles.get(0);
    ff0.assertAttributeEquals("filename", "a");
    ff0.assertAttributeEquals("s3.bucket", "test-bucket");
    String lastModifiedTimestamp = String.valueOf(lastModified.getTime());
    ff0.assertAttributeEquals("s3.lastModified", lastModifiedTimestamp);
    flowFiles.get(1).assertAttributeEquals("filename", "b/c");
    flowFiles.get(2).assertAttributeEquals("filename", "d/e");
    runner.getStateManager().assertStateEquals(ListS3.CURRENT_TIMESTAMP, lastModifiedTimestamp, Scope.CLUSTER);
}
 

/**
 * Creates an S3 object summary.
 *
 * @param filePath the file path
 * @param fileSizeInBytes the file size in bytes
 *
 * @return the S3 object summary
 */
private S3ObjectSummary createS3ObjectSummary(String filePath, long fileSizeInBytes)
{
    S3ObjectSummary s3ObjectSummary = new S3ObjectSummary();
    s3ObjectSummary.setKey(filePath);
    s3ObjectSummary.setSize(fileSizeInBytes);
    return s3ObjectSummary;
}