com.amazonaws.services.s3.model.ListObjectsRequest Java Examples

@Override
protected void doStart() {
    AWSCredentials myCredentials = new BasicAWSCredentials(accessKey, secretKey);
    AmazonS3 s3Client = new AmazonS3Client(myCredentials);
    ListObjectsRequest listObjectsRequest = new ListObjectsRequest().withBucketName(bucket);
    ObjectListing objectListing = s3Client.listObjects(listObjectsRequest);
    ChannelProcessor channelProcessor = getChannelProcessor();
    for (S3ObjectSummary s3ObjectSummary : objectListing.getObjectSummaries()) {
        String file = s3ObjectSummary.getKey();
        LOGGER.info("Read the content of {}", file);
        GetObjectRequest objectRequest = new GetObjectRequest(bucket, file);
        S3Object objectPortion = s3Client.getObject(objectRequest);
        try {
            long startTime = System.currentTimeMillis();
            processLines(channelProcessor, objectPortion.getObjectContent());
            LOGGER.info("Processing of {} took {} ms", file, System.currentTimeMillis() - startTime);
        } catch (IOException e) {
            LOGGER.warn("Cannot process the {}, skipping", file, e);
        }
    }
}
 

public List<String> listCommonPrefixesS3Objects(final String bucketName, final String prefix) {
    final List<String> commonPrefixes = Lists.newArrayList();

    try {
        log.debug("Listing objects in bucket '{}' with prefix '{}'", bucketName, prefix);

        final ListObjectsRequest listObjectsRequest = new ListObjectsRequest()
                .withDelimiter("/")
                .withBucketName(bucketName)
                .withPrefix(prefix);

        ObjectListing objectListing;

        do {
            objectListing = s3client.listObjects(listObjectsRequest);
            objectListing.getCommonPrefixes().stream().map(S3Service::urlDecode).forEach(commonPrefixes::add);
            listObjectsRequest.setMarker(objectListing.getNextMarker());
        } while (objectListing.isTruncated());

    } catch (final AmazonServiceException e) {
        log.error("Could not list common prefixes in S3", e);
    }

    return commonPrefixes;
}
 

private void handleAttachExceptionFallback( String bucket, String keyWithDelimiter, AmazonS3Exception exception )
  throws FileSystemException {
  ListObjectsRequest listObjectsRequest = new ListObjectsRequest()
    .withBucketName( bucket )
    .withPrefix( keyWithDelimiter )
    .withDelimiter( DELIMITER );
  ObjectListing ol = fileSystem.getS3Client().listObjects( listObjectsRequest );

  if ( !( ol.getCommonPrefixes().isEmpty() && ol.getObjectSummaries().isEmpty() ) ) {
    injectType( FileType.FOLDER );
  } else {
    //Folders don't really exist - they will generate a "NoSuchKey" exception
    // confirms key doesn't exist but connection okay
    String errorCode = exception.getErrorCode();
    if ( !errorCode.equals( "NoSuchKey" ) ) {
      // bubbling up other connection errors
      logger.error( "Could not get information on " + getQualifiedName(),
        exception ); // make sure this gets printed for the user
      throw new FileSystemException( "vfs.provider/get-type.error", getQualifiedName(), exception );
    }
  }
}
 

@Test
public void testHandleAttachExceptionEmptyFolder() throws FileSystemException {
  AmazonS3Exception exception = new AmazonS3Exception( "NoSuchKey" );
  exception.setErrorCode( "NoSuchKey" );

  //test the case where the folder exists and contains things; no exception should be thrown
  when( s3ServiceMock.getObject( BUCKET_NAME, origKey ) ).thenThrow( exception );
  when( s3ServiceMock.getObject( BUCKET_NAME, origKey + "/" ) ).thenThrow( exception );
  childObjectListing = mock( ObjectListing.class );
  when( childObjectListing.getObjectSummaries() ).thenReturn( new ArrayList<>() );
  when( childObjectListing.getCommonPrefixes() ).thenReturn( new ArrayList<>() );
  when( s3ServiceMock.listObjects( any( ListObjectsRequest.class ) ) ).thenReturn( childObjectListing );
  try {
    s3FileObjectFileSpy.doAttach();
  } catch ( Exception e ) {
    fail( "Caught exception " + e.getMessage() );
  }
  assertEquals( FileType.IMAGINARY, s3FileObjectFileSpy.getType() );
}
 

@Test
public void testHandleAttachExceptionFileNotFound() throws FileSystemException {
  AmazonS3Exception notFoundException = new AmazonS3Exception( "404 Not Found" );
  notFoundException.setErrorCode( "404 Not Found" );
  AmazonS3Exception noSuchKeyException = new AmazonS3Exception( "NoSuchKey" );
  noSuchKeyException.setErrorCode( "NoSuchKey" );

  //test the case where the file is not found; no exception should be thrown
  when( s3ServiceMock.getObject( BUCKET_NAME, origKey ) ).thenThrow( notFoundException );
  when( s3ServiceMock.getObject( BUCKET_NAME, origKey + "/" ) ).thenThrow( noSuchKeyException );
  childObjectListing = mock( ObjectListing.class );
  when( childObjectListing.getObjectSummaries() ).thenReturn( new ArrayList<>() );
  when( childObjectListing.getCommonPrefixes() ).thenReturn( new ArrayList<>() );
  when( s3ServiceMock.listObjects( any( ListObjectsRequest.class ) ) ).thenReturn( childObjectListing );
  try {
    s3FileObjectFileSpy.doAttach();
  } catch ( Exception e ) {
    fail( "Caught exception " + e.getMessage() );
  }
  assertEquals( FileType.IMAGINARY, s3FileObjectFileSpy.getType() );
}
 

@Override
public List<String> list() {
	List<String> files = new ArrayList<String>();
	if(s3URI == null) return files;
	String[] buckAndKey = this.getBucketAndKey();
	String bucket = buckAndKey[0];
	String key = buckAndKey[1];
	ObjectListing objectListing = s3. listObjects(new ListObjectsRequest().withBucketName(bucket).withPrefix(key));
	s3list: for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) {
		// check for valid extensions
		if(extensions == null){
			files.add("s3://"+bucket+"/"+objectSummary.getKey());
		} else for(String ext : extensions) if(objectSummary.getKey().endsWith(ext)){
			files.add("s3://"+bucket+"/"+objectSummary.getKey());
			continue s3list;
		}
	 }
	return files;
}
 

@Override
public ObjectListing listObjects(final ListObjectsRequest request) throws AmazonClientException {
    assertThat(request.getBucketName(), equalTo(bucket));

    final ObjectListing listing = new ObjectListing();
    listing.setBucketName(request.getBucketName());
    listing.setPrefix(request.getPrefix());

    for (Map.Entry<String, byte[]> blob : blobs.entrySet()) {
        if (Strings.isEmpty(request.getPrefix()) || blob.getKey().startsWith(request.getPrefix())) {
            S3ObjectSummary summary = new S3ObjectSummary();
            summary.setBucketName(request.getBucketName());
            summary.setKey(blob.getKey());
            summary.setSize(blob.getValue().length);
            listing.getObjectSummaries().add(summary);
        }
    }
    return listing;
}
 

@Test
public void copyMultipleObjects() throws Exception {
  // Making sure we only request 1 file at the time so we need to loop
  ListObjectsRequestFactory mockListObjectRequestFactory = Mockito.mock(ListObjectsRequestFactory.class);
  when(mockListObjectRequestFactory.newInstance()).thenReturn(new ListObjectsRequest().withMaxKeys(1));

  client.putObject("source", "bar/data1", inputData);
  client.putObject("source", "bar/data2", inputData);

  Path sourceBaseLocation = new Path("s3://source/bar/");
  Path replicaLocation = new Path("s3://target/foo/");
  List<Path> sourceSubLocations = new ArrayList<>();
  S3S3Copier s3s3Copier = new S3S3Copier(sourceBaseLocation, sourceSubLocations, replicaLocation, s3ClientFactory,
      transferManagerFactory, mockListObjectRequestFactory, registry, s3S3CopierOptions);
  Metrics metrics = s3s3Copier.copy();
  assertThat(metrics.getBytesReplicated(), is(14L));

  S3Object object1 = client.getObject("target", "foo/data1");
  String data1 = IOUtils.toString(object1.getObjectContent());
  assertThat(data1, is("bar foo"));
  S3Object object2 = client.getObject("target", "foo/data2");
  String data2 = IOUtils.toString(object2.getObjectContent());
  assertThat(data2, is("bar foo"));
}
 

private Matcher<ListObjectsRequest> listObjectRequest(final String bucket, final String prefix, @Nullable final String marker) {
    return new BaseMatcher<ListObjectsRequest>() {
        @Override
        public boolean matches(Object item) {
            ListObjectsRequest request = (ListObjectsRequest) item;
            return request != null &&
                    request.getBucketName().equals(bucket) &&
                    request.getPrefix().equals(prefix) &&
                    Objects.equals(request.getMarker(), marker);
        }

        @Override
        public void describeTo(Description description) {
            description.appendText("ListObjectRequest[s3://").appendText(bucket).appendText("/").appendText(prefix);
            if (marker != null) {
                description.appendText(", marker=").appendText(marker);
            }
            description.appendText("]");
        }
    };
}
 

@Override
protected List<String> getFileNames(String lockPrefix) throws Exception
{
    ListObjectsRequest  request = new ListObjectsRequest();
    request.setBucketName(bucket);
    request.setPrefix(lockPrefix);
    ObjectListing objectListing = client.listObjects(request);

    return Lists.transform
    (
        objectListing.getObjectSummaries(),
        new Function<S3ObjectSummary, String>()
        {
            @Override
            public String apply(S3ObjectSummary summary)
            {
                return summary.getKey();
            }
        }
    );
}
 

@Override
public void deleteStorageLocation(final BlobStoreConfiguration blobStoreConfiguration) {
  String bucket = getConfiguredBucket(blobStoreConfiguration);
  ObjectListing listing = s3.listObjects(new ListObjectsRequest().withBucketName(bucket).withMaxKeys(1));
  if (listing.getObjectSummaries().isEmpty()) {
    s3.deleteBucket(bucket);
  }
  else {
    log.info("Not removing S3 bucket {} because it is not empty", bucket);
    BucketLifecycleConfiguration lifecycleConfiguration = s3.getBucketLifecycleConfiguration(bucket);
    List<Rule> nonBlobstoreRules = nonBlobstoreRules(lifecycleConfiguration, blobStoreConfiguration.getName());
    if(!isEmpty(nonBlobstoreRules)) {
      lifecycleConfiguration.setRules(nonBlobstoreRules);
      s3.setBucketLifecycleConfiguration(bucket, lifecycleConfiguration);
    } else {
      s3.deleteBucketLifecycleConfiguration(bucket);
    }
  }
}
 

private void validateConnection(
    Stage.Context context,
    String configPrefix,
    List<Stage.ConfigIssue> issues
) {
  try {
    //check if the credentials are right by trying to list an object in the common prefix
    getS3Client().listObjects(new ListObjectsRequest(bucket, commonPrefix, null, delimiter, 1).withEncodingType("url"));
  } catch (AmazonS3Exception e) {
    LOG.debug(Errors.S3_SPOOLDIR_20.getMessage(), e.toString(), e);
    issues.add(
        context.createConfigIssue(
            Groups.S3.name(),
            configPrefix + S3ConnectionBaseConfig.AWS_CONFIG_PREFIX + "awsAccessKeyId",
            Errors.S3_SPOOLDIR_20,
            e.toString()
        )
    );
  }
}
 

protected void validateTdSparkQueryOutput()
{
    AmazonS3URI resultUri = new AmazonS3URI(tmpS3FolderUri.toString() + "/result/");
    ObjectListing resultListing = s3.listObjects(new ListObjectsRequest().withBucketName(resultUri.getBucket()).withPrefix(resultUri.getKey()));
    List<String> resultLines = resultListing.getObjectSummaries().stream().flatMap(o -> {
        try (S3Object object = s3.getObject(o.getBucketName(), o.getKey())) {
            return CharStreams.readLines(new InputStreamReader(object.getObjectContent())).stream();
        }
        catch (IOException e) {
            throw Throwables.propagate(e);
        }
    }).collect(toList());
    // FIXME
    // we need to specify the version of td-spark that we can use for acceptance tests.
    // In the meantime the assertion is commented out.
    //assertThat(resultLines, Matchers.hasItem(",164.54.104.106,/item/games/4663,/category/electronics,404,Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0),121,GET,1412383598"));
}
 

public static void s3DeleteRecursively(AmazonS3 s3, String bucket, String prefix)
        throws Exception
{
    ListObjectsRequest request = new ListObjectsRequest()
            .withBucketName(bucket)
            .withPrefix(prefix);

    while (true) {
        ObjectListing listing = s3.listObjects(request);
        String[] keys = listing.getObjectSummaries().stream().map(S3ObjectSummary::getKey).toArray(String[]::new);
        for (String key : keys) {
            logger.info("delete s3://{}/{}", bucket, key);
        }
        retryExecutor()
                .retryIf(e -> e instanceof AmazonServiceException)
                .run(() -> s3.deleteObjects(new DeleteObjectsRequest(bucket).withKeys(keys)));
        if (listing.getNextMarker() == null) {
            break;
        }
    }
}
 

@Override
public void remove(String noteId, AuthenticationInfo subject) throws IOException {
  String key = user + "/" + "notebook" + "/" + noteId;
  final ListObjectsRequest listObjectsRequest = new ListObjectsRequest()
      .withBucketName(bucketName).withPrefix(key);

  try {
    ObjectListing objects = s3client.listObjects(listObjectsRequest);
    do {
      for (S3ObjectSummary objectSummary : objects.getObjectSummaries()) {
        s3client.deleteObject(bucketName, objectSummary.getKey());
      }
      objects = s3client.listNextBatchOfObjects(objects);
    } while (objects.isTruncated());
  }
  catch (AmazonClientException ace) {
    throw new IOException("Unable to remove note in S3: " + ace, ace);
  }
}
 

private Answer<ObjectListing> objectListingAnswer(@Nullable final String marker, final String... fileNames) {
    return new Answer<ObjectListing>() {
        @Override
        public ObjectListing answer(InvocationOnMock invocation)
                throws Throwable {
            ListObjectsRequest request = (ListObjectsRequest) invocation.getArguments()[0];

            ObjectListing objectListing = new ObjectListing();
            objectListing.setBucketName(request.getBucketName());
            objectListing.setPrefix(request.getPrefix());

            objectListing.setTruncated(marker != null);
            objectListing.setNextMarker(marker);

            for (String fileName : fileNames) {
                S3ObjectSummary objectSummary = new S3ObjectSummary();
                objectSummary.setKey(request.getPrefix() + fileName);
                objectSummary.setSize(100);
                objectListing.getObjectSummaries().add(objectSummary);
            }

            return objectListing;
        }
    };
}
 

@Test
public void testWriteObjectTags() {
    runner.setProperty(ListS3.REGION, "eu-west-1");
    runner.setProperty(ListS3.BUCKET, "test-bucket");
    runner.setProperty(ListS3.WRITE_OBJECT_TAGS, "true");

    Date lastModified = new Date();
    ObjectListing objectListing = new ObjectListing();
    S3ObjectSummary objectSummary1 = new S3ObjectSummary();
    objectSummary1.setBucketName("test-bucket");
    objectSummary1.setKey("a");
    objectSummary1.setLastModified(lastModified);
    objectListing.getObjectSummaries().add(objectSummary1);

    Mockito.when(mockS3Client.listObjects(Mockito.any(ListObjectsRequest.class))).thenReturn(objectListing);

    runner.run();

    ArgumentCaptor<GetObjectTaggingRequest> captureRequest = ArgumentCaptor.forClass(GetObjectTaggingRequest.class);
    Mockito.verify(mockS3Client, Mockito.times(1)).getObjectTagging(captureRequest.capture());
    GetObjectTaggingRequest request = captureRequest.getValue();

    assertEquals("test-bucket", request.getBucketName());
    assertEquals("a", request.getKey());
    Mockito.verify(mockS3Client, Mockito.never()).listVersions(Mockito.any());
}
 

@Override
public List<OldNoteInfo> list(AuthenticationInfo subject) throws IOException {
  List<OldNoteInfo> infos = new LinkedList<>();
  OldNoteInfo info;
  try {
    ListObjectsRequest listObjectsRequest = new ListObjectsRequest()
            .withBucketName(bucketName)
            .withPrefix(user + "/" + "notebook");
    ObjectListing objectListing;
    do {
      objectListing = s3client.listObjects(listObjectsRequest);
      for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) {
        if (objectSummary.getKey().endsWith("note.json")) {
          info = getNoteInfo(objectSummary.getKey());
          if (info != null) {
            infos.add(info);
          } else {
            LOG.debug("Unable to get notebook info for key: " + objectSummary.getKey());
          }
        }
      }
      listObjectsRequest.setMarker(objectListing.getNextMarker());
    } while (objectListing.isTruncated());
  } catch (AmazonClientException ace) {
    throw new IOException("Unable to list objects in S3: " + ace, ace);
  }
  return infos;
}
 

@Override
public void handleAttachException( String key, String bucket ) throws FileSystemException {
  SimpleEntry<String, String> newPath = fixFilePath( key, bucket );
  String keyWithDelimiter = newPath.getKey() + DELIMITER;
  try {
    s3Object = getS3Object( keyWithDelimiter, newPath.getValue() );
    s3ObjectMetadata = s3Object.getObjectMetadata();
    injectType( FileType.FOLDER );
  } catch ( AmazonS3Exception e2 ) {
    ListObjectsRequest listObjectsRequest = new ListObjectsRequest()
            .withBucketName( newPath.getValue() )
            .withPrefix( keyWithDelimiter )
            .withDelimiter( DELIMITER );
    ObjectListing ol = fileSystem.getS3Client().listObjects( listObjectsRequest );

    if ( !( ol.getCommonPrefixes().isEmpty() && ol.getObjectSummaries().isEmpty() ) ) {
      injectType( FileType.FOLDER );
    } else {
      //Folders don't really exist - they will generate a "NoSuchKey" exception
      String errorCode = e2.getErrorCode();
      // confirms key doesn't exist but connection okay
      if ( !errorCode.equals( "NoSuchKey" ) ) {
        // bubbling up other connection errors
        logger.error( "Could not get information on " + getQualifiedName(),
                e2 ); // make sure this gets printed for the user
        throw new FileSystemException( "vfs.provider/get-type.error", getQualifiedName(), e2 );
      }
    }
  }
}
 

@Test
public void getSummariesStream() {
    answer = true;
    AmazonS3Client client = mock(AmazonS3Client.class);

    ObjectListing objectListing = mock(ObjectListing.class);

    when(objectListing.getObjectSummaries()).thenReturn(createSummaries());
    when(client.listObjects(any(ListObjectsRequest.class))).thenReturn(objectListing);
    when(objectListing.isTruncated()).thenAnswer(__ -> {
        try {
            return answer;
        } finally {
            answer = false;
        }
    });
    // when(objectListing.getObjectSummaries()).thenReturn(summaries);

    S3Utils utils = new S3Utils(
                                client, null, null, false, null);
    verify(objectListing, times(0)).getObjectSummaries();
    Stream<String> stream = utils.getSummariesStream(new ListObjectsRequest(), s -> {
        return s.getKey();
    });

    assertEquals(500, stream.limit(500)
                            .count());

    verify(objectListing, times(1)).getObjectSummaries();

}
 

@Test
public void getSummariesStreamFull() {
    answer = true;
    AmazonS3Client client = mock(AmazonS3Client.class);

    ObjectListing objectListing = mock(ObjectListing.class);

    when(objectListing.getObjectSummaries()).thenReturn(createSummaries());
    when(client.listObjects(any(ListObjectsRequest.class))).thenReturn(objectListing);
    when(objectListing.isTruncated()).thenAnswer(__ -> {
        try {
            return answer;
        } finally {
            answer = false;
        }
    });
    // when(objectListing.getObjectSummaries()).thenReturn(summaries);

    S3Utils utils = new S3Utils(
                                client, null, null, false, null);
    verify(objectListing, times(0)).getObjectSummaries();
    Stream<String> stream = utils.getSummariesStream(new ListObjectsRequest(), s -> {
        return s.getKey();
    });

    assertEquals(2000, stream.limit(2000)
                             .count());

}
 

/**
 * {@inheritDoc}
 * <p/>
 * If the bucket does not exist, returns a listing with an empty list. If a prefix is specified in listObjectsRequest, only keys starting with the prefix
 * will be returned.
 */
@Override
public ObjectListing listObjects(ListObjectsRequest listObjectsRequest, AmazonS3 s3Client)
{
    LOGGER.debug("listObjects(): listObjectsRequest.getBucketName() = " + listObjectsRequest.getBucketName());

    String bucketName = listObjectsRequest.getBucketName();

    if (MOCK_S3_BUCKET_NAME_NO_SUCH_BUCKET_EXCEPTION.equals(bucketName))
    {
        AmazonS3Exception amazonS3Exception = new AmazonS3Exception(MOCK_S3_BUCKET_NAME_NO_SUCH_BUCKET_EXCEPTION);
        amazonS3Exception.setErrorCode("NoSuchBucket");
        throw amazonS3Exception;
    }

    ObjectListing objectListing = new ObjectListing();
    objectListing.setBucketName(bucketName);

    MockS3Bucket mockS3Bucket = mockS3Buckets.get(bucketName);
    if (mockS3Bucket != null)
    {
        for (MockS3Object mockS3Object : mockS3Bucket.getObjects().values())
        {
            String s3ObjectKey = mockS3Object.getKey();
            if (listObjectsRequest.getPrefix() == null || s3ObjectKey.startsWith(listObjectsRequest.getPrefix()))
            {
                S3ObjectSummary s3ObjectSummary = new S3ObjectSummary();
                s3ObjectSummary.setBucketName(bucketName);
                s3ObjectSummary.setKey(s3ObjectKey);
                s3ObjectSummary.setSize(mockS3Object.getData().length);
                s3ObjectSummary.setStorageClass(mockS3Object.getObjectMetadata() != null ? mockS3Object.getObjectMetadata().getStorageClass() : null);

                objectListing.getObjectSummaries().add(s3ObjectSummary);
            }
        }
    }

    return objectListing;
}
 

/**
 * Call S3 to get the most recent object list.
 * <p>
 * This is an object list request to AWS in the given "directory".
 */
private List<S3ObjectSummary> getObjectSummaries()
{
    AmazonS3Client s3client = clientManager.getS3Client();
    AmazonS3URI directoryURI = new AmazonS3URI(bucketUrl.get());

    List<S3ObjectSummary> result = new ArrayList<>();
    try {
        log.info("Getting the listing of objects in the S3 table config directory: bucket %s prefix %s :", directoryURI.getBucket(), directoryURI.getKey());
        ListObjectsRequest request = new ListObjectsRequest()
                .withBucketName(directoryURI.getBucket())
                .withPrefix(directoryURI.getKey() + "/")
                .withDelimiter("/")
                .withMaxKeys(25);
        ObjectListing response;

        do {
            response = s3client.listObjects(request);

            result.addAll(response.getObjectSummaries());
            request.setMarker(response.getNextMarker());
        }
        while (response.isTruncated());

        log.info("Completed getting S3 object listing.");
    }
    catch (AmazonClientException e) {
        log.error("Skipping update as faced error fetching table descriptions from S3 " + e.toString());
    }
    return result;
}
 

@Override
public void list(String keyPrefix, FileListing callback)
{
    checkArgument(keyPrefix != null, "keyPrefix is null");

    String errorMessage = "listing files on bucket " + bucket + " prefix " + keyPrefix;

    ListObjectsRequest req = new ListObjectsRequest();
    req.setBucketName(bucket);
    req.setPrefix(keyPrefix);

    ObjectListing listing;
    do {
        try {
            listing = getWithRetry(errorMessage, () -> client.listObjects(req));
        }
        catch (StorageFileNotFoundException ex) {
            throw Throwables.propagate(ex.getCause());
        }
        callback.accept(Lists.transform(
                    listing.getObjectSummaries(),
                    (summary) -> StorageObjectSummary.builder()
                        .key(summary.getKey())
                        .contentLength(summary.getSize())
                        .lastModified(summary.getLastModified().toInstant())
                        .build()
                    ));
        req.setMarker(listing.getNextMarker());
    }
    while (listing.isTruncated());
}
 

@Test
public void verifyObjectListingRequestIsRight() {
    doReturn(null).when(mockClient).listObjects(any(ListObjectsRequest.class));
    S3ArtifactStore store = new S3ArtifactStore(mockClient, "foo-bar");
    store.getLatestPrefix("pipeline", "stage", "job", "1");

    verify(mockClient).listObjects(listingCaptor.capture());
    ListObjectsRequest request = listingCaptor.getValue();
    assertEquals("foo-bar", request.getBucketName());
    assertEquals("pipeline/stage/job/1.", request.getPrefix());
    assertEquals("/", request.getDelimiter());
}
 

private void findAllResourcesThatMatches(String bucketName, Set<Resource> resources,
		String prefix, String keyPattern) {
	ListObjectsRequest listObjectsRequest = new ListObjectsRequest()
			.withBucketName(bucketName).withPrefix(prefix);
	ObjectListing objectListing = null;

	do {
		try {
			if (objectListing == null) {
				objectListing = this.amazonS3.listObjects(listObjectsRequest);
			}
			else {
				objectListing = this.amazonS3.listNextBatchOfObjects(objectListing);
			}
			Set<Resource> newResources = getResourcesFromObjectSummaries(bucketName,
					keyPattern, objectListing.getObjectSummaries());
			if (!newResources.isEmpty()) {
				resources.addAll(newResources);
			}
		}
		catch (AmazonS3Exception e) {
			if (301 != e.getStatusCode()) {
				throw e;
			}
		}
	}
	while (objectListing != null && objectListing.isTruncated());
}
 

@Test
public void shouldReturnNullWhenObjectListingIsSize0() {
    ObjectListing listing = new ObjectListing();
    doReturn(listing).when(mockClient).listObjects(any(ListObjectsRequest.class));
    S3ArtifactStore store = new S3ArtifactStore(mockClient, "foo-bar");

    String prefix = store.getLatestPrefix("pipeline", "stage", "job", "1");
    assertNull(prefix);
}
 

@Test
public void testListObjectsNothingNew() throws IOException {
    runner.setProperty(ListS3.REGION, "eu-west-1");
    runner.setProperty(ListS3.BUCKET, "test-bucket");

    Calendar calendar = Calendar.getInstance();
    calendar.set(2017, 5, 2);
    Date objectLastModified = calendar.getTime();
    long stateCurrentTimestamp = objectLastModified.getTime();

    Map<String, String> state = new HashMap<>();
    state.put(ListS3.CURRENT_TIMESTAMP, String.valueOf(stateCurrentTimestamp));
    state.put(ListS3.CURRENT_KEY_PREFIX+"0", "test-key");
    MockStateManager mockStateManager = runner.getStateManager();
    mockStateManager.setState(state, Scope.CLUSTER);

    ObjectListing objectListing = new ObjectListing();
    S3ObjectSummary objectSummary1 = new S3ObjectSummary();
    objectSummary1.setBucketName("test-bucket");
    objectSummary1.setKey("test-key");
    objectSummary1.setLastModified(objectLastModified);
    objectListing.getObjectSummaries().add(objectSummary1);
    Mockito.when(mockS3Client.listObjects(Mockito.any(ListObjectsRequest.class))).thenReturn(objectListing);

    runner.run();

    ArgumentCaptor<ListObjectsRequest> captureRequest = ArgumentCaptor.forClass(ListObjectsRequest.class);
    Mockito.verify(mockS3Client, Mockito.times(1)).listObjects(captureRequest.capture());
    ListObjectsRequest request = captureRequest.getValue();
    assertEquals("test-bucket", request.getBucketName());
    Mockito.verify(mockS3Client, Mockito.never()).listVersions(Mockito.any());

    runner.assertAllFlowFilesTransferred(ListS3.REL_SUCCESS, 0);
}
 

/***
 * Get list of S3 objects within a S3 bucket qualified by prefix path
 *
 * @param bucketName S3 bucket name
 * @param prefix S3 prefix to object
 * @return List of {@link S3ObjectSummary} objects within the bucket qualified by prefix path
 */
public List<S3ObjectSummary> listS3Bucket(String bucketName,
    String prefix) {

  final AmazonS3 amazonS3 = getS3Client();

  final ListObjectsRequest listObjectsRequest = new ListObjectsRequest()
      .withBucketName(bucketName)
      .withPrefix(prefix);

  final ObjectListing objectListing = amazonS3.listObjects(listObjectsRequest);
  LOGGER.info("S3 bucket listing for bucket: " + bucketName + " with prefix: " + prefix + " is: " + objectListing);

  return objectListing.getObjectSummaries();
}
 

@Test
public void testBlobListRecursiveImplicitMarker() throws Exception {
    ObjectListing listing = client.listObjects(containerName);
    assertThat(listing.getObjectSummaries()).isEmpty();

    ObjectMetadata metadata = new ObjectMetadata();
    metadata.setContentLength(BYTE_SOURCE.size());
    client.putObject(containerName, "blob1", BYTE_SOURCE.openStream(),
            metadata);
    client.putObject(containerName, "blob2", BYTE_SOURCE.openStream(),
            metadata);

    listing = client.listObjects(new ListObjectsRequest()
            .withBucketName(containerName)
            .withMaxKeys(1));
    assertThat(listing.getObjectSummaries()).hasSize(1);
    assertThat(listing.getObjectSummaries().iterator().next().getKey())
            .isEqualTo("blob1");

    listing = client.listObjects(new ListObjectsRequest()
            .withBucketName(containerName)
            .withMaxKeys(1)
            .withMarker("blob1"));
    assertThat(listing.getObjectSummaries()).hasSize(1);
    assertThat(listing.getObjectSummaries().iterator().next().getKey())
            .isEqualTo("blob2");
}