Java Code Examples for com.amazonaws.services.s3.model.ObjectListing#getNextMarker()

public static void s3DeleteRecursively(AmazonS3 s3, String bucket, String prefix)
        throws Exception
{
    ListObjectsRequest request = new ListObjectsRequest()
            .withBucketName(bucket)
            .withPrefix(prefix);

    while (true) {
        ObjectListing listing = s3.listObjects(request);
        String[] keys = listing.getObjectSummaries().stream().map(S3ObjectSummary::getKey).toArray(String[]::new);
        for (String key : keys) {
            logger.info("delete s3://{}/{}", bucket, key);
        }
        retryExecutor()
                .retryIf(e -> e instanceof AmazonServiceException)
                .run(() -> s3.deleteObjects(new DeleteObjectsRequest(bucket).withKeys(keys)));
        if (listing.getNextMarker() == null) {
            break;
        }
    }
}
 

/**
 * Download to executor
 */
@Override
public Integer invoke(TransferManager transferManager, File base, VirtualChannel channel) throws IOException, InterruptedException {
    if(!base.exists()) {
        if (!base.mkdirs()) {
            throw new IOException("Failed to create directory : " + base);
        }
    }

    int totalCount;
    Downloads downloads = new Downloads();
    ObjectListing objectListing = null;

    do {
        objectListing = transferManager.getAmazonS3Client().listObjects(new ListObjectsRequest()
                .withBucketName(bucketName)
                .withPrefix(pathPrefix)
                .withMarker(objectListing != null ? objectListing.getNextMarker() : null));

        for (S3ObjectSummary summary : objectListing.getObjectSummaries()) {
            downloads.startDownload(transferManager, base, pathPrefix, summary);
        }

    } while (objectListing.getNextMarker() != null);

    // Grab # of files copied
    totalCount = downloads.count();

    // Finish the asynchronous downloading process
    downloads.finishDownloading();

    return totalCount;
}
 

/**
 * Method returns list of all <b>S3ObjectSummary</b> objects, subject to
 * <i>req</i> parameters. Multiple S3 calls will be performed if there are
 * more than 1000 elements there
 * 
 * @param req
 *            - ListObjectRequest to be used.
 * @return List of S3ObjectSummary from bucket,
 */
public List<S3ObjectSummary> getAllSummaries(ListObjectsRequest req) {
    List<S3ObjectSummary> result = new ArrayList<>();
    String marker = null;
    ListObjectsRequest req2 = (ListObjectsRequest) req.clone();
    ObjectListing listing;
    do {
        listing = client.listObjects(req2.withMarker(marker));
        marker = listing.getNextMarker();
        result.addAll(listing.getObjectSummaries());
    } while (listing.isTruncated());

    return result;
}
 

/**
 * Gets all tables available in this stash.
 */
public Iterator<StashTable> listTables() {
    final String root = getRootPath();
    final String prefix = String.format("%s/", root);

    return new AbstractIterator<StashTable>() {
        Iterator<String> _commonPrefixes = Iterators.emptyIterator();
        String _marker = null;
        boolean _truncated = true;

        @Override
        protected StashTable computeNext() {
            String dir = null;

            while (dir == null) {
                if (_commonPrefixes.hasNext()) {
                    dir = _commonPrefixes.next();
                    if (dir.isEmpty()) {
                        // Ignore the empty directory if it comes back
                        dir = null;
                    } else {
                        // Strip the prefix and trailing "/"
                        dir = dir.substring(prefix.length(), dir.length()-1);
                    }
                } else if (_truncated) {
                    ObjectListing response = _s3.listObjects(new ListObjectsRequest()
                            .withBucketName(_bucket)
                            .withPrefix(prefix)
                            .withDelimiter("/")
                            .withMarker(_marker)
                            .withMaxKeys(1000));

                    _commonPrefixes = response.getCommonPrefixes().iterator();
                    _marker = response.getNextMarker();
                    _truncated = response.isTruncated();
                } else {
                    return endOfData();
                }
            }

            String tablePrefix = prefix + dir + "/";
            String tableName = StashUtil.decodeStashTable(dir);
            return new StashTable(_bucket, tablePrefix, tableName);
        }
    };
}
 

/**
 * Gets the metadata for all tables in this stash.  This is a heavier operation that just {@link #listTables()}
 * since it also returns full file details for the entire Stash instead of just table names.
 */
public Iterator<StashTableMetadata> listTableMetadata() {
    final String root = getRootPath();
    final String prefix = String.format("%s/", root);
    final int prefixLength = prefix.length();

    return new AbstractIterator<StashTableMetadata>() {
        PeekingIterator<S3ObjectSummary> _listResponse =
                Iterators.peekingIterator(Iterators.<S3ObjectSummary>emptyIterator());
        String _marker = null;
        boolean _truncated = true;

        @Override
        protected StashTableMetadata computeNext() {
            String tableDir = null;
            List<StashFileMetadata> files = Lists.newArrayListWithCapacity(16);
            boolean allFilesRead = false;

            while (!allFilesRead) {
                if (_listResponse.hasNext()) {
                    // Peek at the next record but don't consume it until we verify it's part of the same table
                    S3ObjectSummary s3File = _listResponse.peek();
                    String key = s3File.getKey();

                    // Don't include the _SUCCESS file or any other stray files we may find
                    String[] parentDirAndFile = key.substring(prefixLength).split("/");
                    if (parentDirAndFile.length != 2) {
                        // Consume and skip this row
                        _listResponse.next();
                    } else {
                        String parentDir = parentDirAndFile[0];
                        if (tableDir == null) {
                            tableDir = parentDir;
                        }

                        if (!parentDir.equals(tableDir)) {
                            allFilesRead = true;
                        } else {
                            // Record is part of this table; consume it now
                            _listResponse.next();
                            files.add(new StashFileMetadata(_bucket, key, s3File.getSize()));
                        }
                    }
                } else if (_truncated) {
                    ObjectListing response = _s3.listObjects(new ListObjectsRequest()
                            .withBucketName(_bucket)
                            .withPrefix(prefix)
                            .withMarker(_marker)
                            .withMaxKeys(1000));

                    _listResponse = Iterators.peekingIterator(response.getObjectSummaries().iterator());
                    _marker = response.getNextMarker();
                    _truncated = response.isTruncated();
                } else {
                    allFilesRead = true;
                }
            }

            if (tableDir == null) {
                // No files read this iteration means all files have been read
                return endOfData();
            }

            String tablePrefix = prefix + tableDir + "/";
            String tableName = StashUtil.decodeStashTable(tableDir);
            return new StashTableMetadata(_bucket, tablePrefix, tableName, files);
        }
    };
}
 

private void scanBucket(Set<TocInfo> toc, Queue<TocInfo> tocQueue) throws Exception {
	
	ListObjectsRequest listRequest = new ListObjectsRequest();
	listRequest.setBucketName(s3BucketName);
	// listRequest.setGeneralProgressListener(this);
	listRequest.setMaxKeys(1000);
	
	String nextMarker = null;
	ObjectListing objectListing = null;
	
	while(true) {
		
		objectListing = s3Client.listObjects(listRequest);
		
		List<S3ObjectSummary> objectSummaries = objectListing.getObjectSummaries();
		
		for (S3ObjectSummary objSummary : objectSummaries) {
			String key = objSummary.getKey();
			
			TocInfo tocInfo = new TocInfo(key, objSummary.getSize());
			
			// is it a "dir/" ?
			if (key.lastIndexOf("/") == (key.length() - 1)) {
				tocInfo.isDirectory = true;
			} else {
				tocInfo.isDirectory = false;
			}
			
			toc.add(tocInfo);
			tocQueue.add(tocInfo);
			tocInfosGenerated++; // increment for logging

		}
		
		// for pagination
		nextMarker = objectListing.getNextMarker();
		if (nextMarker == null) {
			break;
		} else {
			listRequest.setMarker(nextMarker);
			logger.debug("scanBucket() nextMarker we will request listing for => " + nextMarker);
		}
	}
	
}