com.amazonaws.services.s3.model.ListObjectsRequest#setMarker

Source File: S3NotebookRepo.java From zeppelin with Apache License 2.0

6 votes

@Override
public void move(String folderPath, String newFolderPath, AuthenticationInfo subject) throws IOException {
  try {
    ListObjectsRequest listObjectsRequest = new ListObjectsRequest()
            .withBucketName(bucketName)
            .withPrefix(rootFolder + folderPath + "/");
    ObjectListing objectListing;
    do {
      objectListing = s3client.listObjects(listObjectsRequest);
      for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) {
        if (objectSummary.getKey().endsWith(".zpln")) {
          String noteId = getNoteId(objectSummary.getKey());
          String notePath = getNotePath(rootFolder, objectSummary.getKey());
          String newNotePath = newFolderPath + notePath.substring(folderPath.length());
          move(noteId, notePath, newNotePath, subject);
        }
      }
      listObjectsRequest.setMarker(objectListing.getNextMarker());
    } while (objectListing.isTruncated());
  } catch (AmazonClientException ace) {
    throw new IOException("Fail to move folder: " + folderPath + " to " + newFolderPath  + " in S3" , ace);
  }
}

Source File: S3Service.java From fullstop with Apache License 2.0

6 votes

public List<String> listCommonPrefixesS3Objects(final String bucketName, final String prefix) {
    final List<String> commonPrefixes = Lists.newArrayList();

    try {
        log.debug("Listing objects in bucket '{}' with prefix '{}'", bucketName, prefix);

        final ListObjectsRequest listObjectsRequest = new ListObjectsRequest()
                .withDelimiter("/")
                .withBucketName(bucketName)
                .withPrefix(prefix);

        ObjectListing objectListing;

        do {
            objectListing = s3client.listObjects(listObjectsRequest);
            objectListing.getCommonPrefixes().stream().map(S3Service::urlDecode).forEach(commonPrefixes::add);
            listObjectsRequest.setMarker(objectListing.getNextMarker());
        } while (objectListing.isTruncated());

    } catch (final AmazonServiceException e) {
        log.error("Could not list common prefixes in S3", e);
    }

    return commonPrefixes;
}

Source File: S3TableConfigClient.java From presto with Apache License 2.0

5 votes

/**
 * Call S3 to get the most recent object list.
 * <p>
 * This is an object list request to AWS in the given "directory".
 */
private List<S3ObjectSummary> getObjectSummaries()
{
    AmazonS3Client s3client = clientManager.getS3Client();
    AmazonS3URI directoryURI = new AmazonS3URI(bucketUrl.get());

    List<S3ObjectSummary> result = new ArrayList<>();
    try {
        log.info("Getting the listing of objects in the S3 table config directory: bucket %s prefix %s :", directoryURI.getBucket(), directoryURI.getKey());
        ListObjectsRequest request = new ListObjectsRequest()
                .withBucketName(directoryURI.getBucket())
                .withPrefix(directoryURI.getKey() + "/")
                .withDelimiter("/")
                .withMaxKeys(25);
        ObjectListing response;

        do {
            response = s3client.listObjects(request);

            result.addAll(response.getObjectSummaries());
            request.setMarker(response.getNextMarker());
        }
        while (response.isTruncated());

        log.info("Completed getting S3 object listing.");
    }
    catch (AmazonClientException e) {
        log.error("Skipping update as faced error fetching table descriptions from S3 " + e.toString());
    }
    return result;
}

Source File: OldS3NotebookRepo.java From zeppelin with Apache License 2.0

5 votes

@Override
public List<OldNoteInfo> list(AuthenticationInfo subject) throws IOException {
  List<OldNoteInfo> infos = new LinkedList<>();
  OldNoteInfo info;
  try {
    ListObjectsRequest listObjectsRequest = new ListObjectsRequest()
            .withBucketName(bucketName)
            .withPrefix(user + "/" + "notebook");
    ObjectListing objectListing;
    do {
      objectListing = s3client.listObjects(listObjectsRequest);
      for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) {
        if (objectSummary.getKey().endsWith("note.json")) {
          info = getNoteInfo(objectSummary.getKey());
          if (info != null) {
            infos.add(info);
          } else {
            LOG.debug("Unable to get notebook info for key: " + objectSummary.getKey());
          }
        }
      }
      listObjectsRequest.setMarker(objectListing.getNextMarker());
    } while (objectListing.isTruncated());
  } catch (AmazonClientException ace) {
    throw new IOException("Unable to list objects in S3: " + ace, ace);
  }
  return infos;
}

Source File: S3NotebookRepo.java From zeppelin with Apache License 2.0

5 votes

@Override
public Map<String, NoteInfo> list(AuthenticationInfo subject) throws IOException {
  Map<String, NoteInfo> notesInfo = new HashMap<>();
  try {
    ListObjectsRequest listObjectsRequest = new ListObjectsRequest()
            .withBucketName(bucketName)
            .withPrefix(user + "/" + "notebook");
    ObjectListing objectListing;
    do {
      objectListing = s3client.listObjects(listObjectsRequest);
      for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) {
        if (objectSummary.getKey().endsWith(".zpln")) {
          try {
            NoteInfo info = getNoteInfo(objectSummary.getKey());
            notesInfo.put(info.getId(), info);
          } catch (IOException e) {
            LOGGER.warn(e.getMessage());
          }
        }
      }
      listObjectsRequest.setMarker(objectListing.getNextMarker());
    } while (objectListing.isTruncated());
  } catch (AmazonClientException ace) {
    throw new IOException("Fail to list objects in S3", ace);
  }
  return notesInfo;
}

Source File: S3Service.java From fullstop with Apache License 2.0

5 votes

public List<String> listS3Objects(final String bucketName, final String prefix) {
    final List<String> s3Objects = Lists.newArrayList();

    try {
        log.debug("Listing objects");

        final ListObjectsRequest listObjectsRequest = new ListObjectsRequest()
                .withDelimiter("/")
                .withBucketName(bucketName)
                .withPrefix(prefix);

        ObjectListing objectListing;

        do {
            objectListing = s3client.listObjects(listObjectsRequest);
            for (final S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) {
                if (objectSummary.getKey().equals(prefix)) {
                    continue;
                }
                s3Objects.add(objectSummary.getKey());
            }

            listObjectsRequest.setMarker(objectListing.getNextMarker());
        } while (objectListing.isTruncated());

    } catch (final AmazonServiceException e) {
        log.error("Error Message:    " + e.getMessage());
    }

    return s3Objects;
}

Source File: S3Storage.java From digdag with Apache License 2.0

5 votes

@Override
public void list(String keyPrefix, FileListing callback)
{
    checkArgument(keyPrefix != null, "keyPrefix is null");

    String errorMessage = "listing files on bucket " + bucket + " prefix " + keyPrefix;

    ListObjectsRequest req = new ListObjectsRequest();
    req.setBucketName(bucket);
    req.setPrefix(keyPrefix);

    ObjectListing listing;
    do {
        try {
            listing = getWithRetry(errorMessage, () -> client.listObjects(req));
        }
        catch (StorageFileNotFoundException ex) {
            throw Throwables.propagate(ex.getCause());
        }
        callback.accept(Lists.transform(
                    listing.getObjectSummaries(),
                    (summary) -> StorageObjectSummary.builder()
                        .key(summary.getKey())
                        .contentLength(summary.getSize())
                        .lastModified(summary.getLastModified().toInstant())
                        .build()
                    ));
        req.setMarker(listing.getNextMarker());
    }
    while (listing.isTruncated());
}

Source File: S3Utils.java From cloudstack with Apache License 2.0

5 votes

public static List<S3ObjectSummary> listDirectory(final ClientOptions clientOptions, final String bucketName, final String directory) {
    LOGGER.debug(format("Listing S3 directory %1$s in bucket %2$s", directory, bucketName));

    List<S3ObjectSummary> objects = new ArrayList<>();
    ListObjectsRequest listObjectsRequest = new ListObjectsRequest();

    listObjectsRequest.withBucketName(bucketName);
    listObjectsRequest.withPrefix(directory);

    ObjectListing ol = getAmazonS3Client(clientOptions).listObjects(listObjectsRequest);
    if(ol.isTruncated()) {
        do {
            objects.addAll(ol.getObjectSummaries());
            listObjectsRequest.setMarker(ol.getNextMarker());
            ol = getAmazonS3Client(clientOptions).listObjects(listObjectsRequest);
        } while (ol.isTruncated());
    }
    else {
        objects.addAll(ol.getObjectSummaries());
    }

    if (objects.isEmpty()) {
        return emptyList();
    }

    return unmodifiableList(objects);
}

Source File: S3DaoImpl.java From herd with Apache License 2.0

4 votes

@Override
public List<S3ObjectSummary> listDirectory(final S3FileTransferRequestParamsDto params, boolean ignoreZeroByteDirectoryMarkers)
{
    Assert.isTrue(!isRootKeyPrefix(params.getS3KeyPrefix()), "Listing of S3 objects from root directory is not allowed.");

    AmazonS3Client s3Client = getAmazonS3(params);
    List<S3ObjectSummary> s3ObjectSummaries = new ArrayList<>();

    try
    {
        ListObjectsRequest listObjectsRequest = new ListObjectsRequest().withBucketName(params.getS3BucketName()).withPrefix(params.getS3KeyPrefix());
        ObjectListing objectListing;

        do
        {
            objectListing = s3Operations.listObjects(listObjectsRequest, s3Client);

            for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries())
            {
                // Ignore 0 byte objects that represent S3 directories.
                if (!(ignoreZeroByteDirectoryMarkers && objectSummary.getKey().endsWith("/") && objectSummary.getSize() == 0L))
                {
                    s3ObjectSummaries.add(objectSummary);
                }
            }

            listObjectsRequest.setMarker(objectListing.getNextMarker());
        }
        while (objectListing.isTruncated());
    }
    catch (AmazonS3Exception amazonS3Exception)
    {
        if (S3Operations.ERROR_CODE_NO_SUCH_BUCKET.equals(amazonS3Exception.getErrorCode()))
        {
            throw new IllegalArgumentException("The specified bucket '" + params.getS3BucketName() + "' does not exist.", amazonS3Exception);
        }
        throw new IllegalStateException("Error accessing S3", amazonS3Exception);
    }
    catch (AmazonClientException e)
    {
        throw new IllegalStateException(String
            .format("Failed to list keys with prefix \"%s\" from bucket \"%s\". Reason: %s", params.getS3KeyPrefix(), params.getS3BucketName(),
                e.getMessage()), e);
    }
    finally
    {
        // Shutdown the AmazonS3Client instance to release resources.
        s3Client.shutdown();
    }

    return s3ObjectSummaries;
}

Source File: AmazonS3Util.java From datacollector with Apache License 2.0

4 votes

/**
 * Lists objects from AmazonS3 in lexicographical order
 *
 * @param s3Client
 * @param s3ConfigBean
 * @param pathMatcher glob patterns to match file name against
 * @param s3Offset current offset which provides the key name of the previous object
 * @param fetchSize number of objects to fetch in one go
 * @return
 * @throws AmazonClientException
 */
static List<S3ObjectSummary> listObjectsLexicographically(
    AmazonS3 s3Client,
    S3ConfigBean s3ConfigBean,
    AntPathMatcher pathMatcher,
    S3Offset s3Offset,
    int fetchSize
) {
  // Incrementally scan objects after the marker (s3Offset).
  List<S3ObjectSummary> list = new ArrayList<>(fetchSize);

  ListObjectsRequest listObjectsRequest = new ListObjectsRequest();
  listObjectsRequest.setBucketName(s3ConfigBean.s3Config.bucket);
  listObjectsRequest.setPrefix(s3ConfigBean.s3Config.commonPrefix);
  listObjectsRequest.setMaxKeys(BATCH_SIZE);

  if (s3Offset.getKey() != null) {
    if (!s3Offset.getKey().isEmpty() && parseOffset(s3Offset) != -1) {
      S3ObjectSummary currentObjectSummary = getObjectSummary(s3Client, s3ConfigBean.s3Config.bucket, s3Offset.getKey());
      list.add(currentObjectSummary);
    }
    listObjectsRequest.setMarker(s3Offset.getKey());
  }

  ObjectListing objectListing = s3Client.listObjects(listObjectsRequest);

  while (true) {
    for (S3ObjectSummary s : objectListing.getObjectSummaries()) {
      String fullPrefix = s.getKey();
      String remainingPrefix = fullPrefix.substring(s3ConfigBean.s3Config.commonPrefix.length(), fullPrefix.length());
      if (!remainingPrefix.isEmpty()) {
        if (pathMatcher.match(s3ConfigBean.s3FileConfig.prefixPattern, remainingPrefix)) {
          list.add(s);
        }
        // We've got enough objects.
        if (list.size() == fetchSize) {
          return list;
        }
      }
    }
    // Listing is complete. No more objects to be listed.
    if (!objectListing.isTruncated()) {
      break;
    }
    objectListing = s3Client.listNextBatchOfObjects(objectListing);
  }

  return list;
}

Source File: List.java From openbd-core with GNU General Public License v3.0

4 votes

public cfData execute( cfSession _session, cfArgStructData argStruct ) throws cfmRunTimeException {

		AmazonKey amazonKey = getAmazonKey( _session, argStruct );
		AmazonS3 s3Client = getAmazonS3( amazonKey );

		String bucket = getNamedStringParam( argStruct, "bucket", null );
		String prefix = getNamedStringParam( argStruct, "prefix", "" );

		if ( bucket == null )
			throwException( _session, "Please specify a bucket" );

		try {
			// Create the results
			cfQueryResultData qD = new cfQueryResultData( new String[] { "key", "size", "modified", "etag" }, null );
			qD.setQuerySource( "AmazonS3." + amazonKey.getDataSource() );

			ListObjectsRequest listObjectsRequest = new ListObjectsRequest()
					.withBucketName( bucket )
					.withDelimiter( "/" )
					.withPrefix( prefix );
			ObjectListing objectListing;

			do {
				objectListing = s3Client.listObjects( listObjectsRequest );

				java.util.List<String> prefixes = objectListing.getCommonPrefixes();

				// first add the prefixes
				for ( String nextPrefix : prefixes ) {
					qD.addRow( 1 );
					qD.setCurrentRow( qD.getSize() );

					qD.setCell( 1, new cfStringData( nextPrefix ) );
					qD.setCell( 2, new cfNumberData( 0 ) );
					qD.setCell( 3, cfNullData.NULL );
					qD.setCell( 4, cfNullData.NULL );

				}

				for ( S3ObjectSummary objectSummary : objectListing.getObjectSummaries() ) {

					// don't include the prefix being listed
					if ( objectSummary.getKey().equals( prefix ) ) {
						continue;
					}
					qD.addRow( 1 );
					qD.setCurrentRow( qD.getSize() );

					qD.setCell( 1, new cfStringData( objectSummary.getKey() ) );
					qD.setCell( 2, new cfNumberData( objectSummary.getSize() ) );
					qD.setCell( 3, new cfDateData( objectSummary.getLastModified() ) );
					qD.setCell( 4, new cfStringData( objectSummary.getETag() ) );
				}

				listObjectsRequest.setMarker( objectListing.getNextMarker() );
			} while ( objectListing.isTruncated() );

			return qD;
		} catch ( Exception e ) {
			throwException( _session, "AmazonS3: " + e.getMessage() );
			return cfBooleanData.FALSE;
		}
	}

Source File: S3BucketObjectLister.java From s3-bucket-loader with Apache License 2.0

4 votes

private void scanBucket(Set<TocInfo> toc, Queue<TocInfo> tocQueue) throws Exception {
	
	ListObjectsRequest listRequest = new ListObjectsRequest();
	listRequest.setBucketName(s3BucketName);
	// listRequest.setGeneralProgressListener(this);
	listRequest.setMaxKeys(1000);
	
	String nextMarker = null;
	ObjectListing objectListing = null;
	
	while(true) {
		
		objectListing = s3Client.listObjects(listRequest);
		
		List<S3ObjectSummary> objectSummaries = objectListing.getObjectSummaries();
		
		for (S3ObjectSummary objSummary : objectSummaries) {
			String key = objSummary.getKey();
			
			TocInfo tocInfo = new TocInfo(key, objSummary.getSize());
			
			// is it a "dir/" ?
			if (key.lastIndexOf("/") == (key.length() - 1)) {
				tocInfo.isDirectory = true;
			} else {
				tocInfo.isDirectory = false;
			}
			
			toc.add(tocInfo);
			tocQueue.add(tocInfo);
			tocInfosGenerated++; // increment for logging

		}
		
		// for pagination
		nextMarker = objectListing.getNextMarker();
		if (nextMarker == null) {
			break;
		} else {
			listRequest.setMarker(nextMarker);
			logger.debug("scanBucket() nextMarker we will request listing for => " + nextMarker);
		}
	}
	
}

Source File: BucketClass.java From cloudExplorer with GNU General Public License v3.0

4 votes

String getObjectInfo(String key, String access_key,
        String secret_key, String bucket,
        String endpoint, String process
) {
    AWSCredentials credentials = new BasicAWSCredentials(access_key, secret_key);
    AmazonS3 s3Client = new AmazonS3Client(credentials,
            new ClientConfiguration());
    if (endpoint.contains("amazonaws.com")) {
        String aws_endpoint = s3Client.getBucketLocation(new GetBucketLocationRequest(bucket));
        if (aws_endpoint.contains("US")) {
            s3Client.setEndpoint("https://s3.amazonaws.com");
        } else if (aws_endpoint.contains("us-west")) {
            s3Client.setEndpoint("https://s3-" + aws_endpoint + ".amazonaws.com");
        } else if (aws_endpoint.contains("eu-west")) {
            s3Client.setEndpoint("https://s3-" + aws_endpoint + ".amazonaws.com");
        } else if (aws_endpoint.contains("ap-")) {
            s3Client.setEndpoint("https://s3-" + aws_endpoint + ".amazonaws.com");
        } else if (aws_endpoint.contains("sa-east-1")) {
            s3Client.setEndpoint("https://s3-" + aws_endpoint + ".amazonaws.com");
        } else {
            s3Client.setEndpoint("https://s3." + aws_endpoint + ".amazonaws.com");
        }
    } else {
        s3Client.setS3ClientOptions(S3ClientOptions.builder().setPathStyleAccess(true).build());
        s3Client.setEndpoint(endpoint);
    }
    objectlist = null;

    try {
        ObjectListing current = s3Client.listObjects((bucket));

        ListObjectsRequest listObjectsRequest = new ListObjectsRequest().withBucketName(bucket);
        ObjectListing objectListing;
        do {
            objectListing = s3Client.listObjects(listObjectsRequest);

            for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) {
                if (process.contains("checkmd5")) {
                    if (objectSummary.getKey().contains(key)) {
                        objectlist = objectSummary.getETag();
                        break;
                    }
                }
                if (process.contains("objectsize")) {
                    if (objectSummary.getKey().contains(key)) {
                        objectlist = String.valueOf(objectSummary.getSize());
                        break;
                    }
                }

                if (process.contains("objectdate")) {
                    if (objectSummary.getKey().contains(key)) {
                        objectlist = String.valueOf(objectSummary.getLastModified());
                        break;
                    }

                }
            }
            listObjectsRequest.setMarker(objectListing.getNextMarker());
        } while (objectListing.isTruncated());

    } catch (Exception listBucket) {
        //  mainFrame.jTextArea1.append("\n" + listBucket.getMessage());
    }

    return objectlist;
}

Java Code Examples for com.amazonaws.services.s3.model.ListObjectsRequest#setMarker()