Java Code Examples for com.amazonaws.services.s3.iterable.S3Objects#withPrefix()

@Override
public Stream<WriteableConfiguration> getCachedConfigurations() throws IOException {

  Iterable<S3ObjectSummary> objectSummaries = S3Objects.withPrefix(s3, bucket, prefix);
  Stream<S3ObjectSummary> objectStream = StreamSupport.stream(objectSummaries.spliterator(), false);

  return objectStream.map(p -> {
    Integer version = getVersionIfMatch(p.getKey());
    if (version == null) {
      return null;
    }
    return new Pair<>(version, p);
  }).filter(Objects::nonNull)
      .sorted(Comparator.comparing(pair -> ((Pair<Integer, S3ObjectSummary>) pair).getFirst())
            .reversed()).map(pair -> new S3WritableConfiguration(s3, pair.getSecond(), Integer.toString(pair.getFirst())));

}
 

@Override
public Stream<BlobId> getDirectPathBlobIdStream(final String prefix) {
  String subpath = format("%s/%s", DIRECT_PATH_PREFIX, prefix);
  Iterable<S3ObjectSummary> summaries = S3Objects.withPrefix(s3, getConfiguredBucket(), subpath);
  return stream(summaries.spliterator(), false)
    .map(S3ObjectSummary::getKey)
    .filter(key -> key.endsWith(BLOB_ATTRIBUTE_SUFFIX))
    .map(this::attributePathToDirectPathBlobId);
}
 

@Override
public Stream<BlobId> getDirectPathBlobIdStream(final String prefix) {
  String subpath = getBucketPrefix() + format("%s/%s", DIRECT_PATH_PREFIX, prefix);
  Iterable<S3ObjectSummary> summaries = S3Objects.withPrefix(s3, getConfiguredBucket(), subpath);
  return stream(summaries.spliterator(), false)
    .map(S3ObjectSummary::getKey)
    .filter(key -> key.endsWith(BLOB_ATTRIBUTE_SUFFIX))
    .map(this::attributePathToDirectPathBlobId);
}
 

@Override
public Stream<BlobId> getBlobIdStream() {
  Iterable<S3ObjectSummary> summaries = S3Objects.withPrefix(s3, getConfiguredBucket(), CONTENT_PREFIX);
  return blobIdStream(summaries);
}
 

@Override
public Stream<BlobId> getBlobIdStream() {
  Iterable<S3ObjectSummary> summaries = S3Objects.withPrefix(s3, getConfiguredBucket(), getContentPrefix());
  return blobIdStream(summaries);
}
 

/**
 * Lists objects from AmazonS3 in chronological order [lexicographical order if 2 files have same timestamp] which are
 * later than or equal to the timestamp of the previous offset object
 *
 * @param s3Client
 * @param s3ConfigBean
 * @param pathMatcher glob patterns to match file name against
 * @param s3Offset current offset which provides the timestamp of the previous object
 * @param fetchSize number of objects to fetch in one go
 * @return
 * @throws AmazonClientException
 */
static List<S3ObjectSummary> listObjectsChronologically(
    AmazonS3 s3Client,
    S3ConfigBean s3ConfigBean,
    AntPathMatcher pathMatcher,
    S3Offset s3Offset,
    int fetchSize
) {

  //Algorithm:
  // - Full scan all objects that match the file name pattern and which are later than the file in the offset
  // - Select the oldest "fetchSize" number of files and return them.
  TreeSet<S3ObjectSummary> treeSet = new TreeSet<>((o1, o2) -> {
    int result = o1.getLastModified().compareTo(o2.getLastModified());
    if(result != 0) {
      //same modified time. Use name to sort
      return result;
    }
    return o1.getKey().compareTo(o2.getKey());
  });

  S3Objects s3ObjectSummaries = S3Objects
    .withPrefix(s3Client, s3ConfigBean.s3Config.bucket, s3ConfigBean.s3Config.commonPrefix);

  // SDC-9413: since the s3ObjectSummaries is in lexical order, we should get all list of files in one api call
  for (S3ObjectSummary s : s3ObjectSummaries) {
    String fullPrefix = s.getKey();
    String remainingPrefix = fullPrefix.substring(s3ConfigBean.s3Config.commonPrefix.length(), fullPrefix.length());
    if (!remainingPrefix.isEmpty()) {
      // remainingPrefix can be empty.
      // If the user manually creates a prefix "myFolder/mySubFolder" in bucket "myBucket" and uploads "myObject",
      // then the first objects returned here are:
      // myFolder/mySubFolder
      // myFolder/mySubFolder/myObject
      //
      // All is good when pipeline is run but preview returns with no data. So we should ignore the empty file as it
      // has no data
      if (pathMatcher.match(s3ConfigBean.s3FileConfig.prefixPattern, remainingPrefix) && isEligible(s, s3Offset)) {
        treeSet.add(s);
      }
      if (treeSet.size() > fetchSize) {
        treeSet.pollLast();
      }
    }
  }

  return new ArrayList<>(treeSet);
}