Java Code Examples for com.amazonaws.services.s3.model.ListObjectsV2Result#getObjectSummaries()

/**
 * Calls DescribeDBInstances on the AWS RDS Client returning all DB Instances that match the supplied predicate and attempting
 * to push down certain predicates (namely queries for specific DB Instance) to EC2.
 *
 * @See TableProvider
 */
@Override
public void readWithConstraint(BlockSpiller spiller, ReadRecordsRequest recordsRequest, QueryStatusChecker queryStatusChecker)
{
    ValueSet bucketConstraint = recordsRequest.getConstraints().getSummary().get("bucket_name");
    String bucket;
    if (bucketConstraint != null && bucketConstraint.isSingleValue()) {
        bucket = bucketConstraint.getSingleValue().toString();
    }
    else {
        throw new IllegalArgumentException("Queries against the objects table must filter on a single bucket " +
                "(e.g. where bucket_name='my_bucket'.");
    }

    ListObjectsV2Request req = new ListObjectsV2Request().withBucketName(bucket).withMaxKeys(MAX_KEYS);
    ListObjectsV2Result result;
    do {
        result = amazonS3.listObjectsV2(req);
        for (S3ObjectSummary objectSummary : result.getObjectSummaries()) {
            toRow(objectSummary, spiller);
        }
        req.setContinuationToken(result.getNextContinuationToken());
    }
    while (result.isTruncated() && queryStatusChecker.isQueryRunning());
}
 

/**
 * get the compose list from S3 server
 */
@Override
public List<String> getComponentList(String productName, String version) throws DataException {
   List<String> componentList = new ArrayList<String>();
   String filePathPrefix = S3Utils.genProductVersionS3Path(productName, version);
   ListObjectsV2Result result =
         s3Client.getS3Client().listObjectsV2(config.getBucketName(), filePathPrefix);
   if (result == null) {
      throw new DataException("Can't find S3 resource from " + productName + "\\" + version);
   }
   List<S3ObjectSummary> objects = result.getObjectSummaries();
   if (objects == null || objects.size() < 1) {
      throw new DataException("S3 Component list is empty.");
   }
   for (S3ObjectSummary s3os : objects) {
      String resultKey =
            (s3os.getKey().replace(filePathPrefix, "")).split(ConstantsChar.BACKSLASH)[0];
      if (!componentList.contains(resultKey) && (!resultKey.endsWith(ConstantsFile.FILE_TPYE_JSON))) {
         componentList.add(resultKey);
      }
   }
   return componentList;
}
 

/**
  * get one product's all available versions
  */
@Override
public List<String> getVersionList(String productName) throws DataException {
    String basePath = S3Utils.S3_L10N_BUNDLES_PATH+productName +  ConstantsChar.BACKSLASH;
    ListObjectsV2Result versionListResult = s3Client.getS3Client().listObjectsV2(config.getBucketName(),basePath);
    
    if(versionListResult != null) {
        List<S3ObjectSummary> versionListSummary = versionListResult.getObjectSummaries();
        Set<String> versionset = new HashSet<>();
        for (S3ObjectSummary s3productName : versionListSummary) {
            versionset.add(s3productName.getKey().replace(basePath, "").split(ConstantsChar.BACKSLASH)[0]);
          }
        List<String> result = new ArrayList<>();
        for(String version: versionset) {
            result.add(version);
        }
      return result;
      
    }else {
     throw new DataException(productName + " no available version in s3");
    }
}
 

public static void main(String[] args) {
    final String USAGE = "\n" +
            "To run this example, supply the name of a bucket to list!\n" +
            "\n" +
            "Ex: ListObjects <bucket-name>\n";

    if (args.length < 1) {
        System.out.println(USAGE);
        System.exit(1);
    }

    String bucket_name = args[0];

    System.out.format("Objects in S3 bucket %s:\n", bucket_name);
    final AmazonS3 s3 = AmazonS3ClientBuilder.standard().withRegion(Regions.DEFAULT_REGION).build();
    ListObjectsV2Result result = s3.listObjectsV2(bucket_name);
    List<S3ObjectSummary> objects = result.getObjectSummaries();
    for (S3ObjectSummary os : objects) {
        System.out.println("* " + os.getKey());
    }
}
 

/**
 * get locale list from S3 server
 */
@Override
public List<String> getLocaleList(String productName, String version) throws DataException {
   List<String> localeList = new ArrayList<String>();
   String filePathPrefix = S3Utils.genProductVersionS3Path(productName, version);
   ListObjectsV2Result result =
         s3Client.getS3Client().listObjectsV2(config.getBucketName(), S3Utils.S3_L10N_BUNDLES_PATH);
   if (result == null) {
      throw new DataException("Can't find S3 resource from " + productName + "\\" + version);
   }
   List<S3ObjectSummary> objects = result.getObjectSummaries();
   if (objects == null || objects.size() < 1) {
      throw new DataException("S3 Component list is empty.");
   }
   for (S3ObjectSummary s3os : objects) {
      String s3obKey = s3os.getKey().replace(filePathPrefix, "");
      if((!s3obKey.startsWith(ConstantsFile.CREATION_INFO)) && (!s3obKey.startsWith(ConstantsFile.VERSION_FILE))) {
         String resultKey =s3obKey.split(ConstantsChar.BACKSLASH)[1];
         String localeKey = S3Utils.getLocaleByFileName(resultKey);
         if (localeKey != null && !localeList.contains(localeKey)) {
            localeList.add(localeKey);
         }
         
      }
      
      
   }
   return localeList;
}
 

@Override
public List<FileInfo> listDirectoriesInDirectory(Path dirPath) {
    String prefix = dirPath.toString();
    if (!prefix.isEmpty()) {
        prefix += File.separator;
    }

    ListObjectsV2Result result = s3.listObjectsV2(bucketName, prefix);

    Set<String> seenDirectoryNames = Sets.newHashSet();
    ImmutableList.Builder<FileInfo> fileInfos = ImmutableList.builder();
    for (S3ObjectSummary objectSummary : result.getObjectSummaries()) {
        String key = objectSummary.getKey().substring(prefix.length());
        if (key.endsWith(File.separator) || !key.contains(File.separator)) {
            continue;
        }
        key = key.substring(0, key.lastIndexOf(File.separator));
        if (key.contains(File.separator) || seenDirectoryNames.contains(key)) {
            continue;
        }

        seenDirectoryNames.add(key);
        fileInfos.add(new FileInfo.Builder()
                .name(key)
                .size(objectSummary.getSize())
                .lastModifiedTime(objectSummary.getLastModified().toInstant())
                .build());
    }
    return fileInfos.build();
}
 

@Override
public List<FileInfo> listFilesInDirectory(Path dirPath) {
    String prefix = dirPath.toString();
    if (!prefix.isEmpty()) {
        prefix += File.separator;
    }

    ListObjectsV2Result result = s3.listObjectsV2(bucketName, prefix);

    List<FileInfo> fileInfos = Lists.newArrayList();
    for (S3ObjectSummary objectSummary : result.getObjectSummaries()) {
        String key = objectSummary.getKey().substring(prefix.length());
        if (key.contains(File.separator)) {
            continue;
        }

        fileInfos.add(new FileInfo.Builder()
                .name(key)
                .size(objectSummary.getSize())
                .lastModifiedTime(objectSummary.getLastModified().toInstant())
                .build());
    }

    Comparator<String> comparator = new NaturalFilenameComparator();
    fileInfos.sort((FileInfo f1, FileInfo f2) -> comparator.compare(f1.getName(), f2.getName()));
    return ImmutableList.copyOf(fileInfos);
}
 

public void deleteBucket(String bucketName) {
    logger.trace("Deleting S3 objects under {} bucket...", bucketName);
    ListObjectsV2Result listObjectsV2Result = amazonS3.listObjectsV2(bucketName);
    for (S3ObjectSummary objectSummary : listObjectsV2Result.getObjectSummaries()) {
        logger.info("Deleting S3 object: {}", objectSummary.getKey());
        amazonS3.deleteObject(bucketName, objectSummary.getKey());
    }
    logger.info("Deleting S3 bucket: {}", bucketName);
    amazonS3.deleteBucket(bucketName);
}
 

@AfterClass
public static void cleanUpResources() {
    AmazonS3 amazonS3 = SpringCloudAwsTestUtil.amazonS3();
    ListObjectsV2Result listObjectsV2Result = amazonS3.listObjectsV2(bucketName);
    for (S3ObjectSummary objectSummary : listObjectsV2Result.getObjectSummaries()) {
        amazonS3.deleteObject(bucketName, objectSummary.getKey());
    }
    amazonS3.deleteBucket(bucketName);

    new File(testFileToDownload).delete();
    new File(testFileToUpload).delete();
    similarNameFiles.forEach(File::delete);
}
 

private ExpandedGlob expandGlob(S3ResourceId glob) {
  // The S3 API can list objects, filtered by prefix, but not by wildcard.
  // Here, we find the longest prefix without wildcard "*",
  // then filter the results with a regex.
  checkArgument(glob.isWildcard(), "isWildcard");
  String keyPrefix = glob.getKeyNonWildcardPrefix();
  Pattern wildcardRegexp = Pattern.compile(wildcardToRegexp(glob.getKey()));

  LOG.debug(
      "expanding bucket {}, prefix {}, against pattern {}",
      glob.getBucket(),
      keyPrefix,
      wildcardRegexp.toString());

  ImmutableList.Builder<S3ResourceId> expandedPaths = ImmutableList.builder();
  String continuationToken = null;

  do {
    ListObjectsV2Request request =
        new ListObjectsV2Request()
            .withBucketName(glob.getBucket())
            .withPrefix(keyPrefix)
            .withContinuationToken(continuationToken);
    ListObjectsV2Result result;
    try {
      result = amazonS3.get().listObjectsV2(request);
    } catch (AmazonClientException e) {
      return ExpandedGlob.create(glob, new IOException(e));
    }
    continuationToken = result.getNextContinuationToken();

    for (S3ObjectSummary objectSummary : result.getObjectSummaries()) {
      // Filter against regex.
      if (wildcardRegexp.matcher(objectSummary.getKey()).matches()) {
        S3ResourceId expandedPath =
            S3ResourceId.fromComponents(objectSummary.getBucketName(), objectSummary.getKey())
                .withSize(objectSummary.getSize())
                .withLastModified(objectSummary.getLastModified());
        LOG.debug("Expanded S3 object path {}", expandedPath);
        expandedPaths.add(expandedPath);
      }
    }
  } while (continuationToken != null);

  return ExpandedGlob.create(glob, expandedPaths.build());
}
 

private void assertS3Contents(List<Map<String, Object>> expected)
        throws IOException
{
    ListObjectsV2Request req = new ListObjectsV2Request().withBucketName(s3Bucket).withPrefix(s3ParentKey);
    ListObjectsV2Result result;
    List<String> lines = new ArrayList<>();
    do {
        result = s3Client.listObjectsV2(req);

        for (S3ObjectSummary objectSummary : result.getObjectSummaries()) {
            if (objectSummary.getKey().endsWith("_part_00")) {
                try (BufferedReader reader =
                        new BufferedReader(
                                new InputStreamReader(
                                        s3Client.getObject(
                                                objectSummary.getBucketName(),
                                                objectSummary.getKey()).getObjectContent()))) {
                    lines.addAll(reader.lines().collect(Collectors.toList()));
                }
            }
            else {
                assertThat(objectSummary.getKey(), endsWith("_manifest"));
            }

            try {
                s3Client.deleteObject(objectSummary.getBucketName(), objectSummary.getKey());
            }
            catch (Exception e) {
                logger.warn("Failed to delete S3 object: bucket={}, key={}", s3Bucket, objectSummary.getKey(), e);
            }
        }
        req.setContinuationToken(result.getNextContinuationToken());
    } while (result.isTruncated());

    List<ImmutableMap<String, ? extends Serializable>> actual =
            lines.stream()
                    .map(
                            l -> {
                                String[] values = l.split("\\|");
                                assertThat(values.length, is(3));
                                return ImmutableMap.of(
                                        "id", Integer.valueOf(values[0]),
                                        "name", values[1],
                                        "score", Float.valueOf(values[2]));
                            }
                    )
                    .sorted((o1, o2) -> ((Integer)o1.get("id")) - ((Integer) o2.get("id")))
                    .collect(Collectors.toList());
    assertThat(actual, is(expected));
}