Java Code Examples for com.amazonaws.services.s3.model.ObjectListing#getCommonPrefixes()

private List<String> getCommonPrefixFromListing(ObjectListing listing, String prefix) {
    List<String> results = new ArrayList<String>(100);

    for (String commonPrefix : listing.getCommonPrefixes()) {
        final String dirname;
        
        /* remove prefix and trailing delimiter */
        dirname = commonPrefix.substring(prefix.length(), 
                                         commonPrefix.length() - DELIMITER.length());
        if (dirname.length() == 0 || dirname.contains(DELIMITER)) {
            log.error("Error parsing S3 object prefix.  Prefix: " + commonPrefix);
            continue;
        }
        results.add(dirname);
    }
    
    return results;
}
 

private void getObjectsFromNonRootFolder( String key, String bucketName, List<String> childrenList, String realKey ) {
  //Getting files/folders in a folder/bucket
  String prefix = key.isEmpty() || key.endsWith( DELIMITER ) ? key : key + DELIMITER;
  ListObjectsRequest listObjectsRequest = new ListObjectsRequest()
    .withBucketName( bucketName )
    .withPrefix( prefix )
    .withDelimiter( DELIMITER );

  ObjectListing ol = fileSystem.getS3Client().listObjects( listObjectsRequest );

  ArrayList<S3ObjectSummary> allSummaries = new ArrayList<>( ol.getObjectSummaries() );
  ArrayList<String> allCommonPrefixes = new ArrayList<>( ol.getCommonPrefixes() );

  // get full list
  while ( ol.isTruncated() ) {
    ol = fileSystem.getS3Client().listNextBatchOfObjects( ol );
    allSummaries.addAll( ol.getObjectSummaries() );
    allCommonPrefixes.addAll( ol.getCommonPrefixes() );
  }

  for ( S3ObjectSummary s3os : allSummaries ) {
    if ( !s3os.getKey().equals( realKey ) ) {
      childrenList.add( s3os.getKey().substring( prefix.length() ) );
    }
  }

  for ( String commonPrefix : allCommonPrefixes ) {
    if ( !commonPrefix.equals( realKey ) ) {
      childrenList.add( commonPrefix.substring( prefix.length() ) );
    }
  }
}
 

@Test
public void testBlobListRecursive() throws Exception {
    ObjectListing listing = client.listObjects(containerName);
    assertThat(listing.getObjectSummaries()).isEmpty();

    ObjectMetadata metadata = new ObjectMetadata();
    metadata.setContentLength(BYTE_SOURCE.size());
    client.putObject(containerName, "prefix/blob1",
            BYTE_SOURCE.openStream(), metadata);
    client.putObject(containerName, "prefix/blob2",
            BYTE_SOURCE.openStream(), metadata);

    ImmutableList.Builder<String> builder = ImmutableList.builder();
    listing = client.listObjects(new ListObjectsRequest()
            .withBucketName(containerName)
            .withDelimiter("/"));
    assertThat(listing.getObjectSummaries()).isEmpty();
    for (String prefix : listing.getCommonPrefixes()) {
        builder.add(prefix);
    }
    assertThat(builder.build()).containsOnly("prefix/");

    builder = ImmutableList.builder();
    listing = client.listObjects(containerName);
    for (S3ObjectSummary summary : listing.getObjectSummaries()) {
        builder.add(summary.getKey());
    }
    assertThat(builder.build()).containsOnly("prefix/blob1",
            "prefix/blob2");
    assertThat(listing.getCommonPrefixes()).isEmpty();
}
 

/**
 * Searches for matching keys progressively. This means that instead of retrieving all
 * keys given a prefix, it goes down one level at a time and filters out all
 * non-matching results. This avoids a lot of unused requests results. WARNING: This
 * method does not truncate results. Therefore all matching resources will be returned
 * regardless of the truncation.
 * @param bucketName name of the bucket
 * @param resources retrieved resources
 * @param prefix bucket prefix
 * @param keyPattern pattern for key
 */
private void findProgressivelyWithPartialMatch(String bucketName,
		Set<Resource> resources, String prefix, String keyPattern) {
	ListObjectsRequest listObjectsRequest = new ListObjectsRequest()
			.withBucketName(bucketName).withDelimiter("/").withPrefix(prefix);
	ObjectListing objectListing = null;

	do {
		if (objectListing == null) {
			objectListing = this.amazonS3.listObjects(listObjectsRequest);
		}
		else {
			objectListing = this.amazonS3.listNextBatchOfObjects(objectListing);
		}

		Set<Resource> newResources = getResourcesFromObjectSummaries(bucketName,
				keyPattern, objectListing.getObjectSummaries());
		if (!newResources.isEmpty()) {
			resources.addAll(newResources);
		}

		for (String commonPrefix : objectListing.getCommonPrefixes()) {
			if (isKeyPathMatchesPartially(keyPattern, commonPrefix)) {
				findPathMatchingKeyInBucket(bucketName, resources, commonPrefix,
						keyPattern);
			}
		}
	}
	while (objectListing.isTruncated());
}
 

public cfData execute( cfSession _session, cfArgStructData argStruct ) throws cfmRunTimeException {

		AmazonKey amazonKey = getAmazonKey( _session, argStruct );
		AmazonS3 s3Client = getAmazonS3( amazonKey );

		String bucket = getNamedStringParam( argStruct, "bucket", null );
		String prefix = getNamedStringParam( argStruct, "prefix", "" );

		if ( bucket == null )
			throwException( _session, "Please specify a bucket" );

		try {
			// Create the results
			cfQueryResultData qD = new cfQueryResultData( new String[] { "key", "size", "modified", "etag" }, null );
			qD.setQuerySource( "AmazonS3." + amazonKey.getDataSource() );

			ListObjectsRequest listObjectsRequest = new ListObjectsRequest()
					.withBucketName( bucket )
					.withDelimiter( "/" )
					.withPrefix( prefix );
			ObjectListing objectListing;

			do {
				objectListing = s3Client.listObjects( listObjectsRequest );

				java.util.List<String> prefixes = objectListing.getCommonPrefixes();

				// first add the prefixes
				for ( String nextPrefix : prefixes ) {
					qD.addRow( 1 );
					qD.setCurrentRow( qD.getSize() );

					qD.setCell( 1, new cfStringData( nextPrefix ) );
					qD.setCell( 2, new cfNumberData( 0 ) );
					qD.setCell( 3, cfNullData.NULL );
					qD.setCell( 4, cfNullData.NULL );

				}

				for ( S3ObjectSummary objectSummary : objectListing.getObjectSummaries() ) {

					// don't include the prefix being listed
					if ( objectSummary.getKey().equals( prefix ) ) {
						continue;
					}
					qD.addRow( 1 );
					qD.setCurrentRow( qD.getSize() );

					qD.setCell( 1, new cfStringData( objectSummary.getKey() ) );
					qD.setCell( 2, new cfNumberData( objectSummary.getSize() ) );
					qD.setCell( 3, new cfDateData( objectSummary.getLastModified() ) );
					qD.setCell( 4, new cfStringData( objectSummary.getETag() ) );
				}

				listObjectsRequest.setMarker( objectListing.getNextMarker() );
			} while ( objectListing.isTruncated() );

			return qD;
		} catch ( Exception e ) {
			throwException( _session, "AmazonS3: " + e.getMessage() );
			return cfBooleanData.FALSE;
		}
	}
 

@Override
public void prepare(Object configurationObject) {

  streamsConfiguration = StreamsConfigurator.detectConfiguration();

  lineReaderUtil = LineReadWriteUtil.getInstance(s3ReaderConfiguration);

  // Connect to S3
  synchronized (this) {
    // Create the credentials Object
    AWSCredentials credentials = new BasicAWSCredentials(s3ReaderConfiguration.getKey(), s3ReaderConfiguration.getSecretKey());

    ClientConfiguration clientConfig = new ClientConfiguration();
    clientConfig.setProtocol(Protocol.valueOf(s3ReaderConfiguration.getProtocol().toString()));

    // We do not want path style access
    S3ClientOptions clientOptions = new S3ClientOptions();
    clientOptions.setPathStyleAccess(false);

    this.amazonS3Client = new AmazonS3Client(credentials, clientConfig);
    if (StringUtils.isNotEmpty(s3ReaderConfiguration.getRegion())) {
      this.amazonS3Client.setRegion(Region.getRegion(Regions.fromName(s3ReaderConfiguration.getRegion())));
    }
    this.amazonS3Client.setS3ClientOptions(clientOptions);
  }

  final ListObjectsRequest request = new ListObjectsRequest()
      .withBucketName(this.s3ReaderConfiguration.getBucket())
      .withPrefix(s3ReaderConfiguration.getReaderPath())
      .withMaxKeys(500);


  ObjectListing listing = this.amazonS3Client.listObjects(request);

  this.files = new ArrayList<>();

  /*
   * If you can list files that are in this path, then you must be dealing with a directory
   * if you cannot list files that are in this path, then you are most likely dealing with
   * a simple file.
   */
  boolean hasCommonPrefixes = listing.getCommonPrefixes().size() > 0;
  boolean hasObjectSummaries = listing.getObjectSummaries().size() > 0;

  if (hasCommonPrefixes || hasObjectSummaries) {
    // Handle the 'directory' use case
    do {
      if (hasCommonPrefixes) {
        for (String file : listing.getCommonPrefixes()) {
          this.files.add(file);
        }
      } else {
        for (final S3ObjectSummary objectSummary : listing.getObjectSummaries()) {
          this.files.add(objectSummary.getKey());
        }
      }

      // get the next batch.
      listing = this.amazonS3Client.listNextBatchOfObjects(listing);
    }
    while (listing.isTruncated());
  } else {
    // handle the single file use-case
    this.files.add(s3ReaderConfiguration.getReaderPath());
  }

  if (this.files.size() <= 0) {
    LOGGER.error("There are no files to read");
  }

  this.persistQueue = Queues.synchronizedQueue(new LinkedBlockingQueue<StreamsDatum>(streamsConfiguration.getQueueSize().intValue()));
  this.executor = Executors.newSingleThreadExecutor();
}