Java Code Examples for com.amazonaws.services.s3.model.S3ObjectSummary#getSize()

/**
 * This creates a new FileWrapper instance based on the S3ObjectSummary information.
 *
 * @param pathComponentCount The root path component count.
 * @param javaPath           The Path instance for the file.
 * @param entry              The S3ObjectSummary for the file.
 * @return A new FileWrapper instance.
 */
public static FileWrapper createFileWrapperFromFile(int pathComponentCount, Path javaPath, S3ObjectSummary entry) {
	Path pathFileName = javaPath.getFileName();
	if (pathFileName == null) {
		return null;
	}
	return new FileWrapper(
			// Name:
			convertPathToAwsFormat(pathFileName),
			// Path (relative to the `path` parameter):
			convertPathToAwsFormat(javaPath.subpath(pathComponentCount, javaPath.getNameCount())),
			// Directory?
			false,
			// Size:
			entry.getSize(),
			// Last modified (milliseconds):
			entry.getLastModified().getTime()
	);
}
 

/**
 * Calculate the total size of all objects in the indicated bucket
 *
 * @param path to use
 * @return calculated size
 * @throws IOException
 */
@Override
public long calculateSize(Path path) throws IOException {
  long totalBucketSize = 0L;

  if (s3Enabled) {
    String key = pathToKey(path);

    final FileStatus fileStatus =  fs.getFileStatus(path);

    if (fileStatus.isDirectory()) {
      if (!key.isEmpty()) {
        key = key + "/";
      }

      ListObjectsRequest request = new ListObjectsRequest();
      request.setBucketName(uri.getHost());
      request.setPrefix(key);
      request.setMaxKeys(maxKeys);

      if (LOG.isDebugEnabled()) {
        LOG.debug("listStatus: doing listObjects for directory " + key);
      }

      ObjectListing objects = s3.listObjects(request);

      while (true) {
        for (S3ObjectSummary summary : objects.getObjectSummaries()) {
          Path keyPath = keyToPath(summary.getKey()).makeQualified(uri, fs.getWorkingDirectory());

          // Skip over keys that are ourselves and old S3N _$folder$ files
          if (keyPath.equals(path) || summary.getKey().endsWith(S3N_FOLDER_SUFFIX)) {
            if (LOG.isDebugEnabled()) {
              LOG.debug("Ignoring: " + keyPath);
            }
            continue;
          }

          if (!objectRepresentsDirectory(summary.getKey(), summary.getSize())) {
            totalBucketSize += summary.getSize();
          }
        }

        if (objects.isTruncated()) {
          if (LOG.isDebugEnabled()) {
            LOG.debug("listStatus: list truncated - getting next batch");
          }
          objects = s3.listNextBatchOfObjects(objects);
        } else {
          break;
        }
      }
    } else {
      return fileStatus.getLen();
    }
  } else {
    totalBucketSize = fs.getContentSummary(path).getLength();
  }

  return totalBucketSize;
}
 

Map<String, TemplateProp> s3ListTemplate(S3TO s3) {
    String bucket = s3.getBucketName();
    // List the objects in the source directory on S3
    final List<S3ObjectSummary> objectSummaries = S3Utils.listDirectory(s3, bucket, TEMPLATE_ROOT_DIR);
    if (objectSummaries == null) {
        return null;
    }
    Map<String, TemplateProp> tmpltInfos = new HashMap<String, TemplateProp>();
    for (S3ObjectSummary objectSummary : objectSummaries) {
        String key = objectSummary.getKey();
        // String installPath = StringUtils.substringBeforeLast(key,
        // S3Utils.SEPARATOR);
        String uniqueName = determineS3TemplateNameFromKey(key);
        // TODO: isPublic value, where to get?
        TemplateProp tInfo = new TemplateProp(uniqueName, key, objectSummary.getSize(), objectSummary.getSize(), true, false);
        tmpltInfos.put(uniqueName, tInfo);
    }
    return tmpltInfos;

}
 

Map<Long, TemplateProp> s3ListVolume(S3TO s3) {
    String bucket = s3.getBucketName();
    // List the objects in the source directory on S3
    final List<S3ObjectSummary> objectSummaries = S3Utils.listDirectory(s3, bucket, VOLUME_ROOT_DIR);
    if (objectSummaries == null) {
        return null;
    }
    Map<Long, TemplateProp> tmpltInfos = new HashMap<Long, TemplateProp>();
    for (S3ObjectSummary objectSummary : objectSummaries) {
        String key = objectSummary.getKey();
        // String installPath = StringUtils.substringBeforeLast(key,
        // S3Utils.SEPARATOR);
        Long id = determineS3VolumeIdFromKey(key);
        // TODO: how to get volume template name
        TemplateProp tInfo = new TemplateProp(id.toString(), key, objectSummary.getSize(), objectSummary.getSize(), true, false);
        tmpltInfos.put(id, tInfo);
    }
    return tmpltInfos;

}
 

/**
 * Returns the size of the dataset, in bytes. Will be zero if this dataset is a collection or non-existent.
 *
 * @return the size of the dataset
 */
@Override
public long length() {
  // If the summary is already in the cache, return it.
  // It'll have been added by a listDatasets() call on the parent directory.
  S3ObjectSummary objectSummary = objectSummaryCache.getIfPresent(s3uri);
  if (objectSummary != null) {
    return objectSummary.getSize();
  }

  /*
   * Get the metadata directly from S3. This will be expensive.
   * We get punished hard if length() and/or lastModified() is called on a bunch of datasets without
   * listDatasets() first being called on their parent directory.
   *
   * So, is the right thing to do here "getParentDataset().listDatasets()" and then query the cache again?
   * Perhaps, but listDatasets() throws an IOException, and length() and lastModified() do not.
   * We would have to change their signatures and the upstream client code to make it work.
   */
  ObjectMetadata metadata = threddsS3Client.getObjectMetadata(s3uri);
  if (metadata != null) {
    return metadata.getContentLength();
  } else {
    // "this" may be a collection or non-existent. In both cases, we return 0.
    return 0;
  }
}
 

private void initialiseCopyJobsFromListing(
    AmazonS3URI sourceS3Uri,
    final AmazonS3URI targetS3Uri,
    ListObjectsRequest request,
    ObjectListing listing) {
  LOG
      .debug("Found objects to copy {}, for request {}/{}", listing.getObjectSummaries(), request.getBucketName(),
          request.getPrefix());
  List<S3ObjectSummary> objectSummaries = listing.getObjectSummaries();
  for (final S3ObjectSummary s3ObjectSummary : objectSummaries) {
    totalBytesToReplicate += s3ObjectSummary.getSize();
    String fileName = StringUtils.removeStart(s3ObjectSummary.getKey(), sourceS3Uri.getKey());
    final String targetKey = Strings.nullToEmpty(targetS3Uri.getKey()) + fileName;
    CopyObjectRequest copyObjectRequest = new CopyObjectRequest(s3ObjectSummary.getBucketName(),
        s3ObjectSummary.getKey(), targetS3Uri.getBucket(), targetKey);

    if (s3s3CopierOptions.getCannedAcl() != null) {
      copyObjectRequest.withCannedAccessControlList(s3s3CopierOptions.getCannedAcl());
    }

    applyObjectMetadata(copyObjectRequest);

    TransferStateChangeListener stateChangeListener = new BytesTransferStateChangeListener(s3ObjectSummary,
        targetS3Uri, targetKey);
    copyJobRequests.add(new CopyJobRequest(copyObjectRequest, stateChangeListener));
  }
}
 

/**
 * Contructs a StorageObjectSummary object from the S3 equivalent S3ObjectSummary
 *
 * @param objSummary the AWS S3 ObjectSummary object to copy from
 * @return the ObjectSummary object created
 */
public static StorageObjectSummary createFromS3ObjectSummary(S3ObjectSummary objSummary)
{

  return new StorageObjectSummary(
      objSummary.getBucketName(),
      objSummary.getKey(),
      // S3 ETag is not always MD5, but since this code path is only
      // used in skip duplicate files in PUT command, It's not
      // critical to guarantee that it's MD5
      objSummary.getETag(),
      objSummary.getSize()
  );
}
 

/**
 * Create a files status instance from a listing.
 * @param keyPath path to entry
 * @param summary summary from AWS
 * @param blockSize block size to declare
 * @return a status entry
 */
public static COSFileStatus createFileStatus(Path keyPath,
    S3ObjectSummary summary,
    long blockSize) {
  long size = summary.getSize();
  return createFileStatus(keyPath,
      objectRepresentsDirectory(summary.getKey(), size),
      size, summary.getLastModified(), blockSize);
}
 

public static Key make(S3ObjectSummary obj, Futures fs) {
  String fname = obj.getKey();
  Key k = Key.make("s3://" + obj.getBucketName() + "/" + fname);
  long size = obj.getSize();
  Key k2 = Vec.newKey(k);
  new Frame(k).delete_and_lock(null);
  // Insert the top-level FileVec key into the store
  Vec v = new S3FileVec(k2,size);
  DKV.put(k2, v, fs);
  Frame fr = new Frame(k,new String[]{fname},new Vec[]{v});
  fr.update(null);
  fr.unlock(null);
  return k;
}
 

public void processDirectory(AmazonS3 s3, ObjectListing listing, String bucket) throws Exception {

		double time = System.currentTimeMillis();
		ExecutorService executor = Executors.newFixedThreadPool(16);

		int count = 0;

		for (S3ObjectSummary objectSummary : listing.getObjectSummaries()) {
			if ("STANDARD".equalsIgnoreCase(objectSummary.getStorageClass())) {
				long size = objectSummary.getSize();
				logger.debug("*** Processing S3 {}, size: {}", objectSummary.getKey(), size);
				S3Object object = s3.getObject(new GetObjectRequest(bucket, objectSummary.getKey()));

				String bucketName = object.getBucketName();
				String keyName = object.getKey();

				GetObjectTaggingRequest request = new GetObjectTaggingRequest(bucketName, keyName);
				GetObjectTaggingResult result = s3.getObjectTagging(request);
				List<Tag> tags = result.getTagSet();
				String type = null;
				String name = null;

				if (tags.isEmpty()) {
					object.close();
					logger.warn("Error, S3 object: {} has no tags", keyName);
				} else {
					for (Tag tag : tags) {
						String key = tag.getKey();
						String value = tag.getValue();

						if (key.equals("type")) {
							type = value;
						}

						if (key.equals("name")) {
							name = value;
						}
					}

					if (name == null) {
						object.close();
						throw new Exception("Error: " + keyName + " is missing a name tag");
					}
					if (name.contains(" ")) {
						object.close();
						throw new Exception("Error: " + keyName + " has a name attribute with a space in it");
					}
					if (type == null) {
						object.close();
						throw new Exception("Error: " + keyName + " has no type tag");
					}

					if (!name.startsWith("$"))
						name = "$" + name;

					// The runnable will call object.close();
					Runnable w = new AwsWorker(type, name, object, size);
					executor.execute(w);

					count++;
				}
			}
		}
		executor.shutdown();
		executor.awaitTermination(Long.MAX_VALUE, TimeUnit.NANOSECONDS);

		time = System.currentTimeMillis() - time;
		time = time / 60000;
		logger.info("Initialized all {} S3 objects in {} minutes", count, time);
	}
 

/**
 * Validates registered S3 files per list of expected storage files. The validation ignores (does not fail) when detecting unregistered zero byte S3 files.
 *
 * @param expectedStorageFiles the list of expected S3 files represented by storage files
 * @param s3ObjectSummaries the list of actual S3 files represented by S3 object summaries
 * @param storageName the storage name
 * @param businessObjectDataKey the business object data key
 */
public void validateRegisteredS3Files(List<StorageFile> expectedStorageFiles, List<S3ObjectSummary> s3ObjectSummaries, String storageName,
    BusinessObjectDataKey businessObjectDataKey)
{
    // Get a set of actual S3 file paths.
    Set<String> actualS3FilePaths = new HashSet<>(getFilePathsFromS3ObjectSummaries(s3ObjectSummaries));

    // Validate existence and file size for all expected files.
    for (StorageFile expectedStorageFile : expectedStorageFiles)
    {
        if (!actualS3FilePaths.contains(expectedStorageFile.getFilePath()))
        {
            throw new ObjectNotFoundException(
                String.format("Registered file \"%s\" does not exist in \"%s\" storage.", expectedStorageFile.getFilePath(), storageName));
        }
    }

    // Get a set of expected file paths.
    Set<String> expectedFilePaths = new HashSet<>(getFilePathsFromStorageFiles(expectedStorageFiles));

    // Create a JSON representation of the business object data key.
    String businessObjectDataKeyAsJson = jsonHelper.objectToJson(businessObjectDataKey);

    // Validate that no other files in S3 bucket except for expected storage files have the same S3 key prefix.
    // Please note that this validation ignores (does not fail on) any unregistered zero byte S3 files.
    for (S3ObjectSummary s3ObjectSummary : s3ObjectSummaries)
    {
        if (!expectedFilePaths.contains(s3ObjectSummary.getKey()))
        {
            // Ignore unregistered zero byte S3 files.
            if (s3ObjectSummary.getSize() == 0)
            {
                LOGGER.info("Ignoring unregistered zero byte S3 file. s3Key=\"{}\" storageName=\"{}\" businessObjectDataKey={}", s3ObjectSummary.getKey(),
                    storageName, businessObjectDataKeyAsJson);
            }
            else
            {
                throw new IllegalStateException(String
                    .format("Found unregistered non-empty S3 file \"%s\" in \"%s\" storage. Business object data {%s}", s3ObjectSummary.getKey(),
                        storageName, businessObjectDataHelper.businessObjectDataKeyToString(businessObjectDataKey)));
            }
        }
    }
}
 

@Override
public List<S3ObjectSummary> listDirectory(final S3FileTransferRequestParamsDto params, boolean ignoreZeroByteDirectoryMarkers)
{
    Assert.isTrue(!isRootKeyPrefix(params.getS3KeyPrefix()), "Listing of S3 objects from root directory is not allowed.");

    AmazonS3Client s3Client = getAmazonS3(params);
    List<S3ObjectSummary> s3ObjectSummaries = new ArrayList<>();

    try
    {
        ListObjectsRequest listObjectsRequest = new ListObjectsRequest().withBucketName(params.getS3BucketName()).withPrefix(params.getS3KeyPrefix());
        ObjectListing objectListing;

        do
        {
            objectListing = s3Operations.listObjects(listObjectsRequest, s3Client);

            for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries())
            {
                // Ignore 0 byte objects that represent S3 directories.
                if (!(ignoreZeroByteDirectoryMarkers && objectSummary.getKey().endsWith("/") && objectSummary.getSize() == 0L))
                {
                    s3ObjectSummaries.add(objectSummary);
                }
            }

            listObjectsRequest.setMarker(objectListing.getNextMarker());
        }
        while (objectListing.isTruncated());
    }
    catch (AmazonS3Exception amazonS3Exception)
    {
        if (S3Operations.ERROR_CODE_NO_SUCH_BUCKET.equals(amazonS3Exception.getErrorCode()))
        {
            throw new IllegalArgumentException("The specified bucket '" + params.getS3BucketName() + "' does not exist.", amazonS3Exception);
        }
        throw new IllegalStateException("Error accessing S3", amazonS3Exception);
    }
    catch (AmazonClientException e)
    {
        throw new IllegalStateException(String
            .format("Failed to list keys with prefix \"%s\" from bucket \"%s\". Reason: %s", params.getS3KeyPrefix(), params.getS3BucketName(),
                e.getMessage()), e);
    }
    finally
    {
        // Shutdown the AmazonS3Client instance to release resources.
        s3Client.shutdown();
    }

    return s3ObjectSummaries;
}
 

public void processDirectory(AmazonS3Client s3, ObjectListing listing, String bucket) throws Exception {
	for (S3ObjectSummary objectSummary : listing.getObjectSummaries()) {
		long size = objectSummary.getSize();
		logger.info("*** Processing S3 {}, size: {}",objectSummary.getKey(),size);
		S3Object object = s3.getObject(new GetObjectRequest(bucket, objectSummary.getKey()));

		String bucketName = object.getBucketName();
		String keyName = object.getKey();

		GetObjectTaggingRequest request = new GetObjectTaggingRequest(bucketName, keyName);
		GetObjectTaggingResult result = s3.getObjectTagging(request);
		List<Tag> tags = result.getTagSet();
		String type = null;
		String name = null;

		if (tags.isEmpty()) {
			System.err.println("Error: " + keyName + " has no tags");
		} else {
			for (Tag tag : tags) {
				String key = tag.getKey();
				String value = tag.getValue();

				if (key.equals("type")) {
					type = value;
				}

				if (key.equals("name")) {
					name = value;
				}
			}

			if (name == null)
				throw new Exception("Error: " + keyName + " is missing a name tag");
			if (name.contains(" "))
				throw new Exception("Error: " + keyName + " has a name attribute with a space in it");
			if (type == null)
				throw new Exception("Error: " + keyName + " has no type tag");

			if (!name.startsWith("$"))
				name = "$" + name;

			readData(type, name, object, size);
		}
	}
}
 

public static void main(String[] args) throws Exception {
    System.out.println("===========================================");
    System.out.println("Welcome to the AWS Java SDK!");
    System.out.println("===========================================");

    init();

    try {
        /*
         * The Amazon EC2 client allows you to easily launch and configure
         * computing capacity in AWS datacenters.
         *
         * In this sample, we use the EC2 client to list the availability zones
         * in a region, and then list the instances running in those zones.
         */
        DescribeAvailabilityZonesResult availabilityZonesResult = ec2.describeAvailabilityZones();
        List<AvailabilityZone> availabilityZones = availabilityZonesResult.getAvailabilityZones();
        System.out.println("You have access to " + availabilityZones.size() + " availability zones:");
        for (AvailabilityZone zone : availabilityZones) {
            System.out.println(" - " + zone.getZoneName() + " (" + zone.getRegionName() + ")");
        }

        DescribeInstancesResult describeInstancesResult = ec2.describeInstances();
        Set<Instance> instances = new HashSet<Instance>();
        for (Reservation reservation : describeInstancesResult.getReservations()) {
            instances.addAll(reservation.getInstances());
        }

        System.out.println("You have " + instances.size() + " Amazon EC2 instance(s) running.");


        /*
         * The Amazon S3 client allows you to manage and configure buckets
         * and to upload and download data.
         *
         * In this sample, we use the S3 client to list all the buckets in
         * your account, and then iterate over the object metadata for all
         * objects in one bucket to calculate the total object count and
         * space usage for that one bucket. Note that this sample only
         * retrieves the object's metadata and doesn't actually download the
         * object's content.
         *
         * In addition to the low-level Amazon S3 client in the SDK, there
         * is also a high-level TransferManager API that provides
         * asynchronous management of uploads and downloads with an easy to
         * use API:
         *   http://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/services/s3/transfer/TransferManager.html
         */
        List<Bucket> buckets = s3.listBuckets();
        System.out.println("You have " + buckets.size() + " Amazon S3 bucket(s).");

        if (buckets.size() > 0) {
            Bucket bucket = buckets.get(0);

            long totalSize  = 0;
            long totalItems = 0;
            /*
             * The S3Objects and S3Versions classes provide convenient APIs
             * for iterating over the contents of your buckets, without
             * having to manually deal with response pagination.
             */
            for (S3ObjectSummary objectSummary : S3Objects.inBucket(s3, bucket.getName())) {
                totalSize += objectSummary.getSize();
                totalItems++;
            }

            System.out.println("The bucket '" + bucket.getName() + "' contains "+ totalItems + " objects "
                    + "with a total size of " + totalSize + " bytes.");
        }
    } catch (AmazonServiceException ase) {
        /*
         * AmazonServiceExceptions represent an error response from an AWS
         * services, i.e. your request made it to AWS, but the AWS service
         * either found it invalid or encountered an error trying to execute
         * it.
         */
        System.out.println("Error Message:    " + ase.getMessage());
        System.out.println("HTTP Status Code: " + ase.getStatusCode());
        System.out.println("AWS Error Code:   " + ase.getErrorCode());
        System.out.println("Error Type:       " + ase.getErrorType());
        System.out.println("Request ID:       " + ase.getRequestId());
    } catch (AmazonClientException ace) {
        /*
         * AmazonClientExceptions represent an error that occurred inside
         * the client on the local host, either while trying to send the
         * request to AWS or interpret the response. For example, if no
         * network connection is available, the client won't be able to
         * connect to AWS to execute a request and will throw an
         * AmazonClientException.
         */
        System.out.println("Error Message: " + ace.getMessage());
    }
}
 

private void scanBucket(Set<TocInfo> toc, Queue<TocInfo> tocQueue) throws Exception {
	
	ListObjectsRequest listRequest = new ListObjectsRequest();
	listRequest.setBucketName(s3BucketName);
	// listRequest.setGeneralProgressListener(this);
	listRequest.setMaxKeys(1000);
	
	String nextMarker = null;
	ObjectListing objectListing = null;
	
	while(true) {
		
		objectListing = s3Client.listObjects(listRequest);
		
		List<S3ObjectSummary> objectSummaries = objectListing.getObjectSummaries();
		
		for (S3ObjectSummary objSummary : objectSummaries) {
			String key = objSummary.getKey();
			
			TocInfo tocInfo = new TocInfo(key, objSummary.getSize());
			
			// is it a "dir/" ?
			if (key.lastIndexOf("/") == (key.length() - 1)) {
				tocInfo.isDirectory = true;
			} else {
				tocInfo.isDirectory = false;
			}
			
			toc.add(tocInfo);
			tocQueue.add(tocInfo);
			tocInfosGenerated++; // increment for logging

		}
		
		// for pagination
		nextMarker = objectListing.getNextMarker();
		if (nextMarker == null) {
			break;
		} else {
			listRequest.setMarker(nextMarker);
			logger.debug("scanBucket() nextMarker we will request listing for => " + nextMarker);
		}
	}
	
}
 

private S3FileStatus[] listBucketContent(final Path f, final S3BucketObjectPair bop) throws IOException {

		ObjectListing listing = null;
		final List<S3FileStatus> resultList = new ArrayList<S3FileStatus>();

		final int depth = (bop.hasObject() ? getDepth(bop.getObject()) + 1 : 0);

		while (true) {

			if (listing == null) {
				if (bop.hasObject()) {
					listing = this.s3Client.listObjects(bop.getBucket(), bop.getObject());
				} else {
					listing = this.s3Client.listObjects(bop.getBucket());
				}
			} else {
				listing = this.s3Client.listNextBatchOfObjects(listing);
			}

			final List<S3ObjectSummary> list = listing.getObjectSummaries();
			final Iterator<S3ObjectSummary> it = list.iterator();
			while (it.hasNext()) {

				final S3ObjectSummary os = it.next();
				String key = os.getKey();

				final int childDepth = getDepth(os.getKey());

				if (childDepth != depth) {
					continue;
				}

				// Remove the prefix
				if (bop.hasObject()) {
					if (key.startsWith(bop.getObject())) {
						key = key.substring(bop.getObject().length());
					}

					// This has been the prefix itself
					if (key.isEmpty()) {
						continue;
					}
				}

				final long modificationDate = dateToLong(os.getLastModified());

				S3FileStatus fileStatus;
				if (objectRepresentsDirectory(os)) {
					fileStatus = new S3FileStatus(extendPath(f, key), 0, true, modificationDate, 0L);
				} else {
					fileStatus = new S3FileStatus(extendPath(f, key), os.getSize(), false, modificationDate, 0L);
				}

				resultList.add(fileStatus);
			}

			if (!listing.isTruncated()) {
				break;
			}
		}

		/*
		 * System.out.println("---- RETURN CONTENT ----");
		 * for (final FileStatus entry : resultList) {
		 * System.out.println(entry.getPath());
		 * }
		 * System.out.println("------------------------");
		 */

		return resultList.toArray(new S3FileStatus[0]);

	}