Java Code Examples for com.amazonaws.services.s3.model.S3Object#close()

/**
 * Check if the MD5s of manifest.json and manifest.checksum equal
 * if so, pull out the manifest file and map it into a POJO
 * @return inventoryManifestStorage InventoryManifest, which stores all the elements of the manifest.json file
 */
public InventoryManifest getInventoryManifest() throws Exception {
    // Get manifest.json and transfer it to String
    GetObjectRequest requestJson = new GetObjectRequest(bucketName, bucketKeyJson);
    S3Object jsonObject = s3Client.getObject(requestJson);
    String jsonFile = inputStreamToString(jsonObject.getObjectContent());
    jsonObject.close();

    // Get manifest.checksum and transfer it to String with no whitespace
    GetObjectRequest requestChecksum = new GetObjectRequest(bucketName, bucketKeyChecksum);
    S3Object checksumObject = s3Client.getObject(requestChecksum);
    String expectedChecksum = inputStreamToString(checksumObject.getObjectContent())
            .replaceAll("\\s","");
    checksumObject.close();

    // Compare manifest.json and manifest.checksum's MD5 value
    String actualChecksum = DigestUtils.md5Hex(jsonFile);
    if (!actualChecksum.equals(expectedChecksum)) {
        throw new ChecksumMismatchException (expectedChecksum, actualChecksum);
    }

    return mapper.readValue(jsonFile, InventoryManifest.class);
}
 

@Override
public InputStream _readFile(String folder, String filename) throws FileNotFoundException {
	final S3Object file = s3Client.getObject(bucketName, filename);
	InputStream is = file.getObjectContent();
	FilterInputStream fos = new FilterInputStream(is) {
		@Override
		public void close() throws IOException {
			super.close();
			file.close();
		}
	};

	return fos;
}
 

public void processDirectory(AmazonS3 s3, ObjectListing listing, String bucket) throws Exception {

		double time = System.currentTimeMillis();
		ExecutorService executor = Executors.newFixedThreadPool(16);

		int count = 0;

		for (S3ObjectSummary objectSummary : listing.getObjectSummaries()) {
			if ("STANDARD".equalsIgnoreCase(objectSummary.getStorageClass())) {
				long size = objectSummary.getSize();
				logger.debug("*** Processing S3 {}, size: {}", objectSummary.getKey(), size);
				S3Object object = s3.getObject(new GetObjectRequest(bucket, objectSummary.getKey()));

				String bucketName = object.getBucketName();
				String keyName = object.getKey();

				GetObjectTaggingRequest request = new GetObjectTaggingRequest(bucketName, keyName);
				GetObjectTaggingResult result = s3.getObjectTagging(request);
				List<Tag> tags = result.getTagSet();
				String type = null;
				String name = null;

				if (tags.isEmpty()) {
					object.close();
					logger.warn("Error, S3 object: {} has no tags", keyName);
				} else {
					for (Tag tag : tags) {
						String key = tag.getKey();
						String value = tag.getValue();

						if (key.equals("type")) {
							type = value;
						}

						if (key.equals("name")) {
							name = value;
						}
					}

					if (name == null) {
						object.close();
						throw new Exception("Error: " + keyName + " is missing a name tag");
					}
					if (name.contains(" ")) {
						object.close();
						throw new Exception("Error: " + keyName + " has a name attribute with a space in it");
					}
					if (type == null) {
						object.close();
						throw new Exception("Error: " + keyName + " has no type tag");
					}

					if (!name.startsWith("$"))
						name = "$" + name;

					// The runnable will call object.close();
					Runnable w = new AwsWorker(type, name, object, size);
					executor.execute(w);

					count++;
				}
			}
		}
		executor.shutdown();
		executor.awaitTermination(Long.MAX_VALUE, TimeUnit.NANOSECONDS);

		time = System.currentTimeMillis() - time;
		time = time / 60000;
		logger.info("Initialized all {} S3 objects in {} minutes", count, time);
	}