Java Code Examples for com.amazonaws.services.s3.model.ObjectMetadata#getContentLength()

@Override
public AbstractDbArtifact getArtifactBySha1(final String tenant, final String sha1Hash) {
    final String key = objectKey(tenant, sha1Hash);

    LOG.info("Retrieving S3 object from bucket {} and key {}", s3Properties.getBucketName(), key);
    try (final S3Object s3Object = amazonS3.getObject(s3Properties.getBucketName(), key)) {
        if (s3Object == null) {
            return null;
        }

        final ObjectMetadata s3ObjectMetadata = s3Object.getObjectMetadata();

        // the MD5Content is stored in the ETag
        return new S3Artifact(amazonS3, s3Properties, key, sha1Hash,
                new DbArtifactHash(sha1Hash,
                        BaseEncoding.base16().lowerCase()
                                .encode(BaseEncoding.base64().decode(s3ObjectMetadata.getETag())),
                        null),
                s3ObjectMetadata.getContentLength(), s3ObjectMetadata.getContentType());
    } catch (final IOException e) {
        LOG.error("Could not verify S3Object", e);
        return null;
    }
}
 

private ObjectMetadata getConfigMetadata() throws Exception
{
    try
    {
        ObjectMetadata metadata = s3Client.getObjectMetadata(arguments.getBucket(), arguments.getKey());
        if ( metadata.getContentLength() > 0 )
        {
            return metadata;
        }
    }
    catch ( AmazonS3Exception e )
    {
        if ( !isNotFoundError(e) )
        {
            throw e;
        }
    }
    return null;
}
 

@Override
public void copy(final AmazonS3 s3, final String bucket, final String sourcePath, final String destinationPath) {
  ObjectMetadata metadataResult = s3.getObjectMetadata(bucket, sourcePath);
  long length = metadataResult.getContentLength();

  try {
    if (length < chunkSize) {
      copySinglePart(s3, bucket, sourcePath, destinationPath);
    }
    else {
      copyMultiPart(s3, bucket, sourcePath, destinationPath, length);
    }
  }
  catch(SdkClientException e) {
    throw new BlobStoreException("Error copying blob", e, null);
  }
}
 

/**
 * Update object's information
 */
protected boolean updateInfo(S3Object s3object) {
	// Read metadata
	ObjectMetadata om = s3object.getObjectMetadata();

	// Update data
	size = om.getContentLength();
	canRead = true;
	canWrite = true;
	lastModified = om.getLastModified();
	exists = true;
	latestUpdate = new Timer(CACHE_TIMEOUT);

	// Show information
	if (debug) Timer.showStdErr("Updated infromation for '" + this + "'"//
			+ "\n\tcanRead      : " + canRead //
			+ "\n\texists       : " + exists //
			+ "\n\tlast modified: " + lastModified //
			+ "\n\tsize         : " + size //
	);

	return true;
}
 

/**
 * Returns the size of the dataset, in bytes. Will be zero if this dataset is a collection or non-existent.
 *
 * @return the size of the dataset
 */
@Override
public long length() {
  // If the summary is already in the cache, return it.
  // It'll have been added by a listDatasets() call on the parent directory.
  S3ObjectSummary objectSummary = objectSummaryCache.getIfPresent(s3uri);
  if (objectSummary != null) {
    return objectSummary.getSize();
  }

  /*
   * Get the metadata directly from S3. This will be expensive.
   * We get punished hard if length() and/or lastModified() is called on a bunch of datasets without
   * listDatasets() first being called on their parent directory.
   *
   * So, is the right thing to do here "getParentDataset().listDatasets()" and then query the cache again?
   * Perhaps, but listDatasets() throws an IOException, and length() and lastModified() do not.
   * We would have to change their signatures and the upstream client code to make it work.
   */
  ObjectMetadata metadata = threddsS3Client.getObjectMetadata(s3uri);
  if (metadata != null) {
    return metadata.getContentLength();
  } else {
    // "this" may be a collection or non-existent. In both cases, we return 0.
    return 0;
  }
}
 

private static long getObjectSize(Path path, ObjectMetadata metadata)
        throws IOException
{
    Map<String, String> userMetadata = metadata.getUserMetadata();
    String length = userMetadata.get(UNENCRYPTED_CONTENT_LENGTH);
    if (userMetadata.containsKey(SERVER_SIDE_ENCRYPTION) && length == null) {
        throw new IOException(format("%s header is not set on an encrypted object: %s", UNENCRYPTED_CONTENT_LENGTH, path));
    }
    return (length != null) ? Long.parseLong(length) : metadata.getContentLength();
}
 

@Override
public void commitAfterRecovery() throws IOException {
	if (totalLength > 0L) {
		LOG.info("Trying to commit after recovery {} with MPU ID {}", objectName, uploadId);

		try {
			s3AccessHelper.commitMultiPartUpload(objectName, uploadId, parts, totalLength, new AtomicInteger());
		} catch (IOException e) {
			LOG.info("Failed to commit after recovery {} with MPU ID {}. " +
					"Checking if file was committed before...", objectName, uploadId);
			LOG.trace("Exception when committing:", e);

			try {
				ObjectMetadata metadata = s3AccessHelper.getObjectMetadata(objectName);
				if (totalLength != metadata.getContentLength()) {
					String message = String.format("Inconsistent result for object %s: conflicting lengths. " +
									"Recovered committer for upload %s indicates %s bytes, present object is %s bytes",
							objectName, uploadId, totalLength, metadata.getContentLength());
					LOG.warn(message);
					throw new IOException(message, e);
				}
			} catch (FileNotFoundException fnf) {
				LOG.warn("Object {} not existing after failed recovery commit with MPU ID {}", objectName, uploadId);
				throw new IOException(String.format("Recovering commit failed for object %s. " +
						"Object does not exist and MultiPart Upload %s is not valid.", objectName, uploadId), e);
			}
		}
	} else {
		LOG.debug("No data to commit for file: {}", objectName);
	}
}
 

@Override
public void commitAfterRecovery() throws IOException {
	if (totalLength > 0L) {
		LOG.info("Trying to commit after recovery {} with MPU ID {}", objectName, uploadId);

		try {
			s3AccessHelper.commitMultiPartUpload(objectName, uploadId, parts, totalLength, new AtomicInteger());
		} catch (IOException e) {
			LOG.info("Failed to commit after recovery {} with MPU ID {}. " +
					"Checking if file was committed before...", objectName, uploadId);
			LOG.trace("Exception when committing:", e);

			try {
				ObjectMetadata metadata = s3AccessHelper.getObjectMetadata(objectName);
				if (totalLength != metadata.getContentLength()) {
					String message = String.format("Inconsistent result for object %s: conflicting lengths. " +
									"Recovered committer for upload %s indicates %s bytes, present object is %s bytes",
							objectName, uploadId, totalLength, metadata.getContentLength());
					LOG.warn(message);
					throw new IOException(message, e);
				}
			} catch (FileNotFoundException fnf) {
				LOG.warn("Object {} not existing after failed recovery commit with MPU ID {}", objectName, uploadId);
				throw new IOException(String.format("Recovering commit failed for object %s. " +
						"Object does not exist and MultiPart Upload %s is not valid.", objectName, uploadId), e);
			}
		}
	} else {
		LOG.debug("No data to commit for file: {}", objectName);
	}
}
 

@VisibleForTesting
void copy(S3ResourceId sourcePath, S3ResourceId destinationPath) throws IOException {
  try {
    ObjectMetadata sourceObjectMetadata = getObjectMetadata(sourcePath);
    if (sourceObjectMetadata.getContentLength() < MAX_COPY_OBJECT_SIZE_BYTES) {
      atomicCopy(sourcePath, destinationPath, sourceObjectMetadata);
    } else {
      multipartCopy(sourcePath, destinationPath, sourceObjectMetadata);
    }
  } catch (AmazonClientException e) {
    throw new IOException(e);
  }
}
 

@Override
public void commitAfterRecovery() throws IOException {
	if (totalLength > 0L) {
		LOG.info("Trying to commit after recovery {} with MPU ID {}", objectName, uploadId);

		try {
			s3AccessHelper.commitMultiPartUpload(objectName, uploadId, parts, totalLength, new AtomicInteger());
		} catch (IOException e) {
			LOG.info("Failed to commit after recovery {} with MPU ID {}. " +
					"Checking if file was committed before...", objectName, uploadId);
			LOG.trace("Exception when committing:", e);

			try {
				ObjectMetadata metadata = s3AccessHelper.getObjectMetadata(objectName);
				if (totalLength != metadata.getContentLength()) {
					String message = String.format("Inconsistent result for object %s: conflicting lengths. " +
									"Recovered committer for upload %s indicates %s bytes, present object is %s bytes",
							objectName, uploadId, totalLength, metadata.getContentLength());
					LOG.warn(message);
					throw new IOException(message, e);
				}
			} catch (FileNotFoundException fnf) {
				LOG.warn("Object {} not existing after failed recovery commit with MPU ID {}", objectName, uploadId);
				throw new IOException(String.format("Recovering commit failed for object %s. " +
						"Object does not exist and MultiPart Upload %s is not valid.", objectName, uploadId), e);
			}
		}
	} else {
		LOG.debug("No data to commit for file: {}", objectName);
	}
}
 

private boolean isNewResourceAvailable(String filename) {

		File file = new File(downloadDirectory, filename);

		// TODO: partial downloaded file
		if (!file.exists()) {
			return true;
		}

		try {

			ObjectMetadata metadata = manager.getAmazonS3Client()
					.getObjectMetadata(AWSClientManager.S3_BUCKET_NAME,
							filename);
			long remoteLastModified = metadata.getLastModified().getTime();

			filesize = metadata.getContentLength();

			if (file.lastModified() < remoteLastModified) {
				return true;
			} else {
				return false;
			}

		} catch (Exception e) {
			Log.e("isNewResourceAvailable", e.getMessage());
			e.printStackTrace();
		}

		return true;

	}
 

@Override
public FileStatus[] listStatus(final Path f) throws IOException {

	final S3BucketObjectPair bop = this.directoryStructure.toBucketObjectPair(f);

	try {

		if (!bop.hasBucket()) {

			final List<Bucket> list = this.s3Client.listBuckets();
			final S3FileStatus[] array = new S3FileStatus[list.size()];
			final Iterator<Bucket> it = list.iterator();
			int i = 0;
			while (it.hasNext()) {
				final Bucket bucket = it.next();
				final long creationDate = dateToLong(bucket.getCreationDate());
				// S3 does not track access times, so this implementation always sets it to 0
				final S3FileStatus status = new S3FileStatus(extendPath(f, bucket.getName()
					+ S3_DIRECTORY_SEPARATOR), 0, true, creationDate, 0L);
				array[i++] = status;
			}

			return array;
		}

		if (bop.hasBucket() && !bop.hasObject()) {

			// Check if the bucket really exists
			if (!this.s3Client.doesBucketExist(bop.getBucket())) {
				throw new FileNotFoundException("Cannot find " + f.toUri());
			}

			return listBucketContent(f, bop);

		} else {

			final ObjectMetadata omd = this.s3Client.getObjectMetadata(bop.getBucket(), bop.getObject());
			if (objectRepresentsDirectory(bop.getObject(), omd.getContentLength())) {

				return listBucketContent(f, bop);

			} else {
				final S3FileStatus fileStatus = new S3FileStatus(f, omd.getContentLength(), false,
					dateToLong(omd.getLastModified()), 0L);

				return new FileStatus[] { fileStatus };
			}

		}

	} catch (AmazonClientException e) {
		throw new IOException(StringUtils.stringifyException(e));
	}
}
 

public TocPathOpResult validateOnS3(TOCPayload payload) {
	
	if (s3Client == null || s3BucketName == null) {
		throw new RuntimeException("Cannot validateOnS3(), TOCPayloadValidator is not configured w/ s3Client or bucket name");
	}

	try {
		String keyToCheck = toc2Key(payload.tocInfo.getPath(),payload.tocInfo.isDirectory);
		logger.debug("validateOnS3() " + keyToCheck);
		
		ObjectMetadata md = s3Client.getObjectMetadata(getS3BucketName(), keyToCheck);
		
		// size not match!
		if (payload.tocInfo.size != md.getContentLength()) {
			
			logger.error("validateOnS3() S3 object length does not match! " +
					"" + keyToCheck + " expected:" + payload.tocInfo.size + " actual:" + md.getContentLength());;
					
			return new TocPathOpResult(payload.mode, false, payload.tocInfo.getPath(),
							"s3.check.content.length", "expected:"+ payload.tocInfo.size + " actual:"+md.getContentLength());

		} 
		
		// SUCCESS (no 404 so size matches and it exists)
		return new TocPathOpResult(payload.mode, true, payload.tocInfo.getPath(), "s3.check", "ok");
		
	} catch(AmazonS3Exception e) {
		
		// 404
		if (e.getStatusCode() == 404) {
			
			logger.error("validateOnS3() " + payload.tocInfo.getPath() + " s3check returned 404");
			
			return new TocPathOpResult(payload.mode, false, payload.tocInfo.getPath(),
					"s3.check.404", "key not found 404 at " + this.getS3BucketName());
			
		// other error
		} else {
			
			logger.error("validateOnS3() " + payload.tocInfo.getPath() + " unexpected error: " + e.getMessage(),e);
			
			return new TocPathOpResult(payload.mode, false, payload.tocInfo.getPath(),
					"s3.check.error", "error getting object metadata: " + e.getMessage());
		}
	}
	
}