Java Code Examples for com.amazonaws.services.s3.model.S3ObjectInputStream#close()

The following examples show how to use com.amazonaws.services.s3.model.S3ObjectInputStream#close() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: S3Utils.java    From singleton with Eclipse Public License 2.0 6 votes vote down vote up 
/**
 * convert the S3 Object to String
 */
public static String convertS3Obj2Str(S3Object s3Obj) throws IOException {
   S3ObjectInputStream s3is = s3Obj.getObjectContent();
   ByteArrayOutputStream fos = new ByteArrayOutputStream();
   byte[] read_buf = new byte[1024];
   int read_len = 0;
   try {
      while ((read_len = s3is.read(read_buf)) > 0) {
         fos.write(read_buf, 0, read_len);
      }
      return fos.toString(ConstantsUnicode.UTF8);
   } finally {
      s3is.close();
      fos.close();

   }
}
Example 2
Source File: S3RecordReader.java    From attic-apex-malhar with Apache License 2.0 6 votes vote down vote up 
/**
 * S3 block read would be achieved through the AmazonS3 client. Following
 * are the steps to achieve: (1) Create the objectRequest from bucketName
 * and filePath. (2) Set the range to the above created objectRequest. (3)
 * Get the object portion through AmazonS3 client API. (4) Get the object
 * content from the above object portion.
 *
 * @param bytesFromCurrentOffset
 *          bytes read till now from current offset
 * @param bytesToFetch
 *          the number of bytes to be fetched
 * @return the number of bytes read, -1 if 0 bytes read
 * @throws IOException
 */

@Override
protected int readData(final long bytesFromCurrentOffset, final int bytesToFetch) throws IOException
{
  GetObjectRequest rangeObjectRequest = new GetObjectRequest(s3Params.bucketName, s3Params.filePath);
  rangeObjectRequest.setRange(offset + bytesFromCurrentOffset, offset + bytesFromCurrentOffset + bytesToFetch - 1);
  S3Object objectPortion = s3Params.s3Client.getObject(rangeObjectRequest);
  S3ObjectInputStream wrappedStream = objectPortion.getObjectContent();
  buffer = ByteStreams.toByteArray(wrappedStream);
  wrappedStream.close();
  int bufferLength = buffer.length;
  if (bufferLength <= 0) {
    return -1;
  }
  return bufferLength;
}
Example 3
Source File: S3BlockReader.java    From attic-apex-malhar with Apache License 2.0 6 votes vote down vote up 
/**
 * S3 block read would be achieved through the AmazonS3 client. Following are the steps to achieve:
 * (1) Create the objectRequest from bucketName and filePath.
 * (2) Set the range to the above created objectRequest.
 * (3) Get the object portion through AmazonS3 client API.
 * (4) Get the object content from the above object portion.
 * @return the block entity
 * @throws IOException
 */
@Override
protected Entity readEntity() throws IOException
{
  entity.clear();
  GetObjectRequest rangeObjectRequest = new GetObjectRequest(
      bucketName, filePath);
  rangeObjectRequest.setRange(offset, blockMetadata.getLength() - 1);
  S3Object objectPortion = s3Client.getObject(rangeObjectRequest);
  S3ObjectInputStream wrappedStream = objectPortion.getObjectContent();
  byte[] record = ByteStreams.toByteArray(wrappedStream);
  entity.setUsedBytes(record.length);
  entity.setRecord(record);
  wrappedStream.close();
  return entity;
}
Example 4
Source File: DataS3.java    From BigDataScript with Apache License 2.0 4 votes vote down vote up 
/**
 * Download a file
 */
@Override
public boolean download(Data local) {
	try {
		if (!isFile()) return false;
		if (local != null) localPath = local.getAbsolutePath();

		S3Object s3object = getS3().getObject(new GetObjectRequest(bucketName, key));
		if (verbose) System.out.println("Downloading '" + this + "'");
		updateInfo(s3object);

		// Create local file and directories
		mkdirsLocal();
		FileOutputStream os = new FileOutputStream(getLocalPath());

		// Copy S3 object to file
		S3ObjectInputStream is = s3object.getObjectContent();
		int count = 0, total = 0, lastShown = 0;
		byte data[] = new byte[BUFFER_SIZE];
		while ((count = is.read(data, 0, BUFFER_SIZE)) != -1) {
			os.write(data, 0, count);
			total += count;

			if (verbose) {
				// Show every MB
				if ((total - lastShown) > (1024 * 1024)) {
					System.err.print(".");
					lastShown = total;
				}
			}
		}
		if (verbose) System.err.println("");

		// Close streams
		is.close();
		os.close();
		if (verbose) Timer.showStdErr("Donwload finished. Total " + total + " bytes.");

		// Update last modified info
		updateLocalFileLastModified();

		return true;
	} catch (Exception e) {
		Timer.showStdErr("ERROR while downloading " + this);
		throw new RuntimeException(e);
	}
}
Example 5
Source File: S3RecordReader.java    From attic-apex-malhar with Apache License 2.0 3 votes vote down vote up 
/**
 * Reads data from S3 starting from startOffset till the endOffset and
 * returns the number of bytes read
 *
 * @param startOffset
 *          offset from where to read
 * @param endOffset
 *          offset till where to read
 * @return number of bytes read
 * @throws IOException
 */
protected int readData(long startOffset, long endOffset) throws IOException
{
  GetObjectRequest rangeObjectRequest = new GetObjectRequest(s3Params.bucketName, s3Params.filePath);
  rangeObjectRequest.setRange(startOffset, endOffset);
  S3Object objectPortion = s3Params.s3Client.getObject(rangeObjectRequest);
  S3ObjectInputStream wrappedStream = objectPortion.getObjectContent();
  buffer = ByteStreams.toByteArray(wrappedStream);
  wrappedStream.close();
  return buffer.length;
}