Java Code Examples for com.amazonaws.services.s3.AmazonS3#getObject()

/**
 * Fetch error info.
 *
 * @param datasource the datasource
 * @param errorList the error list
 */
private void fetchErrorInfo(String datasource, List<Map<String,String>> errorList){
	if(errorInfo==null){
		ObjectMapper objectMapper = new ObjectMapper();
    	List<Map<String, String>> inventoryErrors = new ArrayList<>();
    	AmazonS3 s3Client = AmazonS3ClientBuilder.standard()
                .withCredentials(new AWSStaticCredentialsProvider(new CredentialProvider().getCredentials(s3Account,s3Role))).withRegion(s3Region).build();
    	try {
	    	S3Object inventoryErrorData = s3Client.getObject(new GetObjectRequest(bucketName,dataPath+"/"+datasource+"-loaderror.data"));
	    	try (BufferedReader reader = new BufferedReader(new InputStreamReader(inventoryErrorData.getObjectContent()))) {
				inventoryErrors = objectMapper.readValue(reader.lines().collect(Collectors.joining("\n")),new TypeReference<List<Map<String, String>>>() {});
	        }
    	} catch (IOException e) {
    		LOGGER.error("Exception in collecting inventory error data",e);
            Map<String,String> errorMap = new HashMap<>();
            errorMap.put(ERROR, "Exception in collecting inventory error data");
            errorMap.put(ERROR_TYPE, WARN);
            errorMap.put(EXCEPTION, e.getMessage());
            errorList.add(errorMap);
		}
    	errorInfo = inventoryErrors.parallelStream().collect(Collectors.groupingBy(obj -> obj.get("type")));
	}
}
 

@Override
protected void doStart() {
    AWSCredentials myCredentials = new BasicAWSCredentials(accessKey, secretKey);
    AmazonS3 s3Client = new AmazonS3Client(myCredentials);
    ListObjectsRequest listObjectsRequest = new ListObjectsRequest().withBucketName(bucket);
    ObjectListing objectListing = s3Client.listObjects(listObjectsRequest);
    ChannelProcessor channelProcessor = getChannelProcessor();
    for (S3ObjectSummary s3ObjectSummary : objectListing.getObjectSummaries()) {
        String file = s3ObjectSummary.getKey();
        LOGGER.info("Read the content of {}", file);
        GetObjectRequest objectRequest = new GetObjectRequest(bucket, file);
        S3Object objectPortion = s3Client.getObject(objectRequest);
        try {
            long startTime = System.currentTimeMillis();
            processLines(channelProcessor, objectPortion.getObjectContent());
            LOGGER.info("Processing of {} took {} ms", file, System.currentTimeMillis() - startTime);
        } catch (IOException e) {
            LOGGER.warn("Cannot process the {}, skipping", file, e);
        }
    }
}
 

private static List<String> getEndpointIds(String s3bucketName, List<String> endpointFileKeys) {

        List<String> endpointIds = new ArrayList<>();

        // Initializes the Amazon S3 client.
        AmazonS3 s3Client = AmazonS3ClientBuilder.defaultClient();

        // Gets the endpoint IDs from the exported endpoints files.
        try {
            for (String key : endpointFileKeys) {
                S3Object endpointFile = s3Client.getObject(s3bucketName, key);
                endpointIds.addAll(getEndpointIdsFromFile(endpointFile));
            }
        } catch (AmazonServiceException e) {
            System.err.println(e.getMessage());
            System.exit(1);
        }

        return endpointIds;
    }
 

public static void downloadFromS3(String s3BucketName, List<String> objectKeys,
                                  String downloadDirectory) {

    // Initializes the Amazon S3 client.
    AmazonS3 s3Client = AmazonS3ClientBuilder.defaultClient();

    try {
        // Downloads each object to the specified file path.
        for (String key : objectKeys) {
            S3Object object = s3Client.getObject(s3BucketName, key);
            String endpointsFileName = key.substring(key.lastIndexOf("/"));
            Path filePath = Paths.get(downloadDirectory + endpointsFileName);

            System.out.format("Downloading %s to %s . . .\n",
                    filePath.getFileName(), filePath.getParent());

            writeObjectToFile(filePath, object);
        }
        System.out.println("Download finished.");
    } catch (AmazonServiceException | NullPointerException e) {
        System.err.println(e.getMessage());
        System.exit(1);
    }

}
 

static S3Object getObject(
    AmazonS3 s3Client,
    String bucket,
    String objectKey,
    boolean useSSE,
    CredentialValue customerKey,
    CredentialValue customerKeyMd5
) throws StageException {
  GetObjectRequest getObjectRequest = new GetObjectRequest(bucket, objectKey);
  if (useSSE) {
    SSECustomerKey sseCustomerKey = new SSECustomerKey(customerKey.get());
    sseCustomerKey.setMd5(customerKeyMd5.get());
    getObjectRequest.setSSECustomerKey(sseCustomerKey);
  }
  return s3Client.getObject(getObjectRequest);
}
 

@Override
public File downloadFile(
    AmazonS3 s3Client,
    FileStore fileStore,
    String jobIdentifier,
    String bucketName,
    String keyName,
    String extension,
    boolean isExpression,
    String targetEntityType)
    throws IOException {
  String key;
  // The key can be a regular expression instead of the actual key.
  // This is indicated by the "isExpression" boolean
  if (isExpression) {
    key = this.getMostRecentMatchingKey(s3Client, bucketName, keyName);
  } else {
    key = keyName;
  }
  S3Object s3Object = s3Client.getObject(new GetObjectRequest(bucketName, key));
  try (InputStream in = s3Object.getObjectContent()) {
    return storeFile(fileStore, key, extension, targetEntityType, jobIdentifier, in);
  }
}
 

/**
 * Fetch error info.
 *
 * @param datasource the datasource
 * @param errorList the error list
 */
private void fetchErrorInfo(List<Map<String,String>> errorList){
	if(errorInfo==null){
		ObjectMapper objectMapper = new ObjectMapper();
    	List<Map<String, String>> inventoryErrors = new ArrayList<>();
    	AmazonS3 s3Client = AmazonS3ClientBuilder.standard()
                .withCredentials(new AWSStaticCredentialsProvider(new CredentialProvider().getCredentials(s3Account,s3Role))).withRegion(s3Region).build();
    	try {
	    	S3Object inventoryErrorData = s3Client.getObject(new GetObjectRequest(bucketName,dataPath+"/"+dataSource+"-loaderror.data"));
	    	try (BufferedReader reader = new BufferedReader(new InputStreamReader(inventoryErrorData.getObjectContent()))) {
				inventoryErrors = objectMapper.readValue(reader.lines().collect(Collectors.joining("\n")),new TypeReference<List<Map<String, String>>>() {});
	        }
    	} catch (IOException e) {
    		LOGGER.error("Exception in collecting inventory error data",e);
            Map<String,String> errorMap = new HashMap<>();
            errorMap.put(ERROR, "Exception in collecting inventory error data");
            errorMap.put(ERROR_TYPE, WARN);
            errorMap.put(EXCEPTION, e.getMessage());
            errorList.add(errorMap);
		}
    	errorInfo = inventoryErrors.parallelStream().collect(Collectors.groupingBy(obj -> obj.get("type")));
	}
}
 

public static <T> List<Map<String, T>> fetchDataFromS3(String s3Account,String s3Region,String s3Role, String bucketName,String path) throws IOException{
	AmazonS3 s3Client = AmazonS3ClientBuilder.standard()
               .withCredentials(new AWSStaticCredentialsProvider(new CredentialProvider().getCredentials(s3Account,s3Role))).withRegion(s3Region).build();
	S3Object entitiesData = s3Client.getObject(new GetObjectRequest(bucketName, path));
	try (BufferedReader reader = new BufferedReader(new InputStreamReader(entitiesData.getObjectContent()))) {
		return new ObjectMapper().readValue(reader.lines().collect(Collectors.joining("\n")),new TypeReference<List<Map<String, T>>>() {});
    }
}
 

private cfData	readToMemory(AmazonS3 s3Client, String bucket, String key, String aes256key, int retry, int retryseconds) throws Exception {
	
	// Let us run around the number of attempts
	int attempts = 0;
	while ( attempts < retry ){
		try{
			
			GetObjectRequest gor = new GetObjectRequest(bucket, key);
			if ( aes256key != null && !aes256key.isEmpty() )
				gor.setSSECustomerKey( new SSECustomerKey(aes256key) );
	
			S3Object s3object = s3Client.getObject(gor);
			String contentType = s3object.getObjectMetadata().getContentType();
			
			ByteArrayOutputStream	baos	= new ByteArrayOutputStream( 32000 );
			StreamUtil.copyTo(s3object.getObjectContent(), baos, false );
			
			if ( contentType.indexOf("text") != -1 || contentType.indexOf("javascript") != -1 ){
				return new cfStringData( baos.toString() );
			}else{
				return new cfBinaryData( baos.toByteArray() );
			}
			
		}catch(Exception e){
			cfEngine.log("Failed: AmazonS3Read(bucket=" + bucket + "; key=" + key + "; attempt=" + (attempts+1) + "; exception=" + e.getMessage() + ")");
			attempts++;
	
			if ( attempts == retry )
				throw e;
			else
				Thread.sleep( retryseconds*1000 );
		}
	}
	
	return null; // should never 
}
 

/**
 * Creates a new S3 writable configuration.
 * @param s3 An S3 {@link AmazonS3 client}.
 * @param s3ObjectSummary The S3 object {@link S3ObjectSummary summary}.
 * @param version The version of the configuration.
 */
public S3WritableConfiguration(AmazonS3 s3, S3ObjectSummary s3ObjectSummary, String version) {

  this.s3 = s3;
  this.s3Object = s3.getObject(s3ObjectSummary.getBucketName(), s3ObjectSummary.getKey());
  this.version = version;

}
 

private S3Object getObject(final URI uri, final String key, final String secret) {

        final AmazonS3 s3 = buildS3(uri, key, secret);
        final String bucketName = extractBucket(uri);
        final String objectName = extractObjectName(uri);
        return s3.getObject(bucketName, objectName);
    }
 

public SecureShellAuthentication(Bucket bucket, AmazonS3 client) {
    factory = new JschConfigSessionFactory() {

        @Override
        public synchronized RemoteSession getSession(URIish uri, CredentialsProvider credentialsProvider, FS fs, int tms) throws TransportException {
            // Do not check for default ssh user config
            fs.setUserHome(null);
            return super.getSession(uri, credentialsProvider, fs, tms);
        }

        @Override
        protected void configure(OpenSshConfig.Host host, Session session) {
            session.setConfig("HashKnownHosts", "no");
            if ("localhost".equalsIgnoreCase(host.getHostName())) {
                session.setConfig("StrictHostKeyChecking", "no");
            }
        }

        @Override
        protected void configureJSch(JSch jsch) {
            S3Object file;
            file = client.getObject(bucket.getName(), ".ssh/known_hosts");
            try (InputStream is = file.getObjectContent()) {
                jsch.setKnownHosts(is);
            } catch (IOException | JSchException e) {
                throw new IllegalArgumentException("Missing known hosts file on s3: .ssh/known_hosts", e);
            }
            file = client.getObject(bucket.getName(), ".ssh/id_rsa");
            try (InputStream is = file.getObjectContent()) {
                jsch.addIdentity("git", IOUtils.toByteArray(is), null, new byte[0]);
            } catch (IOException | JSchException e) {
                throw new IllegalArgumentException("Missing key file on s3: .ssh/id_rsa", e);
            }
        }
    };
}
 

@NotNull
private static S3ObjectSummary getObjectSummary(AmazonS3 s3Client, String bucket, String key) {
  S3Object currentObject = s3Client.getObject(bucket, key);
  S3ObjectSummary currentObjectSummary = new S3ObjectSummary();
  currentObjectSummary.setBucketName(currentObject.getBucketName());
  currentObjectSummary.setKey(currentObject.getKey());
  currentObjectSummary.setETag(currentObject.getObjectMetadata().getETag());
  currentObjectSummary.setSize(currentObject.getObjectMetadata().getContentLength());
  currentObjectSummary.setLastModified(currentObject.getObjectMetadata().getLastModified());
  currentObjectSummary.setStorageClass(currentObject.getObjectMetadata().getStorageClass());
  return currentObjectSummary;
}
 

public static void main(String arg[]) throws Exception {
    
    // The S3 bucket and document
    String document = "";
    String bucket = "";

    
    AmazonS3 s3client = AmazonS3ClientBuilder.standard()
            .withEndpointConfiguration( 
                    new EndpointConfiguration("https://s3.amazonaws.com","us-east-1"))
            .build();
    
           
    // Get the document from S3
    com.amazonaws.services.s3.model.S3Object s3object = s3client.getObject(bucket, document);
    S3ObjectInputStream inputStream = s3object.getObjectContent();
    BufferedImage image = ImageIO.read(inputStream);

    // Call DetectDocumentText
    EndpointConfiguration endpoint = new EndpointConfiguration(
            "https://textract.us-east-1.amazonaws.com", "us-east-1");
    AmazonTextract client = AmazonTextractClientBuilder.standard()
            .withEndpointConfiguration(endpoint).build();


    DetectDocumentTextRequest request = new DetectDocumentTextRequest()
        .withDocument(new Document().withS3Object(new S3Object().withName(document).withBucket(bucket)));

    DetectDocumentTextResult result = client.detectDocumentText(request);
    
    // Create frame and panel.
    JFrame frame = new JFrame("RotateImage");
    frame.setDefaultCloseOperation(JFrame.EXIT_ON_CLOSE);
    DocumentText panel = new DocumentText(result, image);
    panel.setPreferredSize(new Dimension(image.getWidth() , image.getHeight() ));
    frame.setContentPane(panel);
    frame.pack();
    frame.setVisible(true);

}
 

/**
 * Fetchs a remote object from S3; datasource, bucket, key, aes256key supported
 * 
 * @param props
 * @param _Session
 * @throws cfmRunTimeException
 */
private void remoteFetchS3( cfStructData props, cfSession _Session ) throws cfmRunTimeException {

	if ( !props.containsKey( "datasource" ) ||
			!props.containsKey( "bucket" ) ||
			!props.containsKey( "key" ) )
		throw newRunTimeException( "'remote'.type=s3; minimum keys are datasource, bucket and key" );

	String datasource = props.getData( "datasource" ).getString();
	String bucket = props.getData( "bucket" ).getString();
	String key = props.getData( "key" ).getString();

	// Get the Amazon datasource
	AmazonKey amazonKey = AmazonKeyFactory.getDS( datasource );
	if ( amazonKey == null )
		throw newRunTimeException( "Amazon Datasource [" + datasource + "] has not been registered; use AmazonRegisterDataSource()" );

	amazonKey.setDataSource( datasource );

	AmazonS3 s3Client = new AmazonBase().getAmazonS3( amazonKey );

	GetObjectRequest gor = new GetObjectRequest( bucket, key );
	if ( props.containsKey( "aes256key" ) ) {
		String aes256key = props.getData( "aes256key" ).getString();

		if ( !aes256key.isEmpty() )
			gor.setSSECustomerKey( new SSECustomerKey( aes256key ) );
	}

	// Get the object
	try {
	
		S3Object s3object = s3Client.getObject( gor );

		_Session.setContentType( s3object.getObjectMetadata().getContentType() );

		InputStream in = s3object.getObjectContent();

		byte[] buffer = new byte[65536];
		int readCount = 0;

		while ( ( readCount = in.read( buffer ) ) != -1 ) {
			_Session.write( buffer, 0, readCount );
			_Session.pageFlush();
		}

	} catch ( Exception e ) {
		
		if ( e.getMessage().indexOf("404") != -1 ){
			_Session.setStatus( 404 );
			return;
		}else{
			cfEngine.log( e.getMessage() );
			throw newRunTimeException( e.getMessage() + "; key=" + key + "; bucket=" + bucket );
		}
	}
}
 

public static InputStream makeS3FileStream(String bucketName, String key) {
    AmazonS3 s3Client = makeS3Client();
    S3Object object = s3Client.getObject(new GetObjectRequest(bucketName, key));
    return object.getObjectContent();
}
 

@Override
public S3Object getS3Object(GetObjectRequest getObjectRequest, AmazonS3 s3)
{
    return s3.getObject(getObjectRequest);
}
 

private S3Object getS3Object(final AmazonS3 s3Client, final Artifact artifact) {
    final S3ArtifactLocation artifactLocation = artifact.getLocation().getS3Location();
    return s3Client.getObject(artifactLocation.getBucketName(), artifactLocation.getObjectKey());
}
 

public static void main(String[] args) throws IOException {
    //BEGIN_SAMPLE:AmazonS3.CreateClient
    //TITLE:Create an S3 client
    //DESCRIPTION:Create your credentials file at ~/.aws/credentials (C:\Users\USER_NAME\.aws\credentials for Windows users)
    AmazonS3 s3 = AmazonS3ClientBuilder.standard().build();
    Region usWest2 = com.amazonaws.regions.Region.getRegion(Regions.US_WEST_2);
    s3.setRegion(usWest2);
    //END_SAMPLE

    String bucketName = "my-first-s3-bucket-" + UUID.randomUUID();
    String key = "MyObjectKey";

    System.out.println("===========================================");
    System.out.println("Getting Started with Amazon S3");
    System.out.println("===========================================\n");

    try {
        System.out.println("Creating bucket " + bucketName + "\n");

        //BEGIN_SAMPLE:AmazonS3.CreateBucket
        //TITLE:Create an S3 bucket
        //DESCRIPTION:Amazon S3 bucket names are globally unique, so once a bucket name has been taken by any user, you can't create another bucket with that same name.
        s3.createBucket(bucketName);
        //END_SAMPLE


        System.out.println("Listing buckets");
        //BEGIN_SAMPLE:AmazonS3.ListBuckets
        //TITLE:List buckets
        //DESCRIPTION:List the buckets in your account
        for (Bucket bucket : s3.listBuckets()) {
            System.out.println(" - " + bucket.getName());
        }
        System.out.println();
        //END_SAMPLE


        System.out.println("Uploading a new object to S3 from a file\n");
        //BEGIN_SAMPLE:AmazonS3.PutObject
        //TITLE:Upload an object to a bucket
        //DESCRIPTION:You can easily upload a file to S3, or upload directly an InputStream if you know the length of the data in the stream.
        s3.putObject(new PutObjectRequest(bucketName, key, createSampleFile()));
        //END_SAMPLE

        System.out.println("Downloading an object");
        //BEGIN_SAMPLE:AmazonS3.GetObject
        //TITLE:Download an S3 object.
        //DESCRIPTION:When you download an object, you get all of the object's metadata and a stream from which to read the contents.
        S3Object object = s3.getObject(new GetObjectRequest(bucketName, key));
        //END_SAMPLE
        System.out.println("Content-Type: "  + object.getObjectMetadata().getContentType());
        displayTextInputStream(object.getObjectContent());


        System.out.println("Listing objects");

        //BEGIN_SAMPLE:AmazonS3.ListObjects
        //TITLE:List S3 objects in bucket
        //DESCRIPTION:List objects in your bucket by prefix.  Keep in mind that buckets with many objects might truncate their results when listing their objects, so be sure to check if the returned object listing is truncated.
        ObjectListing objectListing = s3.listObjects(new ListObjectsRequest()
                .withBucketName(bucketName)
                .withPrefix("My"));
        for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) {
            System.out.println(" - " + objectSummary.getKey() + "  " +
                    "(size = " + objectSummary.getSize() + ")");
        }
        System.out.println();
        //END_SAMPLE


        System.out.println("Deleting an object\n");

        //BEGIN_SAMPLE:AmazonS3.DeleteObject
        //TITLE:Delete an S3 object
        //DESCRIPTION:Unless versioning has been turned on for your bucket, there is no way to undelete an object, so use caution when deleting objects.
        s3.deleteObject(bucketName, key);
        //END_SAMPLE


        System.out.println("Deleting bucket " + bucketName + "\n");

        //BEGIN_SAMPLE:AmazonS3.DeleteBucket
        //TITLE:Delete an S3 bucket
        //DESCRIPTION:A bucket must be completely empty before it can be deleted, so remember to delete any objects from your buckets before you try to delete them.
        s3.deleteBucket(bucketName);
        //END_SAMPLE
    } catch (AmazonServiceException ase) {
        System.out.println("Caught an AmazonServiceException, which means your request made it "
                + "to Amazon S3, but was rejected with an error response for some reason.");
        System.out.println("Error Message:    " + ase.getMessage());
        System.out.println("HTTP Status Code: " + ase.getStatusCode());
        System.out.println("AWS Error Code:   " + ase.getErrorCode());
        System.out.println("Error Type:       " + ase.getErrorType());
        System.out.println("Request ID:       " + ase.getRequestId());
    } catch (AmazonClientException ace) {
        System.out.println("Caught an AmazonClientException, which means the client encountered "
                + "a serious internal problem while trying to communicate with S3, "
                + "such as not being able to access the network.");
        System.out.println("Error Message: " + ace.getMessage());
    }
}
 

public void run() {
    String message = null;
    AWSCredentials credentials = new BasicAWSCredentials(access_key, secret_key);
    File file = new File(what);
    AmazonS3 s3Client = new AmazonS3Client(credentials,
            new ClientConfiguration());
    if (endpoint.contains("amazonaws.com")) {
        String aws_endpoint = s3Client.getBucketLocation(new GetBucketLocationRequest(bucket));
        if (aws_endpoint.contains("US")) {
            s3Client.setEndpoint("https://s3.amazonaws.com");
        } else if (aws_endpoint.contains("us-west")) {
            s3Client.setEndpoint("https://s3-" + aws_endpoint + ".amazonaws.com");
        } else if (aws_endpoint.contains("eu-west")) {
            s3Client.setEndpoint("https://s3-" + aws_endpoint + ".amazonaws.com");
        } else if (aws_endpoint.contains("ap-")) {
            s3Client.setEndpoint("https://s3-" + aws_endpoint + ".amazonaws.com");
        } else if (aws_endpoint.contains("sa-east-1")) {
            s3Client.setEndpoint("https://s3-" + aws_endpoint + ".amazonaws.com");
        } else {
            s3Client.setEndpoint("https://s3." + aws_endpoint + ".amazonaws.com");
        }
    } else {
        s3Client.setS3ClientOptions(S3ClientOptions.builder().setPathStyleAccess(true).build());
        s3Client.setEndpoint(endpoint);
    }

    try {
        long t1 = System.currentTimeMillis();
        S3Object s3object = s3Client.getObject(new GetObjectRequest(bucket, what, version));
        InputStream objectData = s3object.getObjectContent();
        this.writeFile(objectData, destination);
        long t2 = System.currentTimeMillis();
        long diff = t2 - t1;

        if (!mainFrame.perf) {
            if (terminal) {
                System.out.print("\nDownloaded: " + what + " in " + diff / 1000 + " second(s).\n");
            } else {
                mainFrame.jTextArea1.append("\nDownloaded: " + what + " in " + diff / 1000 + " second(s).");
                mainFrame.calibrateTextArea();
            }
        }

    } catch (AmazonServiceException ase) {
        if (NewJFrame.gui) {
            mainFrame.jTextArea1.append("\n\nError Message:    " + ase.getMessage());
            mainFrame.jTextArea1.append("\nHTTP Status Code: " + ase.getStatusCode());
            mainFrame.jTextArea1.append("\nAWS Error Code:   " + ase.getErrorCode());
            mainFrame.jTextArea1.append("\nError Type:       " + ase.getErrorType());
            mainFrame.jTextArea1.append("\nRequest ID:       " + ase.getRequestId());
            calibrate();
        } else {
            System.out.print("\n\nError Message:    " + ase.getMessage());
            System.out.print("\nHTTP Status Code: " + ase.getStatusCode());
            System.out.print("\nAWS Error Code:   " + ase.getErrorCode());
            System.out.print("\nError Type:       " + ase.getErrorType());
            System.out.print("\nRequest ID:       " + ase.getRequestId());
        }
    } catch (Exception get) {
    }
    calibrate();
}