Java Code Examples for com.amazonaws.services.s3.AmazonS3#shutdown()

The following examples show how to use com.amazonaws.services.s3.AmazonS3#shutdown() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: S3UtilProgram.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up 
private static void numberOfLinesInFilesWithFullAndNamePrefix(ParameterTool params) {
	final String bucket = params.getRequired("bucket");
	final String s3prefix = params.getRequired("s3prefix");
	final String s3filePrefix = params.get("s3filePrefix", "");
	int parallelism = params.getInt("parallelism", 10);

	List<String> files = listByFullPathPrefix(bucket, s3prefix);

	ExecutorService executor = Executors.newFixedThreadPool(parallelism);
	AmazonS3 s3client = AmazonS3ClientBuilder.defaultClient();
	List<CompletableFuture<Integer>> requests =
		submitLineCountingRequestsForFilesAsync(executor, s3client, bucket, files, s3filePrefix);
	int count = waitAndComputeTotalLineCountResult(requests);

	executor.shutdownNow();
	s3client.shutdown();
	System.out.print(count);
}
Example 2
Source File: S3UtilProgram.java    From flink with Apache License 2.0 6 votes vote down vote up 
private static void numberOfLinesInFilesWithFullAndNamePrefix(ParameterTool params) {
	final String bucket = params.getRequired("bucket");
	final String s3prefix = params.getRequired("s3prefix");
	final String s3filePrefix = params.get("s3filePrefix", "");
	int parallelism = params.getInt("parallelism", 10);

	List<String> files = listByFullPathPrefix(bucket, s3prefix);

	ExecutorService executor = Executors.newFixedThreadPool(parallelism);
	AmazonS3 s3client = AmazonS3ClientBuilder.defaultClient();
	List<CompletableFuture<Integer>> requests =
		submitLineCountingRequestsForFilesAsync(executor, s3client, bucket, files, s3filePrefix);
	int count = waitAndComputeTotalLineCountResult(requests);

	executor.shutdownNow();
	s3client.shutdown();
	System.out.print(count);
}
Example 3
Source File: AmazonS3FileSystemTestHelper.java    From iaf with Apache License 2.0 6 votes vote down vote up 
public void cleanUpBucketAndShutDown(AmazonS3 s3Client) {
	if(s3Client.doesBucketExistV2(bucketName)) {
		 ObjectListing objectListing = s3Client.listObjects(bucketName);
            while (true) {
                Iterator<S3ObjectSummary> objIter = objectListing.getObjectSummaries().iterator();
                while (objIter.hasNext()) {
                    s3Client.deleteObject(bucketName, objIter.next().getKey());
                }
    
                // If the bucket contains many objects, the listObjects() call
                // might not return all of the objects in the first listing. Check to
                // see whether the listing was truncated. If so, retrieve the next page of objects 
                // and delete them.
                if (objectListing.isTruncated()) {
                    objectListing = s3Client.listNextBatchOfObjects(objectListing);
                } else {
                    break;
                }
            }
		s3Client.deleteBucket(bucketName);
	}
	if(s3Client != null) {
		s3Client.shutdown();
	}
}
Example 4
Source File: S3UtilProgram.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up 
private static void numberOfLinesInFile(ParameterTool params) {
	final String bucket = params.getRequired("bucket");
	final String s3file = params.getRequired("s3file");
	AmazonS3 s3client = AmazonS3ClientBuilder.defaultClient();
	System.out.print(S3QueryUtil.queryFile(s3client, bucket, s3file, countQuery));
	s3client.shutdown();
}
Example 5
Source File: S3UtilProgram.java    From flink with Apache License 2.0 5 votes vote down vote up 
private static void numberOfLinesInFile(ParameterTool params) {
	final String bucket = params.getRequired("bucket");
	final String s3file = params.getRequired("s3file");
	AmazonS3 s3client = AmazonS3ClientBuilder.defaultClient();
	System.out.print(S3QueryUtil.queryFile(s3client, bucket, s3file, countQuery));
	s3client.shutdown();
}
Example 6
Source File: ListGcsObjects.java    From java-docs-samples with Apache License 2.0 5 votes vote down vote up 
public static void listGcsObjects(
    String googleAccessKeyId, String googleAccessKeySecret, String bucketName) {

  // String googleAccessKeyId = "your-google-access-key-id";
  // String googleAccessKeySecret = "your-google-access-key-secret";
  // String bucketName = "bucket-name";

  // Create a BasicAWSCredentials using Cloud Storage HMAC credentials.
  BasicAWSCredentials googleCreds =
      new BasicAWSCredentials(googleAccessKeyId, googleAccessKeySecret);

  // Create a new client and do the following:
  // 1. Change the endpoint URL to use the Google Cloud Storage XML API endpoint.
  // 2. Use Cloud Storage HMAC Credentials.
  AmazonS3 interopClient =
      AmazonS3ClientBuilder.standard()
          .withEndpointConfiguration(
              new AwsClientBuilder.EndpointConfiguration(
                  "https://storage.googleapis.com", "auto"))
          .withCredentials(new AWSStaticCredentialsProvider(googleCreds))
          .build();

  // Call GCS to list current objects
  ObjectListing objects = interopClient.listObjects(bucketName);

  // Print objects names
  System.out.println("Objects:");
  for (S3ObjectSummary object : objects.getObjectSummaries()) {
    System.out.println(object.getKey());
  }

  // Explicitly clean up client resources.
  interopClient.shutdown();
}
Example 7
Source File: ListGcsBuckets.java    From java-docs-samples with Apache License 2.0 5 votes vote down vote up 
public static void listGcsBuckets(String googleAccessKeyId, String googleAccessKeySecret) {

    // String googleAccessKeyId = "your-google-access-key-id";
    // String googleAccessKeySecret = "your-google-access-key-secret";

    // Create a BasicAWSCredentials using Cloud Storage HMAC credentials.
    BasicAWSCredentials googleCreds =
        new BasicAWSCredentials(googleAccessKeyId, googleAccessKeySecret);

    // Create a new client and do the following:
    // 1. Change the endpoint URL to use the Google Cloud Storage XML API endpoint.
    // 2. Use Cloud Storage HMAC Credentials.
    AmazonS3 interopClient =
        AmazonS3ClientBuilder.standard()
            .withEndpointConfiguration(
                new AwsClientBuilder.EndpointConfiguration(
                    "https://storage.googleapis.com", "auto"))
            .withCredentials(new AWSStaticCredentialsProvider(googleCreds))
            .build();

    // Call GCS to list current buckets
    List<Bucket> buckets = interopClient.listBuckets();

    // Print bucket names
    System.out.println("Buckets:");
    for (Bucket bucket : buckets) {
      System.out.println(bucket.getName());
    }

    // Explicitly clean up client resources.
    interopClient.shutdown();
  }