software.amazon.awssdk.services.s3.model.GetObjectRequest Java Examples

private String getS3Url(String objectKey) {
    AwsCredentialsProvider credentialsProvider = StaticCredentialsProvider.create(AwsBasicCredentials.create(accessKey, secretKey));
    S3Presigner preSigner = S3Presigner.builder()
            .credentialsProvider(credentialsProvider)
            .endpointOverride(URI.create(endpoint))
            .region(clientRegion).build();
    GetObjectRequest getObjectRequest = GetObjectRequest.builder()
            .bucket(bucketName)
            .key(objectKey)
            .build();
    GetObjectPresignRequest getObjectPresignRequest = GetObjectPresignRequest.builder()
            .getObjectRequest(getObjectRequest).signatureDuration(Duration.ofDays(7)).build();
    PresignedGetObjectRequest presignedGetObjectRequest = preSigner.presignGetObject(getObjectPresignRequest);
    String url = presignedGetObjectRequest.url().toString();
    preSigner.close();
    return url;
}
 

/**
 * Read directly from the remote service All reading goes through here or readToByteChannel;
 *
 * 1. https://docs.aws.amazon.com/AmazonS3/latest/dev/RetrievingObjectUsingJava.html
 *
 * @param pos start here in the file
 * @param buff put data into this buffer
 * @param offset buffer offset
 * @param len this number of bytes
 * @return actual number of bytes read
 * @throws IOException on io error
 */
@Override
public int readRemote(long pos, byte[] buff, int offset, int len) throws IOException {

  String range = String.format("bytes=%d-%d", pos, pos + len);
  GetObjectRequest rangeObjectRequest =
      GetObjectRequest.builder().bucket(uri.getBucket()).key(uri.getKey()).range(range).build();

  ResponseBytes<GetObjectResponse> objectPortion = client.getObjectAsBytes(rangeObjectRequest);

  int bytes;
  int totalBytes = 0;
  // read response into buff
  try (InputStream objectData = objectPortion.asInputStream()) {
    bytes = objectData.read(buff, offset + totalBytes, len - totalBytes);
    while ((bytes > 0) && ((len - totalBytes) > 0)) {
      totalBytes += bytes;
      bytes = objectData.read(buff, offset + totalBytes, len - totalBytes);
    }
  }
  return totalBytes;
}
 

@Test
public void equalsAndHashCode_differentProperty_getObjectRequest() {
    GetObjectRequest otherGetObjectRequest = GetObjectRequest.builder()
                                                             .bucket("other-bucket")
                                                             .key("other-key")
                                                             .build();
    GetObjectPresignRequest getObjectPresignRequest =
        GetObjectPresignRequest.builder()
                               .getObjectRequest(GET_OBJECT_REQUEST)
                               .signatureDuration(Duration.ofSeconds(123L))
                               .build();

    GetObjectPresignRequest otherGetObjectPresignRequest =
        GetObjectPresignRequest.builder()
                               .getObjectRequest(otherGetObjectRequest)
                               .signatureDuration(Duration.ofSeconds(123L))
                               .build();

    assertThat(otherGetObjectPresignRequest).isNotEqualTo(getObjectPresignRequest);
    assertThat(otherGetObjectPresignRequest.hashCode()).isNotEqualTo(getObjectPresignRequest.hashCode());
}
 

@Test
public void sse_onBucket_succeeds() throws FileNotFoundException {
    String key = UUID.randomUUID().toString();

    PutObjectRequest request = PutObjectRequest.builder()
                                               .key(key)
                                               .bucket(BUCKET_WITH_SSE)
                                               .build();

    s3.putObject(request, file.toPath());

    GetObjectRequest getObjectRequest = GetObjectRequest.builder()
                                                        .key(key)
                                                        .bucket(BUCKET_WITH_SSE)
                                                        .build();

    String response = s3.getObject(getObjectRequest, ResponseTransformer.toBytes()).asUtf8String();
    SdkAsserts.assertStringEqualsStream(response, new FileInputStream(file));
}
 

@Test
public void sse_AWSKMS_succeeds() throws Exception {
    String key = UUID.randomUUID().toString();
    PutObjectRequest request = PutObjectRequest.builder()
                                               .key(key)
                                               .bucket(BUCKET)
                                               .serverSideEncryption(ServerSideEncryption.AWS_KMS)
                                               .build();

    s3.putObject(request, file.toPath());

    GetObjectRequest getObjectRequest = GetObjectRequest.builder()
                                                        .key(key)
                                                        .bucket(BUCKET)
                                                        .build();

    String response = s3.getObject(getObjectRequest, ResponseTransformer.toBytes()).asUtf8String();
    SdkAsserts.assertStringEqualsStream(response, new FileInputStream(file));
}
 

@Test
public void sse_AES256_succeeds() throws Exception {
    String key = UUID.randomUUID().toString();
    PutObjectRequest request = PutObjectRequest.builder()
                                               .key(key)
                                               .bucket(BUCKET)
                                               .serverSideEncryption(AES256)
                                               .build();

    s3.putObject(request, file.toPath());

    GetObjectRequest getObjectRequest = GetObjectRequest.builder()
                                                        .key(key)
                                                        .bucket(BUCKET)
                                                        .build();

    InputStream response = s3.getObject(getObjectRequest);
    SdkAsserts.assertFileEqualsStream(file, response);
}
 

@Test
public void sse_onBucket_succeeds() throws FileNotFoundException {
    String key = UUID.randomUUID().toString();

    PutObjectRequest request = PutObjectRequest.builder()
                                               .key(key)
                                               .bucket(BUCKET_WITH_SSE)
                                               .build();

    s3Async.putObject(request, file.toPath()).join();

    GetObjectRequest getObjectRequest = GetObjectRequest.builder()
                                                        .key(key)
                                                        .bucket(BUCKET_WITH_SSE)
                                                        .build();

    verifyGetResponse(getObjectRequest);
}
 

@Test
public void sse_AWSKMS_succeeds() throws FileNotFoundException {
    String key = UUID.randomUUID().toString();
    PutObjectRequest request = PutObjectRequest.builder()
                                               .key(key)
                                               .bucket(BUCKET)
                                               .serverSideEncryption(ServerSideEncryption.AWS_KMS)
                                               .build();

    s3Async.putObject(request, file.toPath()).join();

    GetObjectRequest getObjectRequest = GetObjectRequest.builder()
                                                        .key(key)
                                                        .bucket(BUCKET)
                                                        .build();

    verifyGetResponse(getObjectRequest);
}
 

@Test
public void sse_AES256_succeeds() throws FileNotFoundException {
    String key = UUID.randomUUID().toString();
    PutObjectRequest request = PutObjectRequest.builder()
                                               .key(key)
                                               .bucket(BUCKET)
                                               .serverSideEncryption(AES256)
                                               .build();

    s3Async.putObject(request, file.toPath()).join();

    GetObjectRequest getObjectRequest = GetObjectRequest.builder()
                                                        .key(key)
                                                        .bucket(BUCKET)
                                                        .build();

    verifyGetResponse(getObjectRequest);
}
 

@Test
public void modifyResponse_getObjectResponseContainsChecksumHeader_shouldModifyResponse() {
    long contentLength = 50;
    GetObjectResponse response = GetObjectResponse.builder().contentLength(contentLength).build();
    Context.ModifyResponse modifyResponseContext = modifyResponseContext(
        GetObjectRequest.builder().build(),
        response,
        SdkHttpFullResponse.builder()
                           .putHeader(CHECKSUM_ENABLED_RESPONSE_HEADER, ENABLE_MD5_CHECKSUM_HEADER_VALUE).build());

    GetObjectResponse actualResponse = (GetObjectResponse) interceptor.modifyResponse(modifyResponseContext,
                                                                                      new ExecutionAttributes());

    assertThat(actualResponse).isNotEqualTo(response);
    assertThat(actualResponse.contentLength()).isEqualTo(contentLength - S3_MD5_CHECKSUM_LENGTH);
}
 

@Test
public void transfer_Succeeds_UsingAccessPoint_CrossRegion() {
    S3Client s3DifferentRegion =
        s3ClientBuilder().region(Region.US_EAST_1).serviceConfiguration(c -> c.useArnRegionEnabled(true)).build();

    StringJoiner apArn = new StringJoiner(":");
    apArn.add("arn").add("aws").add("s3").add("us-west-2").add(accountId).add("accesspoint").add(AP_NAME);

    s3DifferentRegion.putObject(PutObjectRequest.builder()
                                                .bucket(apArn.toString())
                                                .key(KEY)
                                                .build(), RequestBody.fromString("helloworld"));

    String objectContent = s3DifferentRegion.getObjectAsBytes(GetObjectRequest.builder()
                                                                              .bucket(apArn.toString())
                                                                              .key(KEY)
                                                                              .build()).asUtf8String();

    assertThat(objectContent).isEqualTo("helloworld");
}
 

@Test
public void transfer_Succeeds_UsingAccessPoint() {
    StringJoiner apArn = new StringJoiner(":");
    apArn.add("arn").add("aws").add("s3").add("us-west-2").add(accountId).add("accesspoint").add(AP_NAME);

    s3.putObject(PutObjectRequest.builder()
                                 .bucket(apArn.toString())
                                 .key(KEY)
                                 .build(), RequestBody.fromString("helloworld"));

    String objectContent = s3.getObjectAsBytes(GetObjectRequest.builder()
                                                               .bucket(apArn.toString())
                                                               .key(KEY)
                                                               .build()).asUtf8String();

    assertThat(objectContent).isEqualTo("helloworld");
}
 

/**
 * Returns a PendingDownload for which the caller can waitForCompletion() to wait for the download to complete. This
 * either queues or begins the download (depending on the AWS TransferManager).
 *
 * @param s3ObjectKey
 * @param localFile
 * @return
 */
private PendingDownload saveToLocalAsync(String s3ObjectKey, Path localFile) {
    File file = localFile.toFile();
    // If process stops abruptly and is restarted, it's possible we try to re-download some of the files which
    // already exist on disk because lastValidFileName wasn't updated. AsyncFileResponseTransformer throws
    // exceptions if a file already exists, rather then silently overwrite it. So following check are to avoid
    // log spam of java.nio.file.FileAlreadyExistsException, not for optimization purpose. We can't use old file
    // because it may be half written or it maybe a data file from a node which doesn't match the hash.
    // Overwriting is the only way forward.
    // This is okay for now since we are moving away from storing downloaded S3 data in files.
    if (file.exists()) {
        boolean success = file.delete();
        if (!success) {
            log.error("Failed to delete the file {}. Expect long stack trace with FileAlreadyExistsException below",
                    file);
        }
    }
    var future = s3Client.getObject(
            GetObjectRequest.builder().bucket(downloaderProperties.getCommon().getBucketName()).key(s3ObjectKey)
                    .requestPayer(RequestPayer.REQUESTER)
                    .build(),
            AsyncResponseTransformer.toFile(file));
    return new PendingDownload(future, file, s3ObjectKey);
}
 

@Override
public String get() {
    GetObjectRequest.Builder requestBuilder = GetObjectRequest.builder().bucket(bucket).key(key);
    if (cachedObject != null) {
        requestBuilder.ifNoneMatch(cachedObject.eTag);
    }

    try {
        ResponseBytes<GetObjectResponse> responseBytes = s3Client.getObjectAsBytes(requestBuilder.build());
        return cacheAndReturnObject(responseBytes);
    } catch (S3Exception s3Exception) {
        if (s3Exception.statusCode() == 304) {
            return cachedObject.content;
        }

        throw s3Exception;
    }
}
 

@Override
public synchronized void getBundleVersionContent(final BundleVersionCoordinate versionCoordinate, final OutputStream outputStream)
        throws BundlePersistenceException {
    final String key = getKey(versionCoordinate);
    LOGGER.debug("Retrieving bundle version from S3 bucket '{}' with key '{}'", new Object[]{s3BucketName, key});

    final GetObjectRequest request = GetObjectRequest.builder()
            .bucket(s3BucketName)
            .key(key)
            .build();

    try (final ResponseInputStream<GetObjectResponse> response = s3Client.getObject(request)) {
        IoUtils.copy(response, outputStream);
        LOGGER.debug("Successfully retrieved bundle version from S3 bucket '{}' with key '{}'", new Object[]{s3BucketName, key});
    } catch (Exception e) {
        throw new BundlePersistenceException("Error retrieving bundle version from S3 due to: " + e.getMessage(), e);
    }
}
 

@GetMapping(path="/{filekey}")
public Mono<ResponseEntity<Flux<ByteBuffer>>> downloadFile(@PathVariable("filekey") String filekey) {
    
    GetObjectRequest request = GetObjectRequest.builder()
      .bucket(s3config.getBucket())
      .key(filekey)
      .build();
    
    return Mono.fromFuture(s3client.getObject(request,new FluxResponseProvider()))
      .map( (response) -> {
        checkResult(response.sdkResponse);
        String filename = getMetadataItem(response.sdkResponse,"filename",filekey);            

        log.info("[I65] filename={}, length={}",filename, response.sdkResponse.contentLength() );
        
        return ResponseEntity.ok()
          .header(HttpHeaders.CONTENT_TYPE, response.sdkResponse.contentType())
          .header(HttpHeaders.CONTENT_LENGTH, Long.toString(response.sdkResponse.contentLength()))
          .header(HttpHeaders.CONTENT_DISPOSITION, "attachment; filename=\"" + filename + "\"")
          .body(response.flux);
      });
}
 

@Test
public void modifyHttpResponseContent_getObjectRequest_responseDoesNotContainChecksum_shouldNotModify() {
    ExecutionAttributes executionAttributes = getExecutionAttributesWithChecksumDisabled();
    SdkHttpResponse sdkHttpResponse = SdkHttpResponse.builder()
                                                     .putHeader(CONTENT_LENGTH_HEADER, "100")
                                                     .build();
    Context.ModifyHttpResponse modifyHttpResponse =
        InterceptorTestUtils.modifyHttpResponse(GetObjectRequest.builder().build(), sdkHttpResponse);
    Optional<InputStream> publisher = interceptor.modifyHttpResponseContent(modifyHttpResponse,
                                                                            executionAttributes);
    assertThat(publisher).isEqualTo(modifyHttpResponse.responseBody());
}
 

@Test
public void modifyAsyncHttpResponseContent_getObjectRequest_checksumEnabled_shouldWrapChecksumValidatingPublisher() {
    SdkHttpResponse sdkHttpResponse = getSdkHttpResponseWithChecksumHeader();
    Context.ModifyHttpResponse modifyHttpResponse =
        InterceptorTestUtils.modifyHttpResponse(GetObjectRequest.builder().build(), sdkHttpResponse);
    Optional<Publisher<ByteBuffer>> publisher = interceptor.modifyAsyncHttpResponseContent(modifyHttpResponse,
                                                                                           getExecutionAttributes());
    assertThat(publisher.get()).isExactlyInstanceOf(ChecksumValidatingPublisher.class);
}
 

@Test
public void modifyAsyncHttpResponseContent_getObjectRequest_responseDoesNotContainChecksum_shouldNotModify() {
    ExecutionAttributes executionAttributes = getExecutionAttributesWithChecksumDisabled();
    SdkHttpResponse sdkHttpResponse = SdkHttpResponse.builder()
                                                     .putHeader(CONTENT_LENGTH_HEADER, "100")
                                                     .build();
    Context.ModifyHttpResponse modifyHttpResponse =
        InterceptorTestUtils.modifyHttpResponse(GetObjectRequest.builder().build(), sdkHttpResponse);
    Optional<Publisher<ByteBuffer>> publisher = interceptor.modifyAsyncHttpResponseContent(modifyHttpResponse,
                                                                                           executionAttributes);
    assertThat(publisher).isEqualTo(modifyHttpResponse.responsePublisher());
}
 

/** {@inheritDoc} */
@Override
protected void download(Path tmp, URI baseUri, Artifact.Item item, Progress progress)
        throws IOException {
    String key = item.getUri();
    logger.debug("Downloading artifact from: s3://{}/{} ...", bucket, key);
    GetObjectRequest req = GetObjectRequest.builder().bucket(bucket).key(key).build();
    try (ResponseInputStream<GetObjectResponse> is = client.getObject(req)) {
        save(is, tmp, baseUri, item, progress);
    }
}
 

@Test
public void nonGetBucketPolicyResponse_ShouldNotModifyResponse() {
    GetObjectRequest request = GetObjectRequest.builder().build();
    Context.ModifyHttpResponse context = modifyHttpResponseContent(request, SdkHttpResponse.builder().statusCode(200).build());
    Optional<InputStream> inputStream = interceptor.modifyHttpResponseContent(context, new ExecutionAttributes());
    assertThat(inputStream).isEqualTo(context.responseBody());
}
 

@Test
public void modifyHttpRequest_doesNotSetExpect_whenSdkRequestIsNotPutObject() {

    final SdkHttpRequest modifiedRequest = interceptor.modifyHttpRequest(modifyHttpRequestContext(GetObjectRequest.builder().build()),
                                                                         new ExecutionAttributes());

    assertThat(modifiedRequest.firstMatchingHeader("Expect")).isNotPresent();
}
 

@Test
public void modifyHttpContent_nonPutObjectRequest_shouldNotModify() {
    ExecutionAttributes executionAttributes = getExecutionAttributes();
    Context.ModifyHttpRequest modifyHttpRequest =
        InterceptorTestUtils.modifyHttpRequestContext(GetObjectRequest.builder().build());
    Optional<RequestBody> requestBody = interceptor.modifyHttpContent(modifyHttpRequest,
                                                                      executionAttributes);

    assertThat(requestBody).isEqualTo(modifyHttpRequest.requestBody());
}
 

@Test
public void modifyHttpContent_putObjectRequest_checksumDisabled_shouldNotModify() {
    ExecutionAttributes executionAttributes = getExecutionAttributesWithChecksumDisabled();
    Context.ModifyHttpRequest modifyHttpRequest =
        InterceptorTestUtils.modifyHttpRequestContext(GetObjectRequest.builder().build());

    Optional<RequestBody> requestBody = interceptor.modifyHttpContent(modifyHttpRequest,
                                                                      executionAttributes);
    assertThat(requestBody).isEqualTo(modifyHttpRequest.requestBody());
}
 

@Test
public void modifyHttpResponseContent_getObjectRequest_checksumEnabled_shouldWrapChecksumValidatingPublisher() {
    SdkHttpResponse sdkHttpResponse = getSdkHttpResponseWithChecksumHeader();
    Context.ModifyHttpResponse modifyHttpResponse =
        InterceptorTestUtils.modifyHttpResponse(GetObjectRequest.builder().build(), sdkHttpResponse);
    Optional<InputStream> publisher = interceptor.modifyHttpResponseContent(modifyHttpResponse,
                                                                            getExecutionAttributes());
    assertThat(publisher.get()).isExactlyInstanceOf(ChecksumValidatingInputStream.class);
}
 

@Override
public InputStream open(URI uri)
    throws IOException {
  try {
    String path = sanitizePath(uri.getPath());
    GetObjectRequest getObjectRequest = GetObjectRequest.builder().bucket(uri.getHost()).key(path).build();

    ResponseBytes responseBytes = _s3Client.getObjectAsBytes(getObjectRequest);
    return responseBytes.asInputStream();
  } catch (S3Exception e) {
    throw e;
  }
}
 

@Test
public void modifyHttpRequest_getObjectTrailingChecksumEnabled_shouldAddTrailingChecksumHeader() {
    Context.ModifyHttpRequest modifyHttpRequestContext =
        modifyHttpRequestContext(GetObjectRequest.builder().build());

    SdkHttpRequest sdkHttpRequest = interceptor.modifyHttpRequest(modifyHttpRequestContext, new ExecutionAttributes());

    assertThat(sdkHttpRequest.headers().get(ENABLE_CHECKSUM_REQUEST_HEADER)).containsOnly(ENABLE_MD5_CHECKSUM_HEADER_VALUE);
}
 

@Test
public void modifyHttpRequest_getObjectTrailingChecksumDisabled_shouldNotModifyHttpRequest() {
    Context.ModifyHttpRequest modifyHttpRequestContext =
        modifyHttpRequestContext(GetObjectRequest.builder().build());

    SdkHttpRequest sdkHttpRequest = interceptor.modifyHttpRequest(modifyHttpRequestContext,
                                                                  new ExecutionAttributes().putAttribute(AwsSignerExecutionAttribute.SERVICE_CONFIG,
                                                                                                         S3Configuration.builder().checksumValidationEnabled(false).build()));

    assertThat(sdkHttpRequest).isEqualToComparingFieldByField(modifyHttpRequestContext.httpRequest());
}
 

@Override
public void copyToLocalFile(URI srcUri, File dstFile)
    throws Exception {
  LOGGER.info("Copy {} to local {}", srcUri, dstFile.getAbsolutePath());
  URI base = getBase(srcUri);
  String prefix = sanitizePath(base.relativize(srcUri).getPath());
  GetObjectRequest getObjectRequest = GetObjectRequest.builder().bucket(srcUri.getHost()).key(prefix).build();

  _s3Client.getObject(getObjectRequest, ResponseTransformer.toFile(dstFile));
}
 

public static void getObjectBytes (S3Client s3, String bucketName, String keyName, String path ) {

        try {
            // create a GetObjectRequest instance
            GetObjectRequest objectRequest = GetObjectRequest
                    .builder()
                    .key(keyName)
                    .bucket(bucketName)
                    .build();

            // get the byte[] this AWS S3 object
            ResponseBytes<GetObjectResponse> objectBytes = s3.getObjectAsBytes(objectRequest);
            byte[] data = objectBytes.asByteArray();

            //Write the data to a local file
            File myFile = new File(path );
            OutputStream os = new FileOutputStream(myFile);
            os.write(data);
            System.out.println("Successfully obtained bytes from an S3 object");

            // Close the file
            os.close();
        } catch (IOException ex) {
            ex.printStackTrace();
        } catch (S3Exception e) {
          System.err.println(e.awsErrorDetails().errorMessage());
           System.exit(1);
        }
        // snippet-end:[s3.java2.getobjectdata.main]
    }