software.amazon.awssdk.services.s3.model.GetObjectResponse Java Examples

/**
 * Read directly from the remote service All reading goes through here or readToByteChannel;
 *
 * 1. https://docs.aws.amazon.com/AmazonS3/latest/dev/RetrievingObjectUsingJava.html
 *
 * @param pos start here in the file
 * @param buff put data into this buffer
 * @param offset buffer offset
 * @param len this number of bytes
 * @return actual number of bytes read
 * @throws IOException on io error
 */
@Override
public int readRemote(long pos, byte[] buff, int offset, int len) throws IOException {

  String range = String.format("bytes=%d-%d", pos, pos + len);
  GetObjectRequest rangeObjectRequest =
      GetObjectRequest.builder().bucket(uri.getBucket()).key(uri.getKey()).range(range).build();

  ResponseBytes<GetObjectResponse> objectPortion = client.getObjectAsBytes(rangeObjectRequest);

  int bytes;
  int totalBytes = 0;
  // read response into buff
  try (InputStream objectData = objectPortion.asInputStream()) {
    bytes = objectData.read(buff, offset + totalBytes, len - totalBytes);
    while ((bytes > 0) && ((len - totalBytes) > 0)) {
      totalBytes += bytes;
      bytes = objectData.read(buff, offset + totalBytes, len - totalBytes);
    }
  }
  return totalBytes;
}
 

@Test
public void modifyResponse_getObjectResponseContainsChecksumHeader_shouldModifyResponse() {
    long contentLength = 50;
    GetObjectResponse response = GetObjectResponse.builder().contentLength(contentLength).build();
    Context.ModifyResponse modifyResponseContext = modifyResponseContext(
        GetObjectRequest.builder().build(),
        response,
        SdkHttpFullResponse.builder()
                           .putHeader(CHECKSUM_ENABLED_RESPONSE_HEADER, ENABLE_MD5_CHECKSUM_HEADER_VALUE).build());

    GetObjectResponse actualResponse = (GetObjectResponse) interceptor.modifyResponse(modifyResponseContext,
                                                                                      new ExecutionAttributes());

    assertThat(actualResponse).isNotEqualTo(response);
    assertThat(actualResponse.contentLength()).isEqualTo(contentLength - S3_MD5_CHECKSUM_LENGTH);
}
 

/**
 * S3 currently returns content-range in two possible headers: Content-Range or x-amz-content-range based on the x-amz-te
 * in the request. This will check the x-amz-content-range if the modeled header (Content-Range) wasn't populated.
 */
private SdkResponse fixContentRange(SdkResponse sdkResponse, SdkHttpResponse httpResponse) {
    // Use the modeled content range header, if the service returned it.
    GetObjectResponse getObjectResponse = (GetObjectResponse) sdkResponse;
    if (getObjectResponse.contentRange() != null) {
        return getObjectResponse;
    }

    // If the service didn't use the modeled content range header, check the x-amz-content-range header.
    Optional<String> xAmzContentRange = httpResponse.firstMatchingHeader("x-amz-content-range");
    if (!xAmzContentRange.isPresent()) {
        return getObjectResponse;
    }

    return getObjectResponse.copy(r -> r.contentRange(xAmzContentRange.get()));
}
 

/**
 * Subtract the contentLength of {@link GetObjectResponse} if trailing checksums is enabled.
 */
@Override
public SdkResponse modifyResponse(Context.ModifyResponse context, ExecutionAttributes executionAttributes) {
    SdkResponse response = context.response();
    SdkHttpResponse httpResponse = context.httpResponse();

    if (getObjectChecksumEnabledPerResponse(context.request(), httpResponse)) {
        GetObjectResponse getResponse = (GetObjectResponse) response;
        Long contentLength = getResponse.contentLength();
        Validate.notNull(contentLength, "Service returned null 'Content-Length'.");
        return getResponse.toBuilder()
                          .contentLength(contentLength - S3_MD5_CHECKSUM_LENGTH)
                          .build();
    }

    return response;
}
 

@Override
public synchronized void getBundleVersionContent(final BundleVersionCoordinate versionCoordinate, final OutputStream outputStream)
        throws BundlePersistenceException {
    final String key = getKey(versionCoordinate);
    LOGGER.debug("Retrieving bundle version from S3 bucket '{}' with key '{}'", new Object[]{s3BucketName, key});

    final GetObjectRequest request = GetObjectRequest.builder()
            .bucket(s3BucketName)
            .key(key)
            .build();

    try (final ResponseInputStream<GetObjectResponse> response = s3Client.getObject(request)) {
        IoUtils.copy(response, outputStream);
        LOGGER.debug("Successfully retrieved bundle version from S3 bucket '{}' with key '{}'", new Object[]{s3BucketName, key});
    } catch (Exception e) {
        throw new BundlePersistenceException("Error retrieving bundle version from S3 due to: " + e.getMessage(), e);
    }
}
 

@Override
public String get() {
    GetObjectRequest.Builder requestBuilder = GetObjectRequest.builder().bucket(bucket).key(key);
    if (cachedObject != null) {
        requestBuilder.ifNoneMatch(cachedObject.eTag);
    }

    try {
        ResponseBytes<GetObjectResponse> responseBytes = s3Client.getObjectAsBytes(requestBuilder.build());
        return cacheAndReturnObject(responseBytes);
    } catch (S3Exception s3Exception) {
        if (s3Exception.statusCode() == 304) {
            return cachedObject.content;
        }

        throw s3Exception;
    }
}
 

@SuppressWarnings("unchecked")
private LinkedHashMap<String, JSONArray> fetchAndParseSnapshotFileFromS3(String snapshotFileName) {
    GetObjectRequest request = GetObjectRequest.builder().bucket(INTEGRATION_TEST_BUCKET).key(snapshotFileName).build();

    ResponseInputStream<GetObjectResponse> responseInputStream = s3Client.getObject(request);
    try (
            BufferedInputStream bufferedInputStream = new BufferedInputStream(responseInputStream);
            ZipInputStream zipInputStream = new ZipInputStream(bufferedInputStream)
    ) {
        zipInputStream.getNextEntry();
        return (LinkedHashMap<String, JSONArray>) Configuration.defaultConfiguration().jsonProvider().parse(zipInputStream, Charsets.UTF_8.name());
    } catch (IOException e) {
        throw new RuntimeException(e);
    }
}
 

@SuppressWarnings("unchecked")
private LinkedHashMap<String, JSONArray> fetchAndParseSnapshotFileFromS3(String snapshotFileName) {
    GetObjectRequest request = GetObjectRequest.builder().bucket(INTEGRATION_TEST_BUCKET).key(snapshotFileName).build();

    ResponseInputStream<GetObjectResponse> responseInputStream = s3Client.getObject(request);
    try (
            BufferedInputStream bufferedInputStream = new BufferedInputStream(responseInputStream);
            ZipInputStream zipInputStream = new ZipInputStream(bufferedInputStream)
    ) {
        zipInputStream.getNextEntry();
        return (LinkedHashMap<String, JSONArray>) Configuration.defaultConfiguration().jsonProvider().parse(zipInputStream, Charsets.UTF_8.name());
    } catch (IOException e) {
        throw new RuntimeException(e);
    }
}
 

public static void main(String[] args) {

        final String USAGE = "\n" +
                "Usage:\n" +
                "    S3AsyncOps <bucketname> <objectname> <path>\n\n" +
                "Where:\n" +
                "    bucketname - the name of the bucket (i.e., bucket1)\n\n" +
                "    objectname - the name pf the object object (i.e., book.pdf)\n" +
                "    path - the local path to the file (i.e., C:\\AWS\\book.pdf)\n" +
                "Example:\n" +
                "    bucket1 book.pdf  C:\\AWS\\book.pdf\n";

         String bucketName = args[0];
         String objectKey = args[1];
         String path = args[2];

        S3AsyncClient client = S3AsyncClient.create();
        final CompletableFuture<GetObjectResponse> futureGet = client.getObject(
                GetObjectRequest.builder()
                        .bucket(bucketName)
                        .key(objectKey)
                        .build(),
                AsyncResponseTransformer.toFile(Paths.get(path)));
        futureGet.whenComplete((resp, err) -> {
            try {
                if (resp != null) {
                    System.out.println(resp);
                } else {
                    // Handle error
                    err.printStackTrace();
                }
            } finally {
                // Lets the application shut down. Only close the client when you are completely done with it
                client.close();
            }
        });

        futureGet.join();
    }
 

public static void getObjectBytes (S3Client s3, String bucketName, String keyName, String path ) {

        try {
            // create a GetObjectRequest instance
            GetObjectRequest objectRequest = GetObjectRequest
                    .builder()
                    .key(keyName)
                    .bucket(bucketName)
                    .build();

            // get the byte[] this AWS S3 object
            ResponseBytes<GetObjectResponse> objectBytes = s3.getObjectAsBytes(objectRequest);
            byte[] data = objectBytes.asByteArray();

            //Write the data to a local file
            File myFile = new File(path );
            OutputStream os = new FileOutputStream(myFile);
            os.write(data);
            System.out.println("Successfully obtained bytes from an S3 object");

            // Close the file
            os.close();
        } catch (IOException ex) {
            ex.printStackTrace();
        } catch (S3Exception e) {
          System.err.println(e.awsErrorDetails().errorMessage());
           System.exit(1);
        }
        // snippet-end:[s3.java2.getobjectdata.main]
    }
 

/**
 * Lookup a metadata key in a case-insensitive way.
 * @param sdkResponse
 * @param key
 * @param defaultValue
 * @return
 */
private String getMetadataItem(GetObjectResponse sdkResponse, String key, String defaultValue) {
    for( Entry<String, String> entry : sdkResponse.metadata().entrySet()) {
        if ( entry.getKey().equalsIgnoreCase(key)) {
            return entry.getValue();
        }
    }
    return defaultValue;
}
 

@Test
public void modifyResponse_getObjectResponseNoChecksumHeader_shouldNotModifyResponse() {
    long contentLength = 50;
    GetObjectResponse response = GetObjectResponse.builder().contentLength(contentLength).build();
    Context.ModifyResponse modifyResponseContext = modifyResponseContext(
        GetObjectRequest.builder().build(),
        response,
        SdkHttpFullResponse.builder().build());

    GetObjectResponse actualResponse = (GetObjectResponse) interceptor.modifyResponse(modifyResponseContext,
                                                                                      new ExecutionAttributes());
    assertThat(actualResponse).isEqualTo(response);
}
 

private static void checkResult(GetObjectResponse response) {
    SdkHttpResponse sdkResponse = response.sdkHttpResponse();
    if ( sdkResponse != null && sdkResponse.isSuccessful()) {
        return;
    }
    
    throw new DownloadFailedException(response);
}
 

/** {@inheritDoc} */
@Override
protected void download(Path tmp, URI baseUri, Artifact.Item item, Progress progress)
        throws IOException {
    String key = item.getUri();
    logger.debug("Downloading artifact from: s3://{}/{} ...", bucket, key);
    GetObjectRequest req = GetObjectRequest.builder().bucket(bucket).key(key).build();
    try (ResponseInputStream<GetObjectResponse> is = client.getObject(req)) {
        save(is, tmp, baseUri, item, progress);
    }
}
 

@Test
public void asyncGetObject_resetsChecksumOnRetry() {
    stubSuccessAfterOneRetry(r -> r.withHeader("ETag", bodyEtag)
                                   .withHeader("x-amz-transfer-encoding", "append-md5")
                                   .withHeader("content-length", Integer.toString(bodyWithTrailingChecksum.length))
                                   .withBody(bodyWithTrailingChecksum));

    ResponseBytes<GetObjectResponse> response = s3AsyncClient.getObject(r -> r.bucket("foo").key("bar"), toBytes()).join();
    assertThat(response.response().eTag()).isEqualTo(bodyEtag);
    assertThat(response.asByteArray()).isEqualTo(body);
}
 

@Test
public void syncGetObject_resetsChecksumOnRetry() {
    stubSuccessAfterOneRetry(r -> r.withHeader("ETag", bodyEtag)
                                   .withHeader("x-amz-transfer-encoding", "append-md5")
                                   .withHeader("content-length", Integer.toString(bodyWithTrailingChecksum.length))
                                   .withBody(bodyWithTrailingChecksum));

    ResponseBytes<GetObjectResponse> response = s3Client.getObjectAsBytes(r -> r.bucket("foo").key("bar"));
    assertThat(response.response().eTag()).isEqualTo(bodyEtag);
    assertThat(response.asByteArray()).isEqualTo(body);
}
 

@Test
public void slowTransformer_shouldThrowApiCallTimeoutException() {
    SlowResponseTransformer<GetObjectResponse> handler =
        new SlowResponseTransformer<>();
    assertThatThrownBy(() -> s3ClientWithTimeout.getObject(getObjectRequest(), handler).join())
            .hasCauseInstanceOf(ApiCallTimeoutException.class);
    assertThat(handler.currentCallCount()).isEqualTo(1);
    assertThat(handler.exceptionOccurred).isEqualTo(true);
}
 

@Override
public SdkResponse modifyResponse(Context.ModifyResponse context, ExecutionAttributes executionAttributes) {
    return ((GetObjectResponse) context.response())
            .toBuilder()
            .metadata(ImmutableMap.of("x-amz-assert", "injected-value"))
            .build();
}
 

@Test
public void customResponseHandler_InterceptorRecievesResponsePojo() throws Exception {
    final CompletableFuture<String> cf = new CompletableFuture<>();
    try (S3AsyncClient asyncWithInterceptor = createClientWithInterceptor(new AssertingExecutionInterceptor())) {
        String result = asyncWithInterceptor
                .getObject(getObjectRequest, new AsyncResponseTransformer<GetObjectResponse, String>() {

                    @Override
                    public CompletableFuture<String> prepare() {
                        return cf;
                    }

                    @Override
                    public void onResponse(GetObjectResponse response) {
                        // POJO returned by modifyResponse should be delivered to the AsyncResponseTransformer
                        assertThat(response.metadata()).hasEntrySatisfying("x-amz-assert",
                                                                           s -> assertThat(s).isEqualTo("injected-value"));
                    }

                    @Override
                    public void onStream(SdkPublisher<ByteBuffer> publisher) {
                        publisher.subscribe(new SimpleSubscriber(b -> {
                        }) {
                            @Override
                            public void onComplete() {
                                super.onComplete();
                                cf.complete("result");
                            }
                        });
                    }

                    @Override
                    public void exceptionOccurred(Throwable throwable) {
                        cf.completeExceptionally(throwable);
                    }
                }).join();
        assertThat(result).isEqualTo("result");
    }
}
 

@Test
public void toFile() throws Exception {
    Path path = RandomTempFile.randomUncreatedFile().toPath();
    GetObjectResponse response = null;
    try {
        response = s3Async.getObject(getObjectRequest, path).join();
    } finally {
        assertEquals(Long.valueOf(path.toFile().length()), response.contentLength());
        path.toFile().delete();
    }
}
 

@Test
public void handlerThrowsRetryableException_RetriedUpToLimit() throws Exception {
    RequestCountingResponseTransformer<GetObjectResponse, ?> handler = new RequestCountingResponseTransformer<>(
        (resp, in) -> {
            throw RetryableException.builder().build();
        });
    assertThatThrownBy(() -> s3.getObject(getObjectRequest(), handler))
        .isInstanceOf(SdkClientException.class);
    assertThat(handler.currentCallCount()).isEqualTo(4);
}
 

@Test
public void handlerThrowsNonRetryableException_RequestNotRetried() throws Exception {
    RequestCountingResponseTransformer<GetObjectResponse, ?> handler = new RequestCountingResponseTransformer<>(
        (resp, in) -> {
            throw NonRetryableException.builder().build();
        });
    assertThatThrownBy(() -> s3.getObject(getObjectRequest(), handler))
        .isInstanceOf(SdkClientException.class);
    assertThat(handler.currentCallCount()).isEqualTo(1);
}
 

@GET
@Path("download/{objectKey}")
@Produces(MediaType.APPLICATION_OCTET_STREAM)
public Response downloadFile(@PathParam("objectKey") String objectKey) {
    ByteArrayOutputStream baos = new ByteArrayOutputStream();
    GetObjectResponse object = s3.getObject(buildGetRequest(objectKey), ResponseTransformer.toOutputStream(baos));

    ResponseBuilder response = Response.ok((StreamingOutput) output -> baos.writeTo(output));
    response.header("Content-Disposition", "attachment;filename=" + objectKey);
    response.header("Content-Type", object.contentType());
    return response.build();
}
 

@Test
public void toFile() throws Exception {
    Path path = RandomTempFile.randomUncreatedFile().toPath();

    GetObjectResponse response = null;
    try {
        response = s3.getObject(getObjectRequest, path);
    } finally {
        assertEquals(Long.valueOf(path.toFile().length()), response.contentLength());
        path.toFile().delete();
    }
}
 

@Test
public void testInjectCredentialsAndInvokeV2InputStream() {

    final LoggerProxy loggerProxy = mock(LoggerProxy.class);
    final Credentials credentials = new Credentials("accessKeyId", "secretAccessKey", "sessionToken");
    final ResponseInputStream<?> responseInputStream = mock(ResponseInputStream.class);

    final AmazonWebServicesClientProxy proxy = new AmazonWebServicesClientProxy(loggerProxy, credentials, () -> 1000L);

    final software.amazon.awssdk.services.s3.model.GetObjectRequest wrappedRequest = mock(
        software.amazon.awssdk.services.s3.model.GetObjectRequest.class);

    final software.amazon.awssdk.services.s3.model.GetObjectRequest.Builder builder = mock(
        software.amazon.awssdk.services.s3.model.GetObjectRequest.Builder.class);
    when(builder.overrideConfiguration(any(AwsRequestOverrideConfiguration.class))).thenReturn(builder);
    when(builder.build()).thenReturn(wrappedRequest);
    final software.amazon.awssdk.services.s3.model.GetObjectRequest request = mock(
        software.amazon.awssdk.services.s3.model.GetObjectRequest.class);
    when(request.toBuilder()).thenReturn(builder);

    final S3Client client = mock(S3Client.class);

    doReturn(responseInputStream).when(client)
        .getObject(any(software.amazon.awssdk.services.s3.model.GetObjectRequest.class));

    final ResponseInputStream<
        GetObjectResponse> result = proxy.injectCredentialsAndInvokeV2InputStream(request, client::getObject);

    // verify request is rebuilt for injection
    verify(request).toBuilder();

    // verify the wrapped request is sent over the initiate
    verify(client).getObject(wrappedRequest);

    // ensure the return type matches
    assertThat(result).isEqualTo(responseInputStream);
}
 

@Test
public void testInjectCredentialsAndInvokeV2Bytes() {

    final LoggerProxy loggerProxy = mock(LoggerProxy.class);
    final Credentials credentials = new Credentials("accessKeyId", "secretAccessKey", "sessionToken");
    final ResponseBytes<?> responseBytes = mock(ResponseBytes.class);

    final AmazonWebServicesClientProxy proxy = new AmazonWebServicesClientProxy(loggerProxy, credentials, () -> 1000L);

    final software.amazon.awssdk.services.s3.model.GetObjectRequest wrappedRequest = mock(
        software.amazon.awssdk.services.s3.model.GetObjectRequest.class);

    final software.amazon.awssdk.services.s3.model.GetObjectRequest.Builder builder = mock(
        software.amazon.awssdk.services.s3.model.GetObjectRequest.Builder.class);
    when(builder.overrideConfiguration(any(AwsRequestOverrideConfiguration.class))).thenReturn(builder);
    when(builder.build()).thenReturn(wrappedRequest);
    final software.amazon.awssdk.services.s3.model.GetObjectRequest request = mock(
        software.amazon.awssdk.services.s3.model.GetObjectRequest.class);
    when(request.toBuilder()).thenReturn(builder);

    final S3Client client = mock(S3Client.class);

    doReturn(responseBytes).when(client)
        .getObjectAsBytes(any(software.amazon.awssdk.services.s3.model.GetObjectRequest.class));

    final ResponseBytes<
        GetObjectResponse> result = proxy.injectCredentialsAndInvokeV2Bytes(request, client::getObjectAsBytes);

    // verify request is rebuilt for injection
    verify(request).toBuilder();

    // verify the wrapped request is sent over the initiate
    verify(client).getObjectAsBytes(wrappedRequest);

    // ensure the return type matches
    assertThat(result).isEqualTo(responseBytes);
}
 

@Test
public void toInputStream_loadFromProperties() throws IOException {
    s3.putObject(b -> b.bucket(BUCKET).key(PROPERTY_KEY), RequestBody.fromString("test: test"));
    try (ResponseInputStream<GetObjectResponse> object = s3.getObject(b -> b.bucket(BUCKET).key(PROPERTY_KEY),
                                                                      ResponseTransformer.toInputStream())) {
        Properties properties = new Properties();
        properties.load(object);
        assertThat(properties.getProperty("test")).isEqualTo("test");
    }
}
 

@GET
@Path("blocking")
@Produces(TEXT_PLAIN)
public String testBlockingS3() {
    LOG.info("Testing S3 Blocking client with bucket: " + SYNC_BUCKET);

    String keyValue = UUID.randomUUID().toString();
    String result = null;

    try {
        if (S3Utils.createBucket(s3Client, SYNC_BUCKET)) {
            if (s3Client.putObject(S3Utils.createPutRequest(SYNC_BUCKET, keyValue),
                    RequestBody.fromString(SAMPLE_S3_OBJECT)) != null) {
                ByteArrayOutputStream baos = new ByteArrayOutputStream();

                GetObjectResponse object = s3Client.getObject(S3Utils.createGetRequest(SYNC_BUCKET, keyValue),
                        ResponseTransformer.toOutputStream(baos));

                if (object != null) {
                    result = metadata(object) + "+" + baos.toString();
                }
            }
        }
    } catch (Exception ex) {
        LOG.error("Error during S3 operations.", ex);
        return "ERROR";
    }
    return result;
}
 

@Override
public SdkResponse modifyResponse(Context.ModifyResponse context, ExecutionAttributes executionAttributes) {
    SdkResponse response = context.response();
    if (!(response instanceof GetObjectResponse)) {
        return response;
    }

    return fixContentRange(response, context.httpResponse());
}
 

private String cacheAndReturnObject(ResponseBytes<GetObjectResponse> responseBytes) {
    String eTag = responseBytes.response().eTag();
    String content = responseBytes.asUtf8String();

    cachedObject = new CachedObject(content, eTag);
    return content;
}