software.amazon.awssdk.core.ResponseBytes Java Examples

/**
 * Read directly from the remote service All reading goes through here or readToByteChannel;
 *
 * 1. https://docs.aws.amazon.com/AmazonS3/latest/dev/RetrievingObjectUsingJava.html
 *
 * @param pos start here in the file
 * @param buff put data into this buffer
 * @param offset buffer offset
 * @param len this number of bytes
 * @return actual number of bytes read
 * @throws IOException on io error
 */
@Override
public int readRemote(long pos, byte[] buff, int offset, int len) throws IOException {

  String range = String.format("bytes=%d-%d", pos, pos + len);
  GetObjectRequest rangeObjectRequest =
      GetObjectRequest.builder().bucket(uri.getBucket()).key(uri.getKey()).range(range).build();

  ResponseBytes<GetObjectResponse> objectPortion = client.getObjectAsBytes(rangeObjectRequest);

  int bytes;
  int totalBytes = 0;
  // read response into buff
  try (InputStream objectData = objectPortion.asInputStream()) {
    bytes = objectData.read(buff, offset + totalBytes, len - totalBytes);
    while ((bytes > 0) && ((len - totalBytes) > 0)) {
      totalBytes += bytes;
      bytes = objectData.read(buff, offset + totalBytes, len - totalBytes);
    }
  }
  return totalBytes;
}
 

public <RequestT extends AwsRequest, ResultT extends AwsResponse>
    ResponseBytes<ResultT>
    injectCredentialsAndInvokeV2Bytes(final RequestT request,
                                      final Function<RequestT, ResponseBytes<ResultT>> requestFunction) {

    AwsRequestOverrideConfiguration overrideConfiguration = AwsRequestOverrideConfiguration.builder()
        .credentialsProvider(v2CredentialsProvider).build();

    @SuppressWarnings("unchecked")
    RequestT wrappedRequest = (RequestT) request.toBuilder().overrideConfiguration(overrideConfiguration).build();

    try {
        ResponseBytes<ResultT> response = requestFunction.apply(wrappedRequest);
        logRequestMetadataV2(request, response.response());
        return response;
    } catch (final Throwable e) {
        loggerProxy.log(String.format("Failed to execute remote function: {%s}", e.getMessage()));
        throw e;
    }
}
 

@Override
public String get() {
    GetObjectRequest.Builder requestBuilder = GetObjectRequest.builder().bucket(bucket).key(key);
    if (cachedObject != null) {
        requestBuilder.ifNoneMatch(cachedObject.eTag);
    }

    try {
        ResponseBytes<GetObjectResponse> responseBytes = s3Client.getObjectAsBytes(requestBuilder.build());
        return cacheAndReturnObject(responseBytes);
    } catch (S3Exception s3Exception) {
        if (s3Exception.statusCode() == 304) {
            return cachedObject.content;
        }

        throw s3Exception;
    }
}
 

@Test
public void completionWithNioThreadWorksCorrectly() {
    stubFor(post(urlPathEqualTo(STREAMING_OUTPUT_PATH)).willReturn(aResponse().withStatus(200).withBody("test")));

    Executor mockExecutor = Mockito.spy(new SpyableExecutor());

    ProtocolRestJsonAsyncClient client =
            ProtocolRestJsonAsyncClient.builder()
                                       .region(Region.US_WEST_1)
                                       .endpointOverride(URI.create("http://localhost:" + wireMock.port()))
                                       .credentialsProvider(() -> AwsBasicCredentials.create("akid", "skid"))
                                       .asyncConfiguration(c -> c.advancedOption(FUTURE_COMPLETION_EXECUTOR, mockExecutor))
                                       .build();

    ResponseBytes<StreamingOutputOperationResponse> response =
            client.streamingOutputOperation(StreamingOutputOperationRequest.builder().build(),
                                            AsyncResponseTransformer.toBytes()).join();

    verify(mockExecutor).execute(any());

    byte[] arrayCopy = response.asByteArray();
    assertThat(arrayCopy).containsExactly('t', 'e', 's', 't');
}
 

@Test
public void bytesMethodConvertsCorrectly() {
    stubForSuccess();

    ResponseBytes<StreamingOutputOperationResponse> response =
            testClient().streamingOutputOperationAsBytes(StreamingOutputOperationRequest.builder().build());

    byte[] arrayCopy = response.asByteArray();
    assertThat(arrayCopy).containsExactly('t', 'e', 's', 't', ' ', -16, -97, -104, -126);
    arrayCopy[0] = 'X'; // Mutate the returned byte array to make sure it's a copy

    ByteBuffer buffer = response.asByteBuffer();
    assertThat(buffer.isReadOnly()).isTrue();
    assertThat(BinaryUtils.copyAllBytesFrom(buffer)).containsExactly('t', 'e', 's', 't', ' ', -16, -97, -104, -126);

    assertThat(response.asString(StandardCharsets.UTF_8)).isEqualTo("test \uD83D\uDE02");
    assertThat(response.asUtf8String()).isEqualTo("test \uD83D\uDE02");
}
 

@Test
public void asyncStreaming_shouldContainResponseMetadata() {
    stubResponseWithHeaders();
    CompletableFuture<ResponseBytes<StreamingOutputOperationResponse>> response =
        asyncClient.streamingOutputOperation(SdkBuilder::build, AsyncResponseTransformer.toBytes());
    verifyResponseMetadata(response.join().response());
}
 

@Test
public void streamingOperation_slowTransformer_shouldThrowApiCallAttemptTimeoutException() {
    stubFor(post(anyUrl())
                .willReturn(aResponse()
                                .withStatus(200).withFixedDelay(DELAY_BEFORE_API_CALL_ATTEMPT_TIMEOUT)));

    CompletableFuture<ResponseBytes<StreamingOutputOperationResponse>> future = client
        .streamingOutputOperation(
            StreamingOutputOperationRequest.builder().build(), new SlowResponseTransformer<>());

    assertThatThrownBy(future::join)
        .hasCauseInstanceOf(ApiCallAttemptTimeoutException.class);
}
 

@Override
public CompletableFuture<ResponseBytes<ResponseT>> prepare() {
    return delegate.prepare()
                   .thenApply(r -> {
                       try {
                           Thread.sleep(1_000);
                       } catch (InterruptedException e) {
                           e.printStackTrace();
                       }
                       return r;
                   });
}
 

@Test
public void testStreamingOutputOperation() {
    CompletableFuture<ResponseBytes<StreamingOutputOperationResponse>> responseFuture = client.streamingOutputOperation(r -> {
    }, AsyncResponseTransformer.toBytes());
    responseFuture.cancel(true);
    assertThat(executeFuture.isCompletedExceptionally()).isTrue();
    assertThat(executeFuture.isCancelled()).isTrue();
}
 

@Test
public void testStreamingOutputOperation() {
    CompletableFuture<ResponseBytes<StreamingOutputOperationResponse>> responseFuture = client.streamingOutputOperation(r -> {
    }, AsyncResponseTransformer.toBytes());
    responseFuture.cancel(true);
    assertThat(executeFuture.isCompletedExceptionally()).isTrue();
    assertThat(executeFuture.isCancelled()).isTrue();
}
 

@Test
public void syncStreaming_shouldContainResponseMetadata() {
    stubResponseWithHeaders();
    ResponseBytes<StreamingOutputOperationResponse> streamingOutputOperationResponseResponseBytes =
        client.streamingOutputOperation(SdkBuilder::build, ResponseTransformer.toBytes());
    verifyResponseMetadata(streamingOutputOperationResponseResponseBytes.response());
}
 

@Test
public void asyncStreamingMethodShouldContainSdkHttpDate() {

    stubWithHeaders(EXPECTED_HEADERS);
    ResponseBytes<StreamingOutputOperationResponse> responseBytes = asyncClient
        .streamingOutputOperation(SdkBuilder::build, AsyncResponseTransformer.toBytes()).join();
    StreamingOutputOperationResponse response = responseBytes.response();

    verifySdkHttpResponse(response);
    verifyResponseMetadata(response);
}
 

@Test
public void testStreamingOutputOperation() {
    CompletableFuture<ResponseBytes<StreamingOutputOperationResponse>> responseFuture = client.streamingOutputOperation(r -> {
    }, AsyncResponseTransformer.toBytes());
    responseFuture.cancel(true);
    assertThat(executeFuture.isCompletedExceptionally()).isTrue();
    assertThat(executeFuture.isCancelled()).isTrue();
}
 

/**
 * @return Simple method for streaming output operations to get the content as a byte buffer or other in-memory types.
 */
private MethodSpec bytesSimpleMethod(OperationModel opModel, TypeName responseType, ClassName requestType) {
    TypeName returnType = ParameterizedTypeName.get(ClassName.get(ResponseBytes.class), responseType);
    return MethodSpec.methodBuilder(opModel.getMethodName() + "AsBytes")
                     .returns(returnType)
                     .addModifiers(Modifier.PUBLIC, Modifier.DEFAULT)
                     .addParameter(requestType, opModel.getInput().getVariableName())
                     .addJavadoc(opModel.getDocs(model, ClientType.SYNC, SimpleMethodOverload.BYTES))
                     .addExceptions(getExceptionClasses(model, opModel))
                     .addStatement("return $L($L, $T.toBytes())", opModel.getMethodName(),
                                   opModel.getInput().getVariableName(),
                                   ClassName.get(ResponseTransformer.class))
                     .build();
}
 

/**
 * Creates a response transformer that loads all response content into memory, exposed as {@link ResponseBytes}. This allows
 * for conversion into a {@link String}, {@link ByteBuffer}, etc.
 *
 * @param <ResponseT> Type of unmarshalled response POJO.
 * @return The streaming response transformer that can be used on the client streaming method.
 */
static <ResponseT> ResponseTransformer<ResponseT, ResponseBytes<ResponseT>> toBytes() {
    return (response, inputStream) -> {
        try {
            InterruptMonitor.checkInterrupted();
            return ResponseBytes.fromByteArray(response, IoUtils.toByteArray(inputStream));
        } catch (IOException e) {
            throw RetryableException.builder().message("Failed to read response.").cause(e).build();
        }
    };
}
 

public static void getObjectBytes (S3Client s3, String bucketName, String keyName, String path ) {

        try {
            // create a GetObjectRequest instance
            GetObjectRequest objectRequest = GetObjectRequest
                    .builder()
                    .key(keyName)
                    .bucket(bucketName)
                    .build();

            // get the byte[] this AWS S3 object
            ResponseBytes<GetObjectResponse> objectBytes = s3.getObjectAsBytes(objectRequest);
            byte[] data = objectBytes.asByteArray();

            //Write the data to a local file
            File myFile = new File(path );
            OutputStream os = new FileOutputStream(myFile);
            os.write(data);
            System.out.println("Successfully obtained bytes from an S3 object");

            // Close the file
            os.close();
        } catch (IOException ex) {
            ex.printStackTrace();
        } catch (S3Exception e) {
          System.err.println(e.awsErrorDetails().errorMessage());
           System.exit(1);
        }
        // snippet-end:[s3.java2.getobjectdata.main]
    }
 

@Override
public InputStream open(URI uri)
    throws IOException {
  try {
    String path = sanitizePath(uri.getPath());
    GetObjectRequest getObjectRequest = GetObjectRequest.builder().bucket(uri.getHost()).key(path).build();

    ResponseBytes responseBytes = _s3Client.getObjectAsBytes(getObjectRequest);
    return responseBytes.asInputStream();
  } catch (S3Exception e) {
    throw e;
  }
}
 

@Test
public void syncStreamingShouldContainSdkHttpDate() {

    stubWithHeaders(EXPECTED_HEADERS);
    ResponseBytes<StreamingOutputOperationResponse> responseBytes = client
        .streamingOutputOperation(SdkBuilder::build, ResponseTransformer.toBytes());
    StreamingOutputOperationResponse response = responseBytes.response();

    verifySdkHttpResponse(response);
    verifyResponseMetadata(response);
}
 

@Test
public void byteMethodDownloadFailureRetries() {
    stubForRetriesTimeoutReadingFromStreams();

    ResponseBytes<StreamingOutputOperationResponse> response =
            testClient().streamingOutputOperationAsBytes(StreamingOutputOperationRequest.builder().build());

    assertThat(response.asUtf8String()).isEqualTo("retried");
}
 

@Test
public void asyncGetObject_resetsChecksumOnRetry() {
    stubSuccessAfterOneRetry(r -> r.withHeader("ETag", bodyEtag)
                                   .withHeader("x-amz-transfer-encoding", "append-md5")
                                   .withHeader("content-length", Integer.toString(bodyWithTrailingChecksum.length))
                                   .withBody(bodyWithTrailingChecksum));

    ResponseBytes<GetObjectResponse> response = s3AsyncClient.getObject(r -> r.bucket("foo").key("bar"), toBytes()).join();
    assertThat(response.response().eTag()).isEqualTo(bodyEtag);
    assertThat(response.asByteArray()).isEqualTo(body);
}
 

@Test
public void syncGetObject_resetsChecksumOnRetry() {
    stubSuccessAfterOneRetry(r -> r.withHeader("ETag", bodyEtag)
                                   .withHeader("x-amz-transfer-encoding", "append-md5")
                                   .withHeader("content-length", Integer.toString(bodyWithTrailingChecksum.length))
                                   .withBody(bodyWithTrailingChecksum));

    ResponseBytes<GetObjectResponse> response = s3Client.getObjectAsBytes(r -> r.bucket("foo").key("bar"));
    assertThat(response.response().eTag()).isEqualTo(bodyEtag);
    assertThat(response.asByteArray()).isEqualTo(body);
}
 

@Override
public CompletableFuture<ResponseBytes<ResponseT>> prepare() {
    return delegate.prepare()
            .thenApply(r -> {
                try {
                    Thread.sleep(2_000);
                } catch (InterruptedException e) {
                    e.printStackTrace();;
                }
                return r;
            });
}
 

private String cacheAndReturnObject(ResponseBytes<GetObjectResponse> responseBytes) {
    String eTag = responseBytes.response().eTag();
    String content = responseBytes.asUtf8String();

    cachedObject = new CachedObject(content, eTag);
    return content;
}
 

@Test
public void testInjectCredentialsAndInvokeV2Bytes() {

    final LoggerProxy loggerProxy = mock(LoggerProxy.class);
    final Credentials credentials = new Credentials("accessKeyId", "secretAccessKey", "sessionToken");
    final ResponseBytes<?> responseBytes = mock(ResponseBytes.class);

    final AmazonWebServicesClientProxy proxy = new AmazonWebServicesClientProxy(loggerProxy, credentials, () -> 1000L);

    final software.amazon.awssdk.services.s3.model.GetObjectRequest wrappedRequest = mock(
        software.amazon.awssdk.services.s3.model.GetObjectRequest.class);

    final software.amazon.awssdk.services.s3.model.GetObjectRequest.Builder builder = mock(
        software.amazon.awssdk.services.s3.model.GetObjectRequest.Builder.class);
    when(builder.overrideConfiguration(any(AwsRequestOverrideConfiguration.class))).thenReturn(builder);
    when(builder.build()).thenReturn(wrappedRequest);
    final software.amazon.awssdk.services.s3.model.GetObjectRequest request = mock(
        software.amazon.awssdk.services.s3.model.GetObjectRequest.class);
    when(request.toBuilder()).thenReturn(builder);

    final S3Client client = mock(S3Client.class);

    doReturn(responseBytes).when(client)
        .getObjectAsBytes(any(software.amazon.awssdk.services.s3.model.GetObjectRequest.class));

    final ResponseBytes<
        GetObjectResponse> result = proxy.injectCredentialsAndInvokeV2Bytes(request, client::getObjectAsBytes);

    // verify request is rebuilt for injection
    verify(request).toBuilder();

    // verify the wrapped request is sent over the initiate
    verify(client).getObjectAsBytes(wrappedRequest);

    // ensure the return type matches
    assertThat(result).isEqualTo(responseBytes);
}
 

@Override
public ResponseBytes<ResponseT> transform(ResponseT response, AbortableInputStream inputStream) throws Exception {

    wastingTimeInterruptibly();
    return delegate.transform(response, inputStream);
}
 

@Override
public CompletableFuture<ResponseBytes<StreamingOutputOperationResponse>> prepare() {
    cf = new CompletableFuture<>();
    return cf.thenApply(arr -> ResponseBytes.fromByteArray(response, arr));
}
 

@Test
public void syncStreaming_shouldContainResponseMetadata() {
    ResponseBytes<GetObjectResponse> responseBytes = s3.getObject(b -> b.key(KEY).bucket(BUCKET), ResponseTransformer.toBytes());
    GetObjectResponse response = responseBytes.response();
    verifyResponseMetadata(response);
}
 

private void verifyMultipartUploadResult(String key, List<String> contentsToUpload) throws Exception {
    ResponseBytes<GetObjectResponse> objectAsBytes = s3.getObject(b -> b.bucket(BUCKET).key(key),
                                                                  ResponseTransformer.toBytes());
    String appendedString = String.join("", contentsToUpload);
    assertThat(objectAsBytes.asUtf8String()).isEqualTo(appendedString);
}
 

@Override
public CompletableFuture<ResponseBytes<ResponseT>> prepare() {
    cf = new CompletableFuture<>();
    return cf.thenApply(arr -> ResponseBytes.fromByteArray(response, arr));
}
 

/**
 * Some operation with a streaming output<br/>
 * <p>
 * This is a convenience which creates an instance of the {@link StreamingOutputOperationRequest.Builder} avoiding
 * the need to create one manually via {@link StreamingOutputOperationRequest#builder()}
 * </p>
 *
 * @param streamingOutputOperationRequest
 *        A {@link Consumer} that will call methods on {@link StreamingOutputOperationRequest.Builder} to create a
 *        request.
 * @return A {@link ResponseBytes} that loads the data streamed from the service into memory and exposes it in
 *         convenient in-memory representations like a byte buffer or string. The unmarshalled response object can
 *         be obtained via {@link ResponseBytes#response()}. The service documentation for the response content is
 *         as follows 'This be a stream'.
 * @throws SdkException
 *         Base class for all exceptions that can be thrown by the SDK (both service and client). Can be used for
 *         catch all scenarios.
 * @throws SdkClientException
 *         If any client side error occurs such as an IO related failure, failure to get credentials, etc.
 * @throws JsonException
 *         Base class for all service exceptions. Unknown exceptions will be thrown as an instance of this type.
 * @sample JsonClient.StreamingOutputOperation
 * @see #getObject(streamingOutputOperation, ResponseTransformer)
 * @see <a href="http://docs.aws.amazon.com/goto/WebAPI/json-service-2010-05-08/StreamingOutputOperation"
 *      target="_top">AWS API Documentation</a>
 */
default ResponseBytes<StreamingOutputOperationResponse> streamingOutputOperationAsBytes(
        Consumer<StreamingOutputOperationRequest.Builder> streamingOutputOperationRequest) throws AwsServiceException,
                                                                                                  SdkClientException, JsonException {
    return streamingOutputOperationAsBytes(StreamingOutputOperationRequest.builder()
                                                                          .applyMutation(streamingOutputOperationRequest).build());
}