software.amazon.awssdk.core.ResponseInputStream Java Examples

@Test
public void getObjectShouldReturnStreamWithData() throws Exception {
    // given
    testee.putObject(PutObjectRequest.builder()
                    .bucket("someBucket")
                    .key("someKey")
                    .build(),
            RequestBody.fromString("testdata"));
    //when
    ResponseInputStream<GetObjectResponse> inputStream = testee.getObject(GetObjectRequest.builder()
            .bucket("someBucket")
            .key("someKey")
            .build());

    //then
    String data = IoUtils.toUtf8String(inputStream);
    assertThat(data, is("testdata"));
}
 

@Override
public byte[] serialize(String topic, ResponseInputStream data) {
    ByteArrayOutputStream buffer = new ByteArrayOutputStream();

    int nRead;
    byte[] byteArray = new byte[16384];

    try {
        while ((nRead = data.read(byteArray, 0, byteArray.length)) != -1) {
            buffer.write(byteArray, 0, nRead);
        }
    } catch (IOException e) {
        LOG.warn("I/O error while serializing data from or to topic {}: {} | {}", topic, e.getMessage(), e);
    }

    return buffer.toByteArray();
}
 

public FeatureState retrieveFeatureStateFromS3(final Feature feature) {
    final GetObjectRequest getRequest = GetObjectRequest.builder()
            .bucket(togglzProperties.getS3().getBucketName())
            .key(keyForFeature(feature))
            .build();
    try (final ResponseInputStream<GetObjectResponse> responseStream = s3Client.getObject(getRequest)) {
        if (responseStream != null) {
            final FeatureStateStorageWrapper wrapper = objectMapper.reader()
                    .forType(FeatureStateStorageWrapper.class)
                    .readValue(responseStream);
            return FeatureStateStorageWrapper.featureStateForWrapper(feature, wrapper);
        }
    } catch (final S3Exception ae) {
        if (ERR_NO_SUCH_KEY.equals(ae.awsErrorDetails().errorCode()) ||  ae.awsErrorDetails().sdkHttpResponse().statusCode() == 404) {
            return null;
        }
        throw ae;
    } catch (final IOException e) {
        throw new RuntimeException("Failed to get the feature state", e);
    }
    return null;
}
 

public <RequestT extends AwsRequest, ResultT extends AwsResponse>
    ResponseInputStream<ResultT>
    injectCredentialsAndInvokeV2InputStream(final RequestT request,
                                            final Function<RequestT, ResponseInputStream<ResultT>> requestFunction) {

    AwsRequestOverrideConfiguration overrideConfiguration = AwsRequestOverrideConfiguration.builder()
        .credentialsProvider(v2CredentialsProvider).build();

    @SuppressWarnings("unchecked")
    RequestT wrappedRequest = (RequestT) request.toBuilder().overrideConfiguration(overrideConfiguration).build();

    try {
        ResponseInputStream<ResultT> response = requestFunction.apply(wrappedRequest);
        logRequestMetadataV2(request, response.response());
        return response;
    } catch (final Throwable e) {
        loggerProxy.log(String.format("Failed to execute remote function: {%s}", e.getMessage()));
        throw e;
    }
}
 

@Test
public void getObjectShouldReturnStreamWithData() throws Exception {
    // given
    testee.putObject(PutObjectRequest.builder()
                    .bucket("someBucket")
                    .key("someKey")
                    .build(),
            RequestBody.fromString("testdata"));
    //when
    ResponseInputStream<GetObjectResponse> inputStream = testee.getObject(GetObjectRequest.builder()
            .bucket("someBucket")
            .key("someKey")
            .build());

    //then
    String data = IoUtils.toUtf8String(inputStream);
    assertThat(data, is("testdata"));
}
 

@Override
public synchronized void getBundleVersionContent(final BundleVersionCoordinate versionCoordinate, final OutputStream outputStream)
        throws BundlePersistenceException {
    final String key = getKey(versionCoordinate);
    LOGGER.debug("Retrieving bundle version from S3 bucket '{}' with key '{}'", new Object[]{s3BucketName, key});

    final GetObjectRequest request = GetObjectRequest.builder()
            .bucket(s3BucketName)
            .key(key)
            .build();

    try (final ResponseInputStream<GetObjectResponse> response = s3Client.getObject(request)) {
        IoUtils.copy(response, outputStream);
        LOGGER.debug("Successfully retrieved bundle version from S3 bucket '{}' with key '{}'", new Object[]{s3BucketName, key});
    } catch (Exception e) {
        throw new BundlePersistenceException("Error retrieving bundle version from S3 due to: " + e.getMessage(), e);
    }
}
 

/**
 * @return Simple method for streaming output operations to get content as an input stream.
 */
private MethodSpec inputStreamSimpleMethod(OperationModel opModel, TypeName responseType, ClassName requestType) {
    TypeName returnType = ParameterizedTypeName.get(ClassName.get(ResponseInputStream.class), responseType);
    return MethodSpec.methodBuilder(opModel.getMethodName())
                     .returns(returnType)
                     .addModifiers(Modifier.PUBLIC, Modifier.DEFAULT)
                     .addParameter(requestType, opModel.getInput().getVariableName())
                     .addJavadoc(opModel.getDocs(model, ClientType.SYNC, SimpleMethodOverload.INPUT_STREAM))
                     .addExceptions(getExceptionClasses(model, opModel))
                     .addStatement("return $L($L, $T.toInputStream())", opModel.getMethodName(),
                                   opModel.getInput().getVariableName(),
                                   ClassName.get(ResponseTransformer.class))
                     .build();
}
 

@SuppressWarnings("unchecked")
@Override
public ResponseInputStream<GetObjectResponse> getObject(final GetObjectRequest getObjectRequest) throws S3Exception {
    final Map<String, BucketItem> bucketItemMap = bucketsWithContents.get(getObjectRequest.bucket());
    final BucketItem bucketItem = bucketItemMap.get(getObjectRequest.key());
    if (bucketItem != null) {
        try {
            return new ResponseInputStream<>(GetObjectResponse.builder().build(), toAbortableInputStream(bucketItem));
        } catch (NoSuchMethodException | IllegalAccessException | InvocationTargetException | InstantiationException e) {
            throw SdkClientException.create("", e);
        }
    } else {
        throw NoSuchKeyException.builder().message("NoSuchKey").awsErrorDetails(AwsErrorDetails.builder().errorCode("NoSuchKey").build()).build();
    }
}
 

public static InputStream synthesize(PollyClient polly, String text, Voice voice, OutputFormat format) throws IOException {

        SynthesizeSpeechRequest synthReq = SynthesizeSpeechRequest.builder()
                .text(text)
                .voiceId(voice.id())
                .outputFormat(format)
                .build();

        ResponseInputStream<SynthesizeSpeechResponse> synthRes = polly.synthesizeSpeech(synthReq);
        return synthRes;
    }
 

@SuppressWarnings("unchecked")
@Override
public ResponseInputStream<GetObjectResponse> getObject(final GetObjectRequest getObjectRequest) throws S3Exception {
    final Map<String, BucketItem> bucketItemMap = bucketsWithContents.get(getObjectRequest.bucket());
    final BucketItem bucketItem = bucketItemMap.get(getObjectRequest.key());
    try {
        return new ResponseInputStream<>(GetObjectResponse.builder().build(), toAbortableInputStream(bucketItem));
    } catch (NoSuchMethodException | IllegalAccessException | InvocationTargetException | InstantiationException e) {
        throw SdkClientException.create("", e);
    }

}
 

@SuppressWarnings("unchecked")
private LinkedHashMap<String, JSONArray> fetchAndParseSnapshotFileFromS3(String snapshotFileName) {
    GetObjectRequest request = GetObjectRequest.builder().bucket(INTEGRATION_TEST_BUCKET).key(snapshotFileName).build();

    ResponseInputStream<GetObjectResponse> responseInputStream = s3Client.getObject(request);
    try (
            BufferedInputStream bufferedInputStream = new BufferedInputStream(responseInputStream);
            ZipInputStream zipInputStream = new ZipInputStream(bufferedInputStream)
    ) {
        zipInputStream.getNextEntry();
        return (LinkedHashMap<String, JSONArray>) Configuration.defaultConfiguration().jsonProvider().parse(zipInputStream, Charsets.UTF_8.name());
    } catch (IOException e) {
        throw new RuntimeException(e);
    }
}
 

@SuppressWarnings("unchecked")
private LinkedHashMap<String, JSONArray> fetchAndParseSnapshotFileFromS3(String snapshotFileName) {
    GetObjectRequest request = GetObjectRequest.builder().bucket(INTEGRATION_TEST_BUCKET).key(snapshotFileName).build();

    ResponseInputStream<GetObjectResponse> responseInputStream = s3Client.getObject(request);
    try (
            BufferedInputStream bufferedInputStream = new BufferedInputStream(responseInputStream);
            ZipInputStream zipInputStream = new ZipInputStream(bufferedInputStream)
    ) {
        zipInputStream.getNextEntry();
        return (LinkedHashMap<String, JSONArray>) Configuration.defaultConfiguration().jsonProvider().parse(zipInputStream, Charsets.UTF_8.name());
    } catch (IOException e) {
        throw new RuntimeException(e);
    }
}
 

@Test
public void streamingOut_toInputStream_closeResponseStreamShouldCloseUnderlyingStream() throws IOException {
    ClosableStringInputStream inputStream = new ClosableStringInputStream("{}");
    mockHttpClient.stubNextResponse(mockResponse(inputStream, 200));

    ResponseInputStream<StreamingOutputOperationResponse> responseInputStream =
        client.streamingOutputOperation(b -> b.build(), ResponseTransformer.toInputStream());

    assertThat(inputStream.isClosed()).isFalse();
    responseInputStream.close();
    assertThat(inputStream.isClosed()).isTrue();
}
 

@Test
public void toInputStream_loadFromProperties() throws IOException {
    s3.putObject(b -> b.bucket(BUCKET).key(PROPERTY_KEY), RequestBody.fromString("test: test"));
    try (ResponseInputStream<GetObjectResponse> object = s3.getObject(b -> b.bucket(BUCKET).key(PROPERTY_KEY),
                                                                      ResponseTransformer.toInputStream())) {
        Properties properties = new Properties();
        properties.load(object);
        assertThat(properties.getProperty("test")).isEqualTo("test");
    }
}
 

@Test
public void testInjectCredentialsAndInvokeV2InputStream() {

    final LoggerProxy loggerProxy = mock(LoggerProxy.class);
    final Credentials credentials = new Credentials("accessKeyId", "secretAccessKey", "sessionToken");
    final ResponseInputStream<?> responseInputStream = mock(ResponseInputStream.class);

    final AmazonWebServicesClientProxy proxy = new AmazonWebServicesClientProxy(loggerProxy, credentials, () -> 1000L);

    final software.amazon.awssdk.services.s3.model.GetObjectRequest wrappedRequest = mock(
        software.amazon.awssdk.services.s3.model.GetObjectRequest.class);

    final software.amazon.awssdk.services.s3.model.GetObjectRequest.Builder builder = mock(
        software.amazon.awssdk.services.s3.model.GetObjectRequest.Builder.class);
    when(builder.overrideConfiguration(any(AwsRequestOverrideConfiguration.class))).thenReturn(builder);
    when(builder.build()).thenReturn(wrappedRequest);
    final software.amazon.awssdk.services.s3.model.GetObjectRequest request = mock(
        software.amazon.awssdk.services.s3.model.GetObjectRequest.class);
    when(request.toBuilder()).thenReturn(builder);

    final S3Client client = mock(S3Client.class);

    doReturn(responseInputStream).when(client)
        .getObject(any(software.amazon.awssdk.services.s3.model.GetObjectRequest.class));

    final ResponseInputStream<
        GetObjectResponse> result = proxy.injectCredentialsAndInvokeV2InputStream(request, client::getObject);

    // verify request is rebuilt for injection
    verify(request).toBuilder();

    // verify the wrapped request is sent over the initiate
    verify(client).getObject(wrappedRequest);

    // ensure the return type matches
    assertThat(result).isEqualTo(responseInputStream);
}
 

/** {@inheritDoc} */
@Override
protected void download(Path tmp, URI baseUri, Artifact.Item item, Progress progress)
        throws IOException {
    String key = item.getUri();
    logger.debug("Downloading artifact from: s3://{}/{} ...", bucket, key);
    GetObjectRequest req = GetObjectRequest.builder().bucket(bucket).key(key).build();
    try (ResponseInputStream<GetObjectResponse> is = client.getObject(req)) {
        save(is, tmp, baseUri, item, progress);
    }
}
 

@Override
public R apply(R record) {
    byte[] v = serializer.serialize(record.topic(), (ResponseInputStream)record.value());
    String finalValue = new String(v);
    return record.newRecord(record.topic(), record.kafkaPartition(), null, record.key(), Schema.STRING_SCHEMA, finalValue, record.timestamp());
}
 

@Override
public ResponseInputStream<ResponseT> transform(ResponseT response, AbortableInputStream inputStream) throws Exception {
    wastingTimeInterruptibly();
    return delegate.transform(response, inputStream);
}
 

@Test
public void contentRangeIsReturnedForRangeRequests() {
    ResponseInputStream<GetObjectResponse> stream = s3.getObject(getObjectRequest.copy(r -> r.range("bytes=0-1")));
    stream.abort();
    assertThat(stream.response().contentRange()).isEqualTo("bytes 0-1/10000");
}
 

@Test
public void toInputStream() throws Exception {
    try (ResponseInputStream<GetObjectResponse> content = s3.getObject(getObjectRequest)) {
    }
}
 

private void verifyResponse(String key) {
    ResponseInputStream<GetObjectResponse> responseInputStream = s3.getObject(b -> b.bucket(BUCKET).key(key));
    IoUtils.drainInputStream(responseInputStream);
}
 

@Override
public byte[] fromConnectData(String topic, Schema schema, Object value) {
    return serializer.serialize(topic, (ResponseInputStream)value);
}
 

/**
 * Some operation with a streaming output<br/>
 * <p>
 * This is a convenience which creates an instance of the {@link StreamingOutputOperationRequest.Builder} avoiding
 * the need to create one manually via {@link StreamingOutputOperationRequest#builder()}
 * </p>
 *
 * @param streamingOutputOperationRequest
 *        A {@link Consumer} that will call methods on {@link StreamingOutputOperationRequest.Builder} to create a
 *        request.
 * @return A {@link ResponseInputStream} containing data streamed from service. Note that this is an unmanaged
 *         reference to the underlying HTTP connection so great care must be taken to ensure all data if fully read
 *         from the input stream and that it is properly closed. Failure to do so may result in sub-optimal behavior
 *         and exhausting connections in the connection pool. The unmarshalled response object can be obtained via
 *         {@link ResponseInputStream#response()}. The service documentation for the response content is as follows
 *         'This be a stream'.
 * @throws SdkException
 *         Base class for all exceptions that can be thrown by the SDK (both service and client). Can be used for
 *         catch all scenarios.
 * @throws SdkClientException
 *         If any client side error occurs such as an IO related failure, failure to get credentials, etc.
 * @throws JsonException
 *         Base class for all service exceptions. Unknown exceptions will be thrown as an instance of this type.
 * @sample JsonClient.StreamingOutputOperation
 * @see #getObject(streamingOutputOperation, ResponseTransformer)
 * @see <a href="http://docs.aws.amazon.com/goto/WebAPI/json-service-2010-05-08/StreamingOutputOperation"
 *      target="_top">AWS API Documentation</a>
 */
default ResponseInputStream<StreamingOutputOperationResponse> streamingOutputOperation(
        Consumer<StreamingOutputOperationRequest.Builder> streamingOutputOperationRequest) throws AwsServiceException,
                                                                                                  SdkClientException, JsonException {
    return streamingOutputOperation(StreamingOutputOperationRequest.builder().applyMutation(streamingOutputOperationRequest)
                                                                   .build());
}
 

/**
 * This is a synchronous version of making API calls which implement
 * ResponseInputStream in the SDKv2
 *
 * @param request, the AWS service request that we need to make
 * @param requestFunction, this is a Lambda closure that provide the actual API
 *            that needs to be invoked.
 * @param <RequestT> the request type
 * @param <ResponseT> the response from the request
 * @return the response if successful. Else it will propagate all
 *         {@link software.amazon.awssdk.awscore.exception.AwsServiceException}
 *         that is thrown or
 *         {@link software.amazon.awssdk.core.exception.SdkClientException} if
 *         there is client side problem
 */
default <RequestT extends AwsRequest, ResponseT extends AwsResponse>
    ResponseInputStream<ResponseT>
    injectCredentialsAndInvokeV2InputStream(RequestT request,
                                            Function<RequestT, ResponseInputStream<ResponseT>> requestFunction) {
    throw new UnsupportedOperationException();
}
 

/**
 * Some operation with a streaming output
 *
 * @param streamingOutputOperationRequest
 * @return A {@link ResponseInputStream} containing data streamed from service. Note that this is an unmanaged
 *         reference to the underlying HTTP connection so great care must be taken to ensure all data if fully read
 *         from the input stream and that it is properly closed. Failure to do so may result in sub-optimal behavior
 *         and exhausting connections in the connection pool. The unmarshalled response object can be obtained via
 *         {@link ResponseInputStream#response()}. The service documentation for the response content is as follows
 *         'This be a stream'.
 * @throws SdkException
 *         Base class for all exceptions that can be thrown by the SDK (both service and client). Can be used for
 *         catch all scenarios.
 * @throws SdkClientException
 *         If any client side error occurs such as an IO related failure, failure to get credentials, etc.
 * @throws JsonException
 *         Base class for all service exceptions. Unknown exceptions will be thrown as an instance of this type.
 * @sample JsonClient.StreamingOutputOperation
 * @see #getObject(streamingOutputOperation, ResponseTransformer)
 * @see <a href="http://docs.aws.amazon.com/goto/WebAPI/json-service-2010-05-08/StreamingOutputOperation"
 *      target="_top">AWS API Documentation</a>
 */
default ResponseInputStream<StreamingOutputOperationResponse> streamingOutputOperation(
        StreamingOutputOperationRequest streamingOutputOperationRequest) throws AwsServiceException, SdkClientException,
                                                                                JsonException {
    return streamingOutputOperation(streamingOutputOperationRequest, ResponseTransformer.toInputStream());
}
 

/**
 * Creates a response transformer that returns an unmanaged input stream with the response content. This input stream must
 * be explicitly closed to release the connection. The unmarshalled response object can be obtained via the {@link
 * ResponseInputStream#response} method.
 * <p>
 * Note that the returned stream is not subject to the retry policy or timeout settings (except for socket timeout)
 * of the client. No retries will be performed in the event of a socket read failure or connection reset.
 *
 * @param <ResponseT> Type of unmarshalled response POJO.
 * @return ResponseTransformer instance.
 */
static <ResponseT> ResponseTransformer<ResponseT, ResponseInputStream<ResponseT>> toInputStream() {
    return unmanaged(ResponseInputStream::new);
}