software.amazon.awssdk.utils.IoUtils Java Examples

@Test
public void sign_WithoutUsingSdkClient_ThroughExecutionAttributes() throws Exception {
    Aws4Signer signer = Aws4Signer.create();
    SdkHttpFullRequest httpFullRequest = generateBasicRequest();

    // sign the request
    SdkHttpFullRequest signedRequest = signer.sign(httpFullRequest, constructExecutionAttributes());

    SdkHttpClient httpClient = ApacheHttpClient.builder().build();

    HttpExecuteRequest request = HttpExecuteRequest.builder()
                                                   .request(signedRequest)
                                                   .contentStreamProvider(signedRequest.contentStreamProvider().orElse(null))
                                                   .build();

    HttpExecuteResponse response = httpClient.prepareRequest(request)
                                             .call();

    assertEquals("Non success http status code", 200, response.httpResponse().statusCode());

    String actualResult = IoUtils.toUtf8String(response.responseBody().get());
    assertEquals(getExpectedResult(), actualResult);
}
 

/**
 * Converts the specified AWS policy object to a JSON string, suitable for
 * passing to an AWS service.
 *
 * @param policy
 *            The AWS policy object to convert to a JSON string.
 *
 * @return The JSON string representation of the specified policy object.
 *
 * @throws IllegalArgumentException
 *             If the specified policy is null or invalid and cannot be
 *             serialized to a JSON string.
 */
public String writePolicyToString(Policy policy) {

    if (!isNotNull(policy)) {
        throw new IllegalArgumentException("Policy cannot be null");
    }

    try {
        return jsonStringOf(policy);
    } catch (Exception e) {
        String message = "Unable to serialize policy to JSON string: "
                         + e.getMessage();
        throw new IllegalArgumentException(message, e);
    } finally {
        IoUtils.closeQuietly(writer, log);
    }
}
 

@Test
public void getObjectShouldReturnStreamWithData() throws Exception {
    // given
    testee.putObject(PutObjectRequest.builder()
                    .bucket("someBucket")
                    .key("someKey")
                    .build(),
            RequestBody.fromString("testdata"));
    //when
    ResponseInputStream<GetObjectResponse> inputStream = testee.getObject(GetObjectRequest.builder()
            .bucket("someBucket")
            .key("someKey")
            .build());

    //then
    String data = IoUtils.toUtf8String(inputStream);
    assertThat(data, is("testdata"));
}
 

@Override
public synchronized void getBundleVersionContent(final BundleVersionCoordinate versionCoordinate, final OutputStream outputStream)
        throws BundlePersistenceException {
    final String key = getKey(versionCoordinate);
    LOGGER.debug("Retrieving bundle version from S3 bucket '{}' with key '{}'", new Object[]{s3BucketName, key});

    final GetObjectRequest request = GetObjectRequest.builder()
            .bucket(s3BucketName)
            .key(key)
            .build();

    try (final ResponseInputStream<GetObjectResponse> response = s3Client.getObject(request)) {
        IoUtils.copy(response, outputStream);
        LOGGER.debug("Successfully retrieved bundle version from S3 bucket '{}' with key '{}'", new Object[]{s3BucketName, key});
    } catch (Exception e) {
        throw new BundlePersistenceException("Error retrieving bundle version from S3 due to: " + e.getMessage(), e);
    }
}
 

/**
 * Execute the external process to retrieve credentials.
 */
private String executeCommand() throws IOException, InterruptedException {
    ProcessBuilder processBuilder = new ProcessBuilder(command);

    ByteArrayOutputStream commandOutput = new ByteArrayOutputStream();

    Process process = processBuilder.start();
    try {
        IoUtils.copy(process.getInputStream(), commandOutput, processOutputLimit);

        process.waitFor();

        if (process.exitValue() != 0) {
            throw new IllegalStateException("Command returned non-zero exit value: " + process.exitValue());
        }

        return new String(commandOutput.toByteArray(), StandardCharsets.UTF_8);
    } finally {
        process.destroy();
    }
}
 

@Test
public void remainingByteBufferConstructorOnlyRemainingBytesCopied() throws IOException {
    ByteBuffer bb = ByteBuffer.allocate(4);
    bb.put(new byte[]{1, 2, 3, 4});
    bb.flip();

    bb.get();
    bb.get();

    int originalRemaining = bb.remaining();

    RequestBody requestBody = RequestBody.fromRemainingByteBuffer(bb);

    assertThat(requestBody.contentLength()).isEqualTo(originalRemaining);

    byte[] requestBodyBytes = IoUtils.toByteArray(requestBody.contentStreamProvider().newStream());
    assertThat(ByteBuffer.wrap(requestBodyBytes)).isEqualTo(bb);
}
 

@Test
public void test_SignMethod_WithModeledParam_And_WithoutUsingSdkClient() throws Exception {
    Aws4Signer signer = Aws4Signer.create();
    SdkHttpFullRequest httpFullRequest = generateBasicRequest();

    // sign the request
    SdkHttpFullRequest signedRequest = signer.sign(httpFullRequest, constructSignerParams());

    SdkHttpClient httpClient = ApacheHttpClient.builder().build();

    HttpExecuteRequest request = HttpExecuteRequest.builder()
                                                   .request(signedRequest)
                                                   .contentStreamProvider(signedRequest.contentStreamProvider().orElse(null))
                                                   .build();
    HttpExecuteResponse response = httpClient.prepareRequest(request)
                                             .call();

    assertEquals("Non success http status code", 200, response.httpResponse().statusCode());

    String actualResult = IoUtils.toUtf8String(response.responseBody().get());
    assertEquals(getExpectedResult(), actualResult);
}
 

/**
 * Marshal the request and update the execution context with the result.
 */
private <T> void marshalRequestAndUpdateContext(ExecutionContext execCtx,
                                                Class<T> requestType,
                                                Function<T, SdkHttpFullRequest> requestMarshaller) {
    T sdkRequest = Validate.isInstanceOf(requestType, execCtx.interceptorContext().request(),
                                         "Interceptor generated unsupported type (%s) when %s was expected.",
                                         execCtx.interceptorContext().request().getClass(), requestType);

    SdkHttpFullRequest marshalledRequest = requestMarshaller.apply(sdkRequest);

    // TODO: The core SDK doesn't put the request body into the interceptor context. That should be fixed.
    Optional<RequestBody> requestBody = marshalledRequest.contentStreamProvider()
                                                         .map(ContentStreamProvider::newStream)
                                                         .map(is -> invokeSafely(() -> IoUtils.toByteArray(is)))
                                                         .map(RequestBody::fromBytes);

    execCtx.interceptorContext(execCtx.interceptorContext().copy(r -> r.httpRequest(marshalledRequest)
                                                                       .requestBody(requestBody.orElse(null))));
}
 

@Override
public Optional<InputStream> modifyHttpResponseContent(Context.ModifyHttpResponse context,
                                                       ExecutionAttributes executionAttributes) {
    if (INTERCEPTOR_CONTEXT_PREDICATE.test(context)) {

        String policy = context.responseBody()
                               .map(r -> invokeSafely(() -> IoUtils.toUtf8String(r)))
                               .orElse(null);

        if (policy != null) {
            String xml = XML_ENVELOPE_PREFIX + policy + XML_ENVELOPE_SUFFIX;
            return Optional.of(AbortableInputStream.create(new StringInputStream(xml)));
        }
    }

    return context.responseBody();
}
 

@Test
public void test_SignMethod_WithModeledParam_And_WithoutUsingSdkClient() throws Exception {
    AwsS3V4Signer signer = AwsS3V4Signer.create();
    SdkHttpFullRequest httpFullRequest = generateBasicGetRequest();

    // sign the request
    SdkHttpFullRequest signedRequest = signer.sign(httpFullRequest, constructSignerParams());

    SdkHttpClient httpClient = ApacheHttpClient.builder().build();

    HttpExecuteResponse response = httpClient.prepareRequest(HttpExecuteRequest.builder().request(signedRequest).build())
                                             .call();

    assertEquals("Non success http status code", 200, response.httpResponse().statusCode());

    String actualResult = IoUtils.toUtf8String(response.responseBody().get());
    assertEquals(CONTENT, actualResult);
}
 

@Override
public Optional<RequestBody> modifyHttpContent(Context.ModifyHttpRequest context,
                                               ExecutionAttributes executionAttributes) {

    if (!BLACKLIST_METHODS.contains(context.request().getClass()) && context.requestBody().isPresent()
        && !context.httpRequest().firstMatchingHeader(CONTENT_MD5).isPresent()) {

        try {
            ByteArrayOutputStream baos = new ByteArrayOutputStream();
            IoUtils.copy(context.requestBody().get().contentStreamProvider().newStream(), baos);
            executionAttributes.putAttribute(CONTENT_MD5_ATTRIBUTE, Md5Utils.md5AsBase64(baos.toByteArray()));
            return context.requestBody();
        } catch (IOException e) {
            throw new UncheckedIOException(e);
        }
    }

    return context.requestBody();
}
 

@Test
public void test_SignMethod_WithExecutionAttributes_And_WithoutUsingSdkClient() throws Exception {
    AwsS3V4Signer signer = AwsS3V4Signer.create();
    SdkHttpFullRequest httpFullRequest = generateBasicGetRequest();

    // sign the request
    SdkHttpFullRequest signedRequest = signer.sign(httpFullRequest, constructExecutionAttributes());

    SdkHttpClient httpClient = ApacheHttpClient.builder().build();

    HttpExecuteResponse response = httpClient.prepareRequest(HttpExecuteRequest.builder().request(signedRequest).build())
                                             .call();

    assertEquals("Non success http status code", 200, response.httpResponse().statusCode());

    String actualResult = IoUtils.toUtf8String(response.responseBody().get());
    assertEquals(CONTENT, actualResult);
}
 

@Test
public void browserIncompatiblePresignedUrlWorksWithAdditionalHeaders() throws IOException {
    PresignedGetObjectRequest presigned =
        presigner.presignGetObject(r -> r.signatureDuration(Duration.ofMinutes(5))
                                         .getObjectRequest(gor -> gor.bucket(testBucket)
                                                                     .key(testGetObjectKey)
                                                                     .requestPayer(RequestPayer.REQUESTER)));

    assertThat(presigned.isBrowserExecutable()).isFalse();

    HttpURLConnection connection = (HttpURLConnection) presigned.url().openConnection();

    presigned.httpRequest().headers().forEach((header, values) -> {
        values.forEach(value -> {
            connection.addRequestProperty(header, value);
        });
    });

    try (InputStream content = connection.getInputStream()) {
        assertThat(IoUtils.toUtf8String(content)).isEqualTo(testObjectContent);
    }
}
 

@Override
public HttpExecuteResponse call() throws IOException {
    connection.connect();

    request.contentStreamProvider().ifPresent(provider ->
            invokeSafely(() -> IoUtils.copy(provider.newStream(), connection.getOutputStream())));

    int responseCode = connection.getResponseCode();
    boolean isErrorResponse = HttpStatusFamily.of(responseCode).isOneOf(CLIENT_ERROR, SERVER_ERROR);
    InputStream content = !isErrorResponse ? connection.getInputStream() : connection.getErrorStream();
    AbortableInputStream responseBody = content != null ?
                                        AbortableInputStream.create(content) : null;

    return HttpExecuteResponse.builder()
                              .response(SdkHttpResponse.builder()
                                                   .statusCode(responseCode)
                                                   .statusText(connection.getResponseMessage())
                                                   // TODO: Don't ignore abort?
                                                   .headers(extractHeaders(connection))
                                                   .build())
                              .responseBody(responseBody)
                              .build();
}
 

private void validateResponse(HttpExecuteResponse response, int returnCode, SdkHttpMethod method) throws IOException {
    RequestMethod requestMethod = RequestMethod.fromString(method.name());

    RequestPatternBuilder patternBuilder = RequestPatternBuilder.newRequestPattern(requestMethod, urlMatching("/"))
                                                                       .withHeader("Host", containing("localhost"))
                                                                       .withHeader("User-Agent", equalTo("hello-world!"));

    if (method == SdkHttpMethod.HEAD) {
        patternBuilder.withRequestBody(equalTo(""));
    } else {
        patternBuilder.withRequestBody(equalTo("Body"));
    }

    mockServer.verify(1, patternBuilder);

    if (method == SdkHttpMethod.HEAD) {
        assertThat(response.responseBody()).isEmpty();
    } else {
        assertThat(IoUtils.toUtf8String(response.responseBody().orElse(null))).isEqualTo("hello");
    }

    assertThat(response.httpResponse().firstMatchingHeader("Some-Header")).contains("With Value");
    assertThat(response.httpResponse().statusCode()).isEqualTo(returnCode);
    mockServer.resetMappings();
}
 

/**
 * Reads to the end of the inputStream returning a byte array of the contents
 *
 * @param inputStream
 *            InputStream to drain
 * @return Remaining data in stream as a byte array
 */
public static byte[] drainInputStream(InputStream inputStream) {
    ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();

    try {
        byte[] buffer = new byte[1024];
        long bytesRead = 0;
        while ((bytesRead = inputStream.read(buffer)) > -1) {
            byteArrayOutputStream.write(buffer, 0, (int) bytesRead);
        }
        return byteArrayOutputStream.toByteArray();
    } catch (IOException e) {
        throw new RuntimeException(e);
    } finally {
        IoUtils.closeQuietly(byteArrayOutputStream, null);
    }
}
 

@Test
public void getObjectShouldReturnStreamWithData() throws Exception {
    // given
    testee.putObject(PutObjectRequest.builder()
                    .bucket("someBucket")
                    .key("someKey")
                    .build(),
            RequestBody.fromString("testdata"));
    //when
    ResponseInputStream<GetObjectResponse> inputStream = testee.getObject(GetObjectRequest.builder()
            .bucket("someBucket")
            .key("someKey")
            .build());

    //then
    String data = IoUtils.toUtf8String(inputStream);
    assertThat(data, is("testdata"));
}
 

@BeforeClass
public static void setup() throws IOException {
    try (InputStream successInputStream = HttpCredentialsProviderTest.class.getResourceAsStream
        ("/resources/wiremock/successResponse.json");
         InputStream responseWithInvalidBodyInputStream = HttpCredentialsProviderTest.class.getResourceAsStream
             ("/resources/wiremock/successResponseWithInvalidBody.json")) {
        successResponse = IoUtils.toUtf8String(successInputStream);
        successResponseWithInvalidBody = IoUtils.toUtf8String(responseWithInvalidBodyInputStream);
    }
}
 

@Test
public void postWithContentIsUnaltered() throws Exception {
    byte[] contentBytes = "hello".getBytes(StandardCharsets.UTF_8);
    ContentStreamProvider contentProvider = () -> new ByteArrayInputStream(contentBytes);

    SdkHttpFullRequest request = requestBuilder.contentStreamProvider(contentProvider).build();

    SdkHttpFullRequest output = (SdkHttpFullRequest) interceptor.modifyHttpRequest(
            new HttpRequestOnlyContext(request, null), executionAttributes);

    assertThat(output.rawQueryParameters()).hasSize(1);
    assertThat(output.headers()).hasSize(0);
    assertThat(IoUtils.toByteArray(output.contentStreamProvider().get().newStream())).isEqualTo(contentBytes);
}
 

@Override
public String parseErrorCode(SdkHttpFullResponse response, JsonContent jsonContents) {
    IonReader reader = ionSystem.newReader(jsonContents.getRawContent());
    try {
        IonType type = reader.next();
        if (type != IonType.STRUCT) {
            throw SdkClientException.builder()
                                    .message(String.format("Can only get error codes from structs (saw %s), request id %s",
                                             type, getRequestId(response)))
                                    .build();
        }

        boolean errorCodeSeen = false;
        String errorCode = null;
        String[] annotations = reader.getTypeAnnotations();
        for (String annotation : annotations) {
            if (annotation.startsWith(TYPE_PREFIX)) {
                if (errorCodeSeen) {
                    throw SdkClientException.builder()
                                            .message(String.format("Multiple error code annotations found for request id %s",
                                                               getRequestId(response)))
                                            .build();
                } else {
                    errorCodeSeen = true;
                    errorCode = annotation.substring(TYPE_PREFIX.length());
                }
            }
        }

        return errorCode;
    } finally {
        IoUtils.closeQuietly(reader, log);
    }
}
 

private Mimetype() {
    Optional.ofNullable(CLASS_LOADER).map(loader -> loader.getResourceAsStream(MIME_TYPE_PATH)).ifPresent(
        stream -> {
            try {
                loadAndReplaceMimetypes(stream);
            } catch (IOException e) {
                LOG.debug("Failed to load mime types from file in the classpath: mime.types", e);
            } finally {
                IoUtils.closeQuietly(stream, null);
            }
        }
    );
}
 

@Override
public ProfileFile build() {
    InputStream stream = content != null ? content :
                         FunctionalUtils.invokeSafely(() -> Files.newInputStream(contentLocation));

    Validate.paramNotNull(type, "type");
    Validate.paramNotNull(stream, "content");

    try {
        return new ProfileFile(ProfileFileReader.parseFile(stream, type));
    } finally {
        IoUtils.closeQuietly(stream, null);
    }
}
 

/**
 * Creates a response transformer that loads all response content into memory, exposed as {@link ResponseBytes}. This allows
 * for conversion into a {@link String}, {@link ByteBuffer}, etc.
 *
 * @param <ResponseT> Type of unmarshalled response POJO.
 * @return The streaming response transformer that can be used on the client streaming method.
 */
static <ResponseT> ResponseTransformer<ResponseT, ResponseBytes<ResponseT>> toBytes() {
    return (response, inputStream) -> {
        try {
            InterruptMonitor.checkInterrupted();
            return ResponseBytes.fromByteArray(response, IoUtils.toByteArray(inputStream));
        } catch (IOException e) {
            throw RetryableException.builder().message("Failed to read response.").cause(e).build();
        }
    };
}
 

@Override
public void release() {
    // Don't call IOUtils.release(in, null) or else could lead to infinite loop
    IoUtils.closeQuietly(this, null);
    if (in instanceof Releasable) {
        // This allows any underlying stream that has the close operation
        // disabled to be truly released
        Releasable r = (Releasable) in;
        r.release();
    }
}
 

/**
 * Close the input stream if required.
 */
private void closeInputStreamIfNeeded(SdkHttpFullResponse httpResponse,
                                      boolean didRequestFail) {
    // Always close on failed requests. Close on successful requests unless it needs connection left open
    if (didRequestFail || !successResponseHandler.needsConnectionLeftOpen()) {
        Optional.ofNullable(httpResponse)
                .flatMap(SdkHttpFullResponse::content) // If no content, no need to close
                .ifPresent(s -> IoUtils.closeQuietly(s, log));
    }
}
 

/**
 * Static factory method to create a JsonContent object from the contents of the HttpResponse
 * provided
 */
public static JsonContent createJsonContent(SdkHttpFullResponse httpResponse,
                                            JsonFactory jsonFactory) {

    byte[] rawJsonContent = httpResponse.content().map(c -> {
        try {
            return IoUtils.toByteArray(c);
        } catch (IOException e) {
            LOG.debug("Unable to read HTTP response content", e);
        }
        return null;
    }).orElse(null);
    return new JsonContent(rawJsonContent, jsonFactory);
}
 

/**
 * @see HttpResponseHandler#handle(SdkHttpFullResponse, ExecutionAttributes)
 */
public T handle(SdkHttpFullResponse response, ExecutionAttributes executionAttributes) throws Exception {
    SdkStandardLogger.REQUEST_LOGGER.trace(() -> "Parsing service response JSON.");

    SdkStandardLogger.REQUEST_ID_LOGGER.debug(() -> X_AMZN_REQUEST_ID_HEADER + " : " +
                                                    response.firstMatchingHeader(X_AMZN_REQUEST_ID_HEADER)
                                                            .orElse("not available"));

    SdkStandardLogger.REQUEST_ID_LOGGER.debug(() -> X_AMZ_ID_2_HEADER + " : " +
                                                    response.firstMatchingHeader(X_AMZ_ID_2_HEADER)
                                                            .orElse("not available"));

    try {
        T result = unmarshaller.unmarshall(pojoSupplier.apply(response), response);

        // Make sure we read all the data to get an accurate CRC32 calculation.
        // See https://github.com/aws/aws-sdk-java/issues/1018
        if (shouldParsePayloadAsJson() && response.content().isPresent()) {
            IoUtils.drainInputStream(response.content().get());
        }

        SdkStandardLogger.REQUEST_LOGGER.trace(() -> "Done parsing service response.");
        return result;
    } finally {
        if (!needsConnectionLeftOpen) {
            response.content().ifPresent(i -> FunctionalUtils.invokeSafely(i::close));
        }
    }
}
 

/**
 * Close the input stream if required.
 */
private void closeInputStreamIfNeeded(SdkHttpFullResponse httpResponse,
                                      boolean didRequestFail) {
    // Always close on failed requests. Close on successful requests unless it needs connection left open
    if (didRequestFail || !needsConnectionLeftOpen) {
        Optional.ofNullable(httpResponse)
                .flatMap(SdkHttpFullResponse::content) // If no content, no need to close
                .ifPresent(s -> IoUtils.closeQuietly(s, log));
    }
}
 

private static String getExpectedClass(ClassSpec spec, String testFile) {
    try {
        InputStream resource = spec.getClass().getResourceAsStream(testFile);
        Validate.notNull(resource, "Failed to load test file: " + testFile);
        return processor.apply(IoUtils.toUtf8String(resource));
    } catch (IOException e) {
        throw new RuntimeException(e);
    }
}
 

private String loadDefaultFileHeader() throws IOException {
    try (InputStream inputStream = getClass()
        .getResourceAsStream("/software/amazon/awssdk/codegen/lite/DefaultFileHeader.txt")) {
        return IoUtils.toUtf8String(inputStream)
                      .replaceFirst("%COPYRIGHT_DATE_RANGE%", getCopyrightDateRange());
    }
}