org.apache.http.protocol.HttpContext Java Examples

@Override
public HttpUriRequest getRedirect(HttpRequest request, HttpResponse response, HttpContext context) throws ProtocolException {
    URI uri = getLocationURI(request, response, context);
    String method = request.getRequestLine().getMethod();
    if ("post".equalsIgnoreCase(method)) {
        try {
            HttpRequestWrapper httpRequestWrapper = (HttpRequestWrapper) request;
            httpRequestWrapper.setURI(uri);
            httpRequestWrapper.removeHeaders("Content-Length");
            return httpRequestWrapper;
        } catch (Exception e) {

            log.error("强转为HttpRequestWrapper出错");
        }
        return new HttpPost(uri);
    } else {
        return new HttpGet(uri);
    }
}
 

@Test(expected = AbortedException.class)
public void
clientExecutionTimeoutEnabled_aborted_exception_occurs_timeout_not_expired()
        throws Exception {
    ClientConfiguration config = new ClientConfiguration()
            .withClientExecutionTimeout(CLIENT_EXECUTION_TIMEOUT)
            .withMaxErrorRetry(0);
    ConnectionManagerAwareHttpClient rawHttpClient =
            createRawHttpClientSpy(config);

    doThrow(new AbortedException()).when(rawHttpClient).execute(any
            (HttpRequestBase.class), any(HttpContext.class));

    httpClient = new AmazonHttpClient(config, rawHttpClient, null);

    execute(httpClient, createMockGetRequest());
}
 

/**
 * UnixHttpClient can execute the HttpRequest with the given host
 * and context.
 * @throws IOException If something goes wrong.
 */
@Test
public void executesRequestWithHostAndContext() throws IOException {
    final HttpHost host = new HttpHost("127.0.0.1");
    final HttpRequest req = Mockito.mock(HttpRequest.class);
    final HttpContext context = Mockito.mock(HttpContext.class);
    final HttpResponse resp = Mockito.mock(HttpResponse.class);
    final HttpClient decorated = Mockito.mock(HttpClient.class);
    Mockito.when(
            decorated.execute(host, req, context)
    ).thenReturn(resp);

    final HttpClient unix = new UnixHttpClient(() -> decorated);
    MatcherAssert.assertThat(
        unix.execute(host, req, context), Matchers.is(resp)
    );
    Mockito.verify(
        decorated, Mockito.times(1)
    ).execute(host, req, context);
}
 

@Override
public void process(HttpRequest request, HttpContext context) throws HttpException, IOException {
    AuthState authState = (AuthState) context.getAttribute(ClientContext.TARGET_AUTH_STATE);

    if (authState.getAuthScheme() == null) {
        AuthScheme authScheme = (AuthScheme) context.getAttribute("preemptive-auth");
        CredentialsProvider credsProvider = (CredentialsProvider) context
                .getAttribute(ClientContext.CREDS_PROVIDER);
        HttpHost targetHost = (HttpHost) context.getAttribute(ExecutionContext.HTTP_TARGET_HOST);
        if (authScheme != null) {
            Credentials creds = credsProvider
                    .getCredentials(new AuthScope(targetHost.getHostName(), targetHost.getPort()));
            if (creds == null) {
                throw new HttpException("No credentials for preemptive authentication");
            }
            authState.update(authScheme, creds);
        }
    }
}
 

@Override
public HttpUriRequest getRedirect(HttpRequest request, HttpResponse response, HttpContext context) throws ProtocolException {
    URI uri = getLocationURI(request, response, context);
    String method = request.getRequestLine().getMethod();
    if (HttpConstants.POST.equalsIgnoreCase(method)) {
        try {
            HttpRequestWrapper httpRequestWrapper = (HttpRequestWrapper) request;
            httpRequestWrapper.setURI(uri);
            httpRequestWrapper.removeHeaders("Content-Length");
            return httpRequestWrapper;
        } catch (Exception e) {
            e.printStackTrace();
        }
        return new HttpPost(uri);
    } else {
        return new HttpGet(uri);
    }
}
 

@Override
public boolean retryRequest(final IOException exception, final int executionCount, final HttpContext context) {
    final Throwable cause = ExceptionUtils.getRootCause(exception);
    if(cause instanceof RuntimeException) {
        log.error(String.format("Cancel retry request with execution count %d for failure %s", executionCount, cause));
        return false;
    }
    final boolean retry = super.retryRequest(exception, executionCount, context);
    if(retry) {
        log.info(String.format("Retry request with failure %s", exception));
    }
    else {
        log.warn(String.format("Cancel retry request with execution count %d for failure %s", executionCount, exception));
    }
    return retry;
}
 

@Override
public void handle(HttpRequest httpRequest, HttpResponse httpResponse, HttpContext httpContext) throws IOException {
    lock.lock();
    try {
        HttpEntity entity = ((HttpEntityEnclosingRequest) httpRequest).getEntity();
        String content = EntityUtils.toString(entity);

        replies.add(content);
        if (replies.size() == expected) {
            receivedExpectedMessages.signal();
        }

        httpResponse.setStatusCode(HttpStatus.SC_OK);
    } finally {
        lock.unlock();
    }
}
 

@Test
void shouldSaveConnectionDetailsForSecuredConnectionAndDoNotCheckStalenessForResponsesWithEntity()
{
    String protocol = "TLSv1.3";
    SSLSession sslSession = mock(SSLSession.class);
    when(sslSession.getProtocol()).thenReturn(protocol);
    HttpContext context = mock(HttpContext.class);
    ManagedHttpClientConnection connection = mock(ManagedHttpClientConnection.class);
    when(context.getAttribute(HttpCoreContext.HTTP_CONNECTION)).thenReturn(connection);
    when(connection.isOpen()).thenReturn(true);
    when(connection.getSSLSession()).thenReturn(sslSession);
    HttpResponse response = mock(HttpResponse.class);
    when(response.getEntity()).thenReturn(mock(HttpEntity.class));
    interceptor.process(response, context);
    verify(httpTestContext).putConnectionDetails(argThat(connectionDetails -> connectionDetails.isSecure()
            && protocol.equals(connectionDetails.getSecurityProtocol())));
    verify(connection, never()).isStale();
}
 

@Override
public HttpUriRequest getRedirect(final HttpRequest request, final HttpResponse response, final HttpContext context)
        throws ProtocolException
{
    final String method = request.getRequestLine().getMethod();
    HttpUriRequest redirect = super.getRedirect(request, response, context);
    if (HttpPost.METHOD_NAME.equalsIgnoreCase(method))
    {
        HttpPost httpPostRequest = new HttpPost(redirect.getURI());
        if (request instanceof HttpEntityEnclosingRequest)
        {
            httpPostRequest.setEntity(((HttpEntityEnclosingRequest) request).getEntity());
        }
        return httpPostRequest;
    }
    return redirect;
}
 

@Override
public CloseableHttpResponse execute(
    HttpHost target, HttpRequest request, HttpContext context) throws IOException, ClientProtocolException {
    Subsegment subsegment = getRecorder().beginSubsegment(target.getHostName());
    try {
        if (null != subsegment) {
            TracedHttpClient.addRequestInformation(subsegment, request, TracedHttpClient.getUrl(target, request));
        }
        CloseableHttpResponse response = super.execute(target, request, context);
        if (null != subsegment) {
            TracedResponseHandler.addResponseInformation(subsegment, response);
        }
        return response;
    } catch (Exception e) {
        if (null != subsegment) {
            subsegment.addException(e);
        }
        throw e;
    } finally {
        if (null != subsegment) {
            getRecorder().endSubsegment();
        }
    }
}
 

@Test
public void requestTimeoutEnabled_RequestCompletesWithinTimeout_EntityNotBufferedForStreamedResponse()
        throws Exception {
    ClientConfiguration config = new ClientConfiguration().withRequestTimeout(5 * 1000);
    ConnectionManagerAwareHttpClient rawHttpClient = createRawHttpClientSpy(config);

    HttpResponseProxy responseProxy = createHttpResponseProxySpy();
    doReturn(responseProxy).when(rawHttpClient).execute(any(HttpRequestBase.class), any(HttpContext.class));

    httpClient = new AmazonHttpClient(config, rawHttpClient, null);

    try {
        httpClient.requestExecutionBuilder().request(createMockGetRequest()).execute(new ErrorDuringUnmarshallingResponseHandler().leaveConnectionOpen());
        fail("Exception expected");
    } catch (AmazonClientException e) {
    }

    assertResponseWasNotBuffered(responseProxy);
}
 

/**
 * UnixHttpClient can execute the HttpRequest with the given host,
 * response handler and context.
 * @throws IOException If something goes wrong.
 */
@Test
public void executesRequestWithHostHandlerAndContext() throws IOException {
    final HttpHost host = new HttpHost("127.0.0.1");
    final HttpRequest req = Mockito.mock(HttpRequest.class);
    final ResponseHandler<String> handler = Mockito.mock(
        ResponseHandler.class
    );
    final HttpContext context = Mockito.mock(HttpContext.class);
    final HttpClient decorated = Mockito.mock(HttpClient.class);
    Mockito.when(
        decorated.execute(host, req, handler, context)
    ).thenReturn("executed");

    final HttpClient unix = new UnixHttpClient(() -> decorated);
    MatcherAssert.assertThat(
        unix.execute(host, req, handler, context),
        Matchers.equalTo("executed")
    );
    Mockito.verify(
        decorated, Mockito.times(1)
    ).execute(host, req, handler, context);
}
 

@Override
public void process(final HttpRequest request, final HttpContext context) {
    if(StringUtils.contains(request.getRequestLine().getUri(), "b2_authorize_account")) {
        // Skip setting token for
        if(log.isDebugEnabled()) {
            log.debug("Skip setting token in b2_authorize_account");
        }
        return;
    }
    switch(request.getRequestLine().getMethod()) {
        case "POST":
            // Do not override Authorization header for upload requests with upload URL token
            if(StringUtils.contains(request.getRequestLine().getUri(), "b2_upload_part")
                || StringUtils.contains(request.getRequestLine().getUri(), "b2_upload_file")) {
                break;
            }
        default:
            if(StringUtils.isNotBlank(authorizationToken)) {
                request.removeHeaders(HttpHeaders.AUTHORIZATION);
                request.addHeader(HttpHeaders.AUTHORIZATION, authorizationToken);
            }
    }
}
 

@Test
void testDoHttpHead() throws Exception
{
    CloseableHttpResponse closeableHttpResponse = mock(CloseableHttpResponse.class);
    HttpContext context = null;
    when(closeableHttpResponse.getEntity()).thenReturn(null);
    StatusLine statusLine = mock(StatusLine.class);
    int statusCode = HttpStatus.SC_MOVED_PERMANENTLY;
    Header[] headers = { header };
    when(closeableHttpResponse.getAllHeaders()).thenReturn(headers);
    when(statusLine.getStatusCode()).thenReturn(statusCode);
    when(closeableHttpResponse.getStatusLine()).thenReturn(statusLine);
    when(closeableHttpClient.execute(isA(HttpHead.class), eq(context)))
            .thenAnswer(getAnswerWithSleep(closeableHttpResponse));
    HttpResponse httpResponse = httpClient.doHttpHead(URI_TO_GO);
    assertEquals(HEAD, httpResponse.getMethod());
    assertEquals(URI_TO_GO, httpResponse.getFrom());
    assertEquals(statusCode, httpResponse.getStatusCode());
    assertThat(httpResponse.getResponseTimeInMs(), greaterThan(0L));
    assertThat(httpResponse.getResponseHeaders(), is(equalTo(headers)));
}
 

@Test
public void executeAsyncDelegatesToConfiguredAsyncClient() {

    // given
    HttpClient client = Mockito.spy(createDefaultTestObject());
    CloseableHttpAsyncClient asyncClient = mockAsyncClient(client);

    BatchRequest request = createDefaultTestBatchRequest();

    // when
    client.executeAsync(request, createMockTestResultHandler());

    // then
    verify(client).getAsyncClient();
    verify(asyncClient).execute(
            any(HttpAsyncRequestProducer.class),
            any(HttpAsyncResponseConsumer.class),
            any(HttpContext.class),
            any(FutureCallback.class));

}
 

/**
 * {@inheritDoc}
 */
@Override
public HttpRoute determineRoute(final HttpHost host, final HttpRequest request,
        final HttpContext context) throws HttpException {

    Boolean ignoreNoProxy = (Boolean) context.getAttribute(IGNORE_NO_PROXY);
    URI requestUri = (URI) context.getAttribute(REQUEST_URI);

    if (proxyHost != null &&
            (Boolean.TRUE.equals(ignoreNoProxy) || useProxyFor(requestUri))) {
        if (log.isDebugEnabled()) {
            log.debug("Using proxy: " + proxyHost);
        }
        return super.determineRoute(host, request, context);
    }
    if (log.isDebugEnabled()) {
        log.debug("Using a direct connection (no proxy)");
    }
    // Return direct route
    return new HttpRoute(host);
}
 

@Override
public HttpResponse execute(
    final HttpHost target, final HttpRequest request,
    final HttpContext context
) throws IOException {
    throw new UnsupportedOperationException();
}
 

/**
 * {@inheritDoc}
 */
@Override
public Socket createSocket(final HttpContext context) throws IOException {
    final HttpHost socksProxy = getSocksProxy(context);
    if (socksProxy != null) {
        return createSocketWithSocksProxy(socksProxy);
    }
    return super.createSocket(context);
}
 

@Test
void testExecuteHttpRequestIOException() throws IOException
{
    when(httpClient.execute(argThat(e -> e instanceof HttpRequestBase && URL.equals(e.getURI().toString())),
            nullable(HttpContext.class))).thenThrow(new IOException());
    assertThrows(IOException.class,
        () -> httpRequestExecutor.executeHttpRequest(HttpMethod.GET, URL, Optional.empty()));
    verify(httpTestContext).releaseRequestData();
}
 

@Override
public Proxy determineProxy(HttpHost host, HttpRequest request, HttpContext context, IPPool ipPool,
        CrawlerSession crawlerSession) {
    HttpClientContext httpClientContext = HttpClientContext.adapt(context);

    Proxy bind = (Proxy) context.getAttribute(VSCrawlerConstant.VSCRAWLER_AVPROXY_KEY);
    String accessUrl = null;
    if (request instanceof HttpRequestWrapper || request instanceof HttpGet) {
        accessUrl = HttpUriRequest.class.cast(request).getURI().toString();
    }
    if (!PoolUtil.isDungProxyEnabled(httpClientContext)) {
        log.info("{}不会被代理", accessUrl);
        return null;
    }

    if (bind == null || bind.isDisable()) {
        bind = ipPool.getIP(host.getHostName(), accessUrl);
    }

    if (bind == null) {
        return null;
    }

    log.info("{} 当前使用IP为:{}:{}", host.getHostName(), bind.getIp(), bind.getPort());
    // 将绑定IP放置到context,用于后置拦截器统计这个IP的使用情况
    return bind;
}
 

@Test
public void testUserAgentPrefixAndSuffixAreAdded() throws Exception {
    String prefix = "somePrefix", suffix = "someSuffix";
    Request<?> request = mockRequest(SERVER_NAME, HttpMethodName.PUT, URI_NAME, true);

    HttpResponseHandler<AmazonWebServiceResponse<Object>> handler = createStubResponseHandler();
    EasyMock.replay(handler);
    ClientConfiguration config =
            new ClientConfiguration().withUserAgentPrefix(prefix).withUserAgentSuffix(suffix);

    Capture<HttpRequestBase> capturedRequest = new Capture<HttpRequestBase>();

    EasyMock.reset(httpClient);
    EasyMock
            .expect(httpClient.execute(
                    EasyMock.capture(capturedRequest), EasyMock.<HttpContext>anyObject()))
            .andReturn(createBasicHttpResponse())
            .once();
    EasyMock.replay(httpClient);

    AmazonHttpClient client = new AmazonHttpClient(config, httpClient, null);

    client.requestExecutionBuilder().request(request).execute(handler);

    String userAgent = capturedRequest.getValue().getFirstHeader("User-Agent").getValue();
    Assert.assertTrue(userAgent.startsWith(prefix));
    Assert.assertTrue(userAgent.endsWith(suffix));
}
 

@Override
protected HttpContext createHttpContext(HttpMethod httpMethod, URI uri) {
	final AuthCache authCache = new BasicAuthCache();

	final BasicScheme basicAuth = new BasicScheme();
	authCache.put(host, basicAuth);

	final BasicHttpContext localcontext = new BasicHttpContext();
	localcontext.setAttribute(HttpClientContext.AUTH_CACHE, authCache);
	return localcontext;
}
 

private static ManagedHttpClientConnection mockHttpConnection(Boolean stale, HttpContext context, Boolean closed)
{
    ManagedHttpClientConnection connection = mock(ManagedHttpClientConnection.class);
    when(connection.isStale()).thenReturn(stale);
    when(context.getAttribute(HttpCoreContext.HTTP_CONNECTION)).thenReturn(connection);
    when(connection.isOpen()).thenReturn(closed);
    return connection;
}
 

@Test
void shouldMarkValidationAsBrokenIfExceptionOccurs() throws IOException
{
    IOException ioException = new IOException();
    when(httpClient.execute(argThat(r -> HEAD.equals(r.getMethod())), any(HttpContext.class)))
        .thenThrow(ioException);
    ResourceValidation resourceValidation = new ResourceValidation(FIRST, CSS_SELECTOR);
    ResourceValidation result = resourceValidator.perform(resourceValidation);
    assertEquals(CheckStatus.BROKEN, result.getCheckStatus());
    verify(httpClient).execute(any(HttpUriRequest.class), any(HttpContext.class));
    verify(softAssert).recordFailedAssertion(eq("Exception occured during check of: https://vividus.org"),
            eq(ioException));
}
 

@Test
void shouldSaveNoConnectionDetailsForStaleConnection()
{
    HttpContext context = mock(HttpContext.class);
    mockHttpConnection(Boolean.TRUE, context, Boolean.TRUE);
    intercept(context);
    verifyNoInteractions(httpTestContext);
}
 

public final HttpContext createContext(URI uri,
        UsernamePasswordCredentials creds) throws Exception {
    CredentialsProvider credsProvider = new BasicCredentialsProvider();
    credsProvider.setCredentials(
            new AuthScope(uri.getHost(), uri.getPort()),
            creds);
    org.apache.http.HttpHost host = new org.apache.http.HttpHost(uri.getHost(), uri.getPort(), uri.getScheme());
    AuthCache authCache = new BasicAuthCache();
    BasicScheme basicAuth = new BasicScheme();
    authCache.put(host, basicAuth);
    HttpClientContext context1 = HttpClientContext.create();
    context1.setCredentialsProvider(credsProvider);
    context1.setAuthCache(authCache);
    return context1;
}
 

@Override
protected StoregateApiClient connect(final Proxy proxy, final HostKeyCallback key, final LoginCallback prompt) {
    final HttpClientBuilder configuration = builder.build(proxy, this, prompt);
    authorizationService = new OAuth2RequestInterceptor(builder.build(proxy, this, prompt).addInterceptorLast(new HttpRequestInterceptor() {
        @Override
        public void process(final HttpRequest request, final HttpContext context) {
            request.addHeader(HttpHeaders.AUTHORIZATION,
                String.format("Basic %s", Base64.encodeToString(String.format("%s:%s", host.getProtocol().getOAuthClientId(), host.getProtocol().getOAuthClientSecret()).getBytes(StandardCharsets.UTF_8), false)));
        }
    }).build(),
        host).withRedirectUri(CYBERDUCK_REDIRECT_URI.equals(host.getProtocol().getOAuthRedirectUrl()) ? host.getProtocol().getOAuthRedirectUrl() :
        Scheme.isURL(host.getProtocol().getOAuthRedirectUrl()) ? host.getProtocol().getOAuthRedirectUrl() : new HostUrlProvider().withUsername(false).withPath(true).get(
            host.getProtocol().getScheme(), host.getPort(), null, host.getHostname(), host.getProtocol().getOAuthRedirectUrl())
    );
    // Force login even if browser session already exists
    authorizationService.withParameter("prompt", "login");
    configuration.setServiceUnavailableRetryStrategy(new OAuth2ErrorResponseInterceptor(host, authorizationService, prompt));
    configuration.addInterceptorLast(authorizationService);
    final CloseableHttpClient apache = configuration.build();
    final StoregateApiClient client = new StoregateApiClient(apache);
    final int timeout = PreferencesFactory.get().getInteger("connection.timeout.seconds") * 1000;
    client.setConnectTimeout(timeout);
    client.setBasePath(new HostUrlProvider().withUsername(false).withPath(true).get(host.getProtocol().getScheme(), host.getPort(),
        null, host.getHostname(), host.getProtocol().getContext()));
    client.setHttpClient(ClientBuilder.newClient(new ClientConfig()
        .register(new InputStreamProvider())
        .register(MultiPartFeature.class)
        .register(new JSON())
        .register(JacksonFeature.class)
        .connectorProvider(new HttpComponentsProvider(apache))));
    client.setUserAgent(new PreferencesUseragentProvider().get());
    return client;
}
 

private void configureRetryHandler(DefaultHttpClient httpClient) {
    httpClient.setHttpRequestRetryHandler(new HttpRequestRetryHandler() {
        public boolean retryRequest(IOException exception, int executionCount, HttpContext context) {
            return false;
        }
    });
}
 

@Override
protected HttpHost determineProxy(HttpHost host, HttpRequest request, HttpContext context) throws HttpException {
    HttpClientContext httpClientContext = HttpClientContext.adapt(context);
    Proxy proxy = proxyPlanner.determineProxy(host, request, context, ipPool, crawlerSession);

    if (proxy == null) {
        return null;
    }
    if (log.isDebugEnabled()) {
        log.debug("{} 当前使用IP为:{}:{}", host.getHostName(), proxy.getIp(), proxy.getPort());
    }
    context.setAttribute(VSCRAWLER_AVPROXY_KEY, proxy);
    crawlerSession.setExtInfo(VSCRAWLER_AVPROXY_KEY, proxy);

    if (proxy.getAuthenticationHeaders() != null) {
        for (Header header : proxy.getAuthenticationHeaders()) {
            request.addHeader(header);
        }
    }

    if (StringUtils.isNotEmpty(proxy.getUsername()) && StringUtils.isNotEmpty(proxy.getPassword())) {
        BasicCredentialsProvider credsProvider1 = new BasicCredentialsProvider();
        httpClientContext.setCredentialsProvider(credsProvider1);
        credsProvider1.setCredentials(AuthScope.ANY,
                new UsernamePasswordCredentials(proxy.getUsername(), proxy.getPassword()));
    }
    return new HttpHost(proxy.getIp(), proxy.getPort());
}
 

/**
 * retry逻辑
 */
@Override
public boolean retryRequest(HttpResponse response, int executionCount, HttpContext context) {
	if (response.getStatusLine().getStatusCode() == (408 | 502 | 503 | 504)
			&& executionCount <= this.executionCount) {
		return true;
	} else
		return false;
}