Java Code Examples for org.apache.http.protocol.HttpContext#getAttribute()

@Override
public boolean retryRequest(final IOException exception, final int executionCount, final HttpContext context) {
    if(super.retryRequest(exception, executionCount, context)) {
        final Object attribute = context.getAttribute(HttpCoreContext.HTTP_REQUEST);
        if(attribute instanceof HttpUriRequest) {
            final HttpUriRequest method = (HttpUriRequest) attribute;
            log.warn(String.format("Retrying request %s", method));
            try {
                // Build the authorization string for the method.
                authorizer.authorizeHttpRequest(method, context, null);
                return true;
            }
            catch(ServiceException e) {
                log.warn("Unable to generate updated authorization string for retried request", e);
            }
        }
    }
    return false;
}
 

@Override
public void process(HttpRequest request, HttpContext context) throws HttpException, IOException {
    AuthState authState = (AuthState) context.getAttribute(ClientContext.TARGET_AUTH_STATE);

    if (authState.getAuthScheme() == null) {
        AuthScheme authScheme = (AuthScheme) context.getAttribute("preemptive-auth");
        CredentialsProvider credsProvider = (CredentialsProvider) context
                .getAttribute(ClientContext.CREDS_PROVIDER);
        HttpHost targetHost = (HttpHost) context.getAttribute(ExecutionContext.HTTP_TARGET_HOST);
        if (authScheme != null) {
            Credentials creds = credsProvider
                    .getCredentials(new AuthScope(targetHost.getHostName(), targetHost.getPort()));
            if (creds == null) {
                throw new HttpException("No credentials for preemptive authentication");
            }
            authState.update(authScheme, creds);
        }
    }
}
 

/**
 * {@inheritDoc}
 */
@Override
public HttpRoute determineRoute(final HttpHost host, final HttpRequest request,
        final HttpContext context) throws HttpException {

    Boolean ignoreNoProxy = (Boolean) context.getAttribute(IGNORE_NO_PROXY);
    URI requestUri = (URI) context.getAttribute(REQUEST_URI);

    if (proxyHost != null &&
            (Boolean.TRUE.equals(ignoreNoProxy) || useProxyFor(requestUri))) {
        if (log.isDebugEnabled()) {
            log.debug("Using proxy: " + proxyHost);
        }
        return super.determineRoute(host, request, context);
    }
    if (log.isDebugEnabled()) {
        log.debug("Using a direct connection (no proxy)");
    }
    // Return direct route
    return new HttpRoute(host);
}
 

public void process(final HttpRequest request, final HttpContext context) throws HttpException, IOException {

            AuthState authState = (AuthState) context.getAttribute(ClientContext.TARGET_AUTH_STATE);

            if (authState.getAuthScheme() != null || authState.hasAuthOptions()) {
                return;
            }

            // If no authState has been established and this is a PUT or POST request, add preemptive authorisation
            String requestMethod = request.getRequestLine().getMethod();
            if (requestMethod.equals(HttpPut.METHOD_NAME) || requestMethod.equals(HttpPost.METHOD_NAME)) {
                CredentialsProvider credentialsProvider = (CredentialsProvider) context.getAttribute(ClientContext.CREDS_PROVIDER);
                HttpHost targetHost = (HttpHost) context.getAttribute(ExecutionContext.HTTP_TARGET_HOST);
                Credentials credentials = credentialsProvider.getCredentials(new AuthScope(targetHost.getHostName(), targetHost.getPort()));
                if (credentials == null) {
                    throw new HttpException("No credentials for preemptive authentication");
                }
                authState.update(authScheme, credentials);
            }
        }
 

public void process(final HttpRequest request, final HttpContext context) throws HttpException, IOException {

            AuthState authState = (AuthState) context.getAttribute(ClientContext.TARGET_AUTH_STATE);

            if (authState.getAuthScheme() != null || authState.hasAuthOptions()) {
                return;
            }

            // If no authState has been established and this is a PUT or POST request, add preemptive authorisation
            String requestMethod = request.getRequestLine().getMethod();
            if (requestMethod.equals(HttpPut.METHOD_NAME) || requestMethod.equals(HttpPost.METHOD_NAME)) {
                CredentialsProvider credentialsProvider = (CredentialsProvider) context.getAttribute(ClientContext.CREDS_PROVIDER);
                HttpHost targetHost = (HttpHost) context.getAttribute(ExecutionContext.HTTP_TARGET_HOST);
                Credentials credentials = credentialsProvider.getCredentials(new AuthScope(targetHost.getHostName(), targetHost.getPort()));
                if (credentials == null) {
                    throw new HttpException("No credentials for preemptive authentication");
                }
                authState.update(authScheme, credentials);
            }
        }
 

@Override
public void process(HttpResponse response, HttpContext context)
{
    ManagedHttpClientConnection routedConnection = (ManagedHttpClientConnection) context
            .getAttribute(HttpCoreContext.HTTP_CONNECTION);
    // Connection may be stale, when no response body is returned
    if (routedConnection.isOpen() && (response.getEntity() != null || !routedConnection.isStale()))
    {
        SSLSession sslSession = routedConnection.getSSLSession();
        boolean secure = sslSession != null;
        ConnectionDetails connectionDetails = new ConnectionDetails();
        connectionDetails.setSecure(secure);
        if (secure)
        {
            connectionDetails.setSecurityProtocol(sslSession.getProtocol());
        }
        httpTestContext.putConnectionDetails(connectionDetails);
    }
}
 

static Tags generateTagsForRoute(HttpContext context) {
    String targetScheme = "UNKNOWN";
    String targetHost = "UNKNOWN";
    String targetPort = "UNKNOWN";
    Object routeAttribute = context.getAttribute("http.route");
    if (routeAttribute instanceof HttpRoute) {
        HttpHost host = ((HttpRoute) routeAttribute).getTargetHost();
        targetScheme = host.getSchemeName();
        targetHost = host.getHostName();
        targetPort = String.valueOf(host.getPort());
    }
    return Tags.of(
            "target.scheme", targetScheme,
            "target.host", targetHost,
            "target.port", targetPort
    );
}
 

@Override
public boolean retryRequest(IOException exception,
		int executionCount, HttpContext context) {
	if (executionCount >= RETRY_TIME) {
		// Do not retry if over max retry count
		return false;
	}
	if (exception instanceof InterruptedIOException) {
		// Timeout
		return false;
	}
	if (exception instanceof UnknownHostException) {
		// Unknown host
		return false;
	}
	if (exception instanceof ConnectException) {
		// Connection refused
		return false;
	}
	if (exception instanceof SSLException) {
		// SSL handshake exception
		return false;
	}
	HttpRequest request = (HttpRequest) context
			.getAttribute(ExecutionContext.HTTP_REQUEST);
	boolean idempotent = !(request instanceof HttpEntityEnclosingRequest);
	if (idempotent) {
		// Retry if the request is considered idempotent
		return true;
	}
	return false;
}
 

@Override public boolean retryRequest(
        final IOException exception,
        int executionCount,
        final HttpContext context) {
    boolean retry = super.retryRequest(exception, executionCount, context);
    if (retry) {
        AWSRequestMetrics awsRequestMetrics = (AWSRequestMetrics) context
                .getAttribute(AWSRequestMetrics.class.getSimpleName());
        if (awsRequestMetrics != null) {
            awsRequestMetrics.incrementCounter(Field.HttpClientRetryCount);
        }
    }
    return retry;
}
 

@Override
public void process(org.apache.http.HttpRequest request, HttpContext httpContext) throws HttpException, IOException {
    HttpRequest actual = new HttpRequest();
    int id = counter.incrementAndGet();
    String uri = (String) httpContext.getAttribute(ApacheHttpClient.URI_CONTEXT_KEY);
    String method = request.getRequestLine().getMethod();
    actual.setUri(uri);
    actual.setMethod(method);        
    StringBuilder sb = new StringBuilder();
    sb.append("request:\n").append(id).append(" > ").append(method).append(' ').append(uri).append('\n');
    HttpLogModifier requestModifier = logModifier == null ? null : logModifier.enableForUri(uri) ? logModifier : null;
    LoggingUtils.logHeaders(requestModifier, sb, id, '>', request, actual);
    if (request instanceof HttpEntityEnclosingRequest) {
        HttpEntityEnclosingRequest entityRequest = (HttpEntityEnclosingRequest) request;
        HttpEntity entity = entityRequest.getEntity();
        if (LoggingUtils.isPrintable(entity)) {
            LoggingEntityWrapper wrapper = new LoggingEntityWrapper(entity); // todo optimize, preserve if stream
            String buffer = FileUtils.toString(wrapper.getContent());
            if (context.getConfig().isLogPrettyRequest()) {
                buffer = FileUtils.toPrettyString(buffer);
            }
            if (requestModifier != null) {
                buffer = requestModifier.request(uri, buffer);
            }
            sb.append(buffer).append('\n');
            actual.setBody(wrapper.getBytes());
            entityRequest.setEntity(wrapper);
        }
    }
    context.setPrevRequest(actual);
    context.logger.debug(sb.toString());
    actual.startTimer();
}
 

@Override
protected HttpResponse doSendRequest(
        final HttpRequest request,
        final HttpClientConnection conn,
        final HttpContext context)
            throws IOException, HttpException {
    AWSRequestMetrics awsRequestMetrics = (AWSRequestMetrics) context
            .getAttribute(AWSRequestMetrics.class.getSimpleName());

    if (awsRequestMetrics == null) {
        return super.doSendRequest(request, conn, context);
    }
    if (conn instanceof ManagedHttpClientConnection) {
        ManagedHttpClientConnection managedConn = (ManagedHttpClientConnection)conn;
        Socket sock = managedConn.getSocket();
        if (sock instanceof SdkMetricsSocket) {
            SdkMetricsSocket sdkMetricsSocket = (SdkMetricsSocket)sock;
            sdkMetricsSocket.setMetrics(awsRequestMetrics);
        } else if (sock instanceof SdkSSLMetricsSocket) {
            SdkSSLMetricsSocket sdkSSLMetricsSocket = (SdkSSLMetricsSocket)sock;
            sdkSSLMetricsSocket.setMetrics(awsRequestMetrics);
        }
    }
    awsRequestMetrics.startEvent(Field.HttpClientSendRequestTime);
    try {
        return super.doSendRequest(request, conn, context);
    } finally {
        awsRequestMetrics.endEvent(Field.HttpClientSendRequestTime);
    }
}
 

@Override
public boolean retryRequest(IOException exception, int executionCount, HttpContext context) {
    boolean retry = true;

    Boolean b = (Boolean) context.getAttribute(ExecutionContext.HTTP_REQ_SENT);
    boolean sent = (b != null && b);

    if (executionCount > maxRetries) {
        // Do not retry if over max retry count
        retry = false;
    } else if (isInList(exceptionWhitelist, exception)) {
        // immediately retry if error is whitelisted
        retry = true;
    } else if (isInList(exceptionBlacklist, exception)) {
        // immediately cancel retry if the error is blacklisted
        retry = false;
    } else if (!sent) {
        // for most other errors, retry only if request hasn't been fully sent yet
        retry = true;
    }

    if (retry) {
        // resend all idempotent requests
        HttpUriRequest currentReq = (HttpUriRequest) context.getAttribute(ExecutionContext.HTTP_REQUEST);
        if (currentReq == null) {
            return false;
        }
    }

    if (retry) {
        SystemClock.sleep(retrySleepTimeMS);
    } else {
        exception.printStackTrace();
    }

    return retry;
}
 

@Override
public Proxy determineProxy(HttpHost host, HttpRequest request, HttpContext context, IPPool ipPool,
        CrawlerSession crawlerSession) {
    HttpClientContext httpClientContext = HttpClientContext.adapt(context);

    Proxy bind = (Proxy) context.getAttribute(VSCrawlerConstant.VSCRAWLER_AVPROXY_KEY);
    String accessUrl = null;
    if (request instanceof HttpRequestWrapper || request instanceof HttpGet) {
        accessUrl = HttpUriRequest.class.cast(request).getURI().toString();
    }
    if (!PoolUtil.isDungProxyEnabled(httpClientContext)) {
        log.info("{}不会被代理", accessUrl);
        return null;
    }

    if (bind == null || bind.isDisable()) {
        bind = ipPool.getIP(host.getHostName(), accessUrl);
    }

    if (bind == null) {
        return null;
    }

    log.info("{} 当前使用IP为:{}:{}", host.getHostName(), bind.getIp(), bind.getPort());
    // 将绑定IP放置到context,用于后置拦截器统计这个IP的使用情况
    return bind;
}
 

public void process(final HttpRequest request, final HttpContext context) throws HttpException, IOException {
    AuthState authState = (AuthState) context.getAttribute(ClientContext.TARGET_AUTH_STATE);
    CredentialsProvider credsProvider = (CredentialsProvider) context.getAttribute(
            ClientContext.CREDS_PROVIDER);
    HttpHost targetHost = (HttpHost) context.getAttribute(ExecutionContext.HTTP_TARGET_HOST);

    if (authState.getAuthScheme() == null) {
        AuthScope authScope = new AuthScope(targetHost.getHostName(), targetHost.getPort());
        Credentials creds = credsProvider.getCredentials(authScope);
        if (creds != null) {
            authState.setAuthScheme(new BasicScheme());
            authState.setCredentials(creds);
        }
    }
}
 

@Override
public ClientHttpRequest createRequest(URI uri, HttpMethod httpMethod) throws IOException {
	HttpUriRequest httpRequest = createHttpUriRequest(httpMethod, uri);
	postProcessHttpRequest(httpRequest);
	HttpContext context = createHttpContext(httpMethod, uri);
	if (context == null) {
		context = HttpClientContext.create();
	}

	// Request configuration not set in the context
	if (context.getAttribute(HttpClientContext.REQUEST_CONFIG) == null) {
		// Use request configuration given by the user, when available
		RequestConfig config = null;
		if (httpRequest instanceof Configurable) {
			config = ((Configurable) httpRequest).getConfig();
		}
		if (config == null) {
			config = createRequestConfig(getHttpClient());
		}
		if (config != null) {
			context.setAttribute(HttpClientContext.REQUEST_CONFIG, config);
		}
	}

	if (this.bufferRequestBody) {
		return new HttpComponentsClientHttpRequest(getHttpClient(), httpRequest, context);
	}
	else {
		return new HttpComponentsStreamingClientHttpRequest(getHttpClient(), httpRequest, context);
	}
}
 

@Override
public ClientHttpRequest createRequest(URI uri, HttpMethod httpMethod) throws IOException {
	HttpClient client = getHttpClient();

	HttpUriRequest httpRequest = createHttpUriRequest(httpMethod, uri);
	postProcessHttpRequest(httpRequest);
	HttpContext context = createHttpContext(httpMethod, uri);
	if (context == null) {
		context = HttpClientContext.create();
	}

	// Request configuration not set in the context
	if (context.getAttribute(HttpClientContext.REQUEST_CONFIG) == null) {
		// Use request configuration given by the user, when available
		RequestConfig config = null;
		if (httpRequest instanceof Configurable) {
			config = ((Configurable) httpRequest).getConfig();
		}
		if (config == null) {
			config = createRequestConfig(client);
		}
		if (config != null) {
			context.setAttribute(HttpClientContext.REQUEST_CONFIG, config);
		}
	}

	if (this.bufferRequestBody) {
		return new HttpComponentsClientHttpRequest(client, httpRequest, context);
	}
	else {
		return new HttpComponentsStreamingClientHttpRequest(client, httpRequest, context);
	}
}
 

private static String getSession() throws IOException {
    CloseableHttpClient httpClient = HttpClients.createDefault();
    HttpGet httpGet = new HttpGet(TEST_SERVICE_BASE_URL);
    HttpContext httpContext = new BasicHttpContext();
    httpClient.execute(httpGet, httpContext);
    HttpUriRequest currentReq = (HttpUriRequest) httpContext.getAttribute(HttpCoreContext.HTTP_REQUEST);
    return currentReq.getURI().getPath().split("/")[2];
}
 

static HttpHost getSocksProxy(final HttpContext context) {
    return (HttpHost) context.getAttribute(SOCKS_PROXY);
}
 

@Override
public void process(final HttpRequest request, final HttpContext context)
        throws IOException
{
    String method = request.getRequestLine().getMethod();

    URI uri = URI.create(request.getRequestLine().getUri());
    URIBuilder uriBuilder = new URIBuilder(uri);

    Map<String, List<String>> parameters = new TreeMap<>(CASE_INSENSITIVE_ORDER);
    for (NameValuePair parameter : uriBuilder.getQueryParams()) {
        parameters.computeIfAbsent(parameter.getName(), key -> new ArrayList<>())
                .add(parameter.getValue());
    }

    Map<String, String> headers = Arrays.stream(request.getAllHeaders())
            .collect(toImmutableMap(Header::getName, Header::getValue));

    InputStream content = null;
    if (request instanceof HttpEntityEnclosingRequest) {
        HttpEntityEnclosingRequest enclosingRequest = (HttpEntityEnclosingRequest) request;
        if (enclosingRequest.getEntity() != null) {
            content = enclosingRequest.getEntity().getContent();
        }
    }

    DefaultRequest<?> awsRequest = new DefaultRequest<>(SERVICE_NAME);

    HttpHost host = (HttpHost) context.getAttribute(HTTP_TARGET_HOST);
    if (host != null) {
        awsRequest.setEndpoint(URI.create(host.toURI()));
    }
    awsRequest.setHttpMethod(HttpMethodName.fromValue(method));
    awsRequest.setResourcePath(uri.getRawPath());
    awsRequest.setContent(content);
    awsRequest.setParameters(parameters);
    awsRequest.setHeaders(headers);

    signer.sign(awsRequest, credentialsProvider.getCredentials());

    Header[] newHeaders = awsRequest.getHeaders().entrySet().stream()
            .map(entry -> new BasicHeader(entry.getKey(), entry.getValue()))
            .toArray(Header[]::new);

    request.setHeaders(newHeaders);

    InputStream newContent = awsRequest.getContent();
    checkState(newContent == null || request instanceof HttpEntityEnclosingRequest);
    if (newContent != null) {
        BasicHttpEntity entity = new BasicHttpEntity();
        entity.setContent(newContent);
        ((HttpEntityEnclosingRequest) request).setEntity(entity);
    }
}
 

/**
 * Print AuthState in HttpContext for debugging purpose.
 * <p>
 * If the proxy server requires 407 and resend cycle, this method logs as followings, for Basic Auth:
 * <ul><li>state:UNCHALLENGED;</li>
 * <li>state:CHALLENGED;auth scheme:basic;credentials present</li></ul>
 * </p>
 * <p>
 * For Digest Auth:
 * <ul><li>state:UNCHALLENGED;</li>
 * <li>state:CHALLENGED;auth scheme:digest;credentials present</li></ul>
 * </p>
 * <p>
 * But if the proxy uses the same connection, it doesn't return 407, in such case
 * this method is called only once with:
 * <ul><li>state:UNCHALLENGED</li></ul>
 * </p>
 */
private void debugProxyAuthState(HttpContext context) {
    final AuthState proxyAuthState;
    if (shouldCheckProxyAuth()
            && logger.isDebugEnabled()
            && (proxyAuthState = (AuthState)context.getAttribute("http.auth.proxy-scope")) != null){
        logger.debug("authProxyScope={}", proxyAuthState);
    }
}