org.apache.http.impl.NoConnectionReuseStrategy Java Examples

public static CrawlerHttpClientBuilder setDefault(CrawlerHttpClientBuilder proxyFeedBackDecorateHttpClientBuilder) {
    SocketConfig socketConfig = SocketConfig.custom().setSoKeepAlive(true).setSoLinger(-1).setSoReuseAddress(false)
            .setSoTimeout(ProxyConstant.SOCKETSO_TIMEOUT).setTcpNoDelay(true).build();
    proxyFeedBackDecorateHttpClientBuilder.setDefaultSocketConfig(socketConfig)
            // .setSSLSocketFactory(sslConnectionSocketFactory)
            // dungproxy0.0.6之后的版本,默认忽略https证书检查
            .setRedirectStrategy(new LaxRedirectStrategy())
            // 注意,这里使用ua生产算法自动产生ua,如果是mobile,可以使用
            // com.virjar.vscrawler.core.net.useragent.UserAgentBuilder.randomAppUserAgent()
            .setUserAgent(UserAgentBuilder.randomUserAgent())
            // 对于爬虫来说,连接池没啥卵用,直接禁止掉(因为我们可能创建大量HttpClient,每个HttpClient一个连接池,会把系统socket资源撑爆)
            // 测试开80个httpClient抓数据大概一个小时系统就会宕机
            .setConnectionReuseStrategy(NoConnectionReuseStrategy.INSTANCE)
            // 现在有些网站的网络响应头部数据有非ASCII数据,httpclient默认只使用ANSI标准解析header,这可能导致带有中文的header数据无法解析
            .setDefaultConnectionConfig(ConnectionConfig.custom().setCharset(Charsets.UTF_8).build());
    return proxyFeedBackDecorateHttpClientBuilder;
}
 

public CloseableHttpClient createHttpClient(HttpClientConnectionManager httpClientConnectionManager,
                                         RaptorHttpClientProperties httpClientProperties) {

    RequestConfig defaultRequestConfig = RequestConfig.custom()
            .setConnectTimeout(httpClientProperties.getConnectionTimeout())
            .setSocketTimeout(httpClientProperties.getReadTimeout())
            .setRedirectsEnabled(httpClientProperties.isFollowRedirects())
            .build();

    HttpRequestRetryHandler retryHandler = new DefaultHttpRequestRetryHandler(httpClientProperties.getRetryCount(),
            httpClientProperties.isRequestSentRetryEnabled());
    HttpClientBuilder httpClientBuilder = HttpClientBuilder.create().disableContentCompression()
            .disableCookieManagement()
            .useSystemProperties()
            .setRetryHandler(retryHandler)
            .setConnectionManager(httpClientConnectionManager)
            .setDefaultRequestConfig(defaultRequestConfig);

    if(!keepAlive){
        httpClientBuilder.setConnectionReuseStrategy(NoConnectionReuseStrategy.INSTANCE);
    }

    return httpClientBuilder.build();
}
 

/**
 * Default constructor.
 * @param defaultFilterProcessesUrl the url of the filter
 */
public GitHubAuthFilter(final String defaultFilterProcessesUrl) {
	super(defaultFilterProcessesUrl);
	this.jreader = ParaObjectUtils.getJsonReader(Map.class);
	int timeout = 30 * 1000;
	this.httpclient = HttpClientBuilder.create().
			setConnectionReuseStrategy(new NoConnectionReuseStrategy()).
			setDefaultRequestConfig(RequestConfig.custom().
					setConnectTimeout(timeout).
					setConnectionRequestTimeout(timeout).
					setCookieSpec(CookieSpecs.STANDARD).
					setSocketTimeout(timeout).
					build()).
			build();
}
 

/**
 * Create an {@link HttpClientBuilder} object. Invoked by {@link #createHttpClient(Http4FileSystemConfigBuilder, GenericFileName, FileSystemOptions)}.
 *
 * @param builder Configuration options builder for HTTP4 provider
 * @param rootName The root path
 * @param fileSystemOptions The FileSystem options
 * @return an {@link HttpClientBuilder} object
 * @throws FileSystemException if an error occurs
 */
protected HttpClientBuilder createHttpClientBuilder(final Http4FileSystemConfigBuilder builder, final GenericFileName rootName,
        final FileSystemOptions fileSystemOptions) throws FileSystemException {
    final List<Header> defaultHeaders = new ArrayList<>();
    defaultHeaders.add(new BasicHeader(HTTP.USER_AGENT, builder.getUserAgent(fileSystemOptions)));

    final ConnectionReuseStrategy connectionReuseStrategy = builder.isKeepAlive(fileSystemOptions)
            ? DefaultConnectionReuseStrategy.INSTANCE
            : NoConnectionReuseStrategy.INSTANCE;

    final HttpClientBuilder httpClientBuilder =
            HttpClients.custom()
            .setRoutePlanner(createHttpRoutePlanner(builder, fileSystemOptions))
            .setConnectionManager(createConnectionManager(builder, fileSystemOptions))
            .setSSLContext(createSSLContext(builder, fileSystemOptions))
            .setSSLHostnameVerifier(createHostnameVerifier(builder, fileSystemOptions))
            .setConnectionReuseStrategy(connectionReuseStrategy)
            .setDefaultRequestConfig(createDefaultRequestConfig(builder, fileSystemOptions))
            .setDefaultHeaders(defaultHeaders)
            .setDefaultCookieStore(createDefaultCookieStore(builder, fileSystemOptions));

    if (!builder.getFollowRedirect(fileSystemOptions)) {
        httpClientBuilder.disableRedirectHandling();
    }

    return httpClientBuilder;
}
 

public ListenerThread(final ApiServer requestHandler, final int port) {
    try {
        _serverSocket = new ServerSocket(port);
    } catch (final IOException ioex) {
        s_logger.error("error initializing api server", ioex);
        return;
    }

    _params = new BasicHttpParams();
    _params.setIntParameter(CoreConnectionPNames.SO_TIMEOUT, 30000)
    .setIntParameter(CoreConnectionPNames.SOCKET_BUFFER_SIZE, 8 * 1024)
    .setBooleanParameter(CoreConnectionPNames.STALE_CONNECTION_CHECK, false)
    .setBooleanParameter(CoreConnectionPNames.TCP_NODELAY, true)
    .setParameter(CoreProtocolPNames.ORIGIN_SERVER, "HttpComponents/1.1");

    // Set up the HTTP protocol processor
    final BasicHttpProcessor httpproc = new BasicHttpProcessor();
    httpproc.addInterceptor(new ResponseDate());
    httpproc.addInterceptor(new ResponseServer());
    httpproc.addInterceptor(new ResponseContent());
    httpproc.addInterceptor(new ResponseConnControl());

    // Set up request handlers
    final HttpRequestHandlerRegistry reqistry = new HttpRequestHandlerRegistry();
    reqistry.register("*", requestHandler);

    // Set up the HTTP service
    _httpService = new HttpService(httpproc, new NoConnectionReuseStrategy(), new DefaultHttpResponseFactory());
    _httpService.setParams(_params);
    _httpService.setHandlerResolver(reqistry);
}
 

/**
 * Default constructor.
 * @param dao dao
 */
@Inject
public OXRCurrencyConverter(DAO dao) {
	this.dao = dao;
	int timeout = 30 * 1000;
	this.httpClient = HttpClientBuilder.create().
			setConnectionReuseStrategy(new NoConnectionReuseStrategy()).
			setDefaultRequestConfig(RequestConfig.custom().
					setConnectTimeout(timeout).
					setConnectionRequestTimeout(timeout).
					setSocketTimeout(timeout).
					build()).
			build();
}
 

/**
 * Default constructor.
 * @param defaultFilterProcessesUrl the url of the filter
 */
public MicrosoftAuthFilter(final String defaultFilterProcessesUrl) {
	super(defaultFilterProcessesUrl);
	this.jreader = ParaObjectUtils.getJsonReader(Map.class);
	int timeout = 30 * 1000;
	this.httpclient = HttpClientBuilder.create().
			setConnectionReuseStrategy(new NoConnectionReuseStrategy()).
			setDefaultRequestConfig(RequestConfig.custom().
					setConnectTimeout(timeout).
					setConnectionRequestTimeout(timeout).
					setCookieSpec(CookieSpecs.STANDARD).
					setSocketTimeout(timeout).
					build()).
			build();
}
 

/**
 * Default constructor.
 * @param defaultFilterProcessesUrl the url of the filter
 */
public FacebookAuthFilter(String defaultFilterProcessesUrl) {
	super(defaultFilterProcessesUrl);
	this.jreader = ParaObjectUtils.getJsonReader(Map.class);
	int timeout = 30 * 1000;
	this.httpclient = HttpClientBuilder.create().
			setConnectionReuseStrategy(new NoConnectionReuseStrategy()).
			setDefaultRequestConfig(RequestConfig.custom().
					setConnectTimeout(timeout).
					setConnectionRequestTimeout(timeout).
					setCookieSpec(CookieSpecs.STANDARD).
					setSocketTimeout(timeout).
					build()).
			build();
}
 

/**
 * Default constructor.
 * @param defaultFilterProcessesUrl the url of the filter
 */
public GenericOAuth2Filter(final String defaultFilterProcessesUrl) {
	super(defaultFilterProcessesUrl);
	this.jreader = ParaObjectUtils.getJsonReader(Map.class);
	int timeout = 30 * 1000;
	this.httpclient = HttpClientBuilder.create().
			setConnectionReuseStrategy(new NoConnectionReuseStrategy()).
			setDefaultRequestConfig(RequestConfig.custom().
					setConnectTimeout(timeout).
					setConnectionRequestTimeout(timeout).
					setCookieSpec(CookieSpecs.STANDARD).
					setSocketTimeout(timeout).
					build()).
			build();
}
 

/**
 * Default constructor.
 * @param defaultFilterProcessesUrl the url of the filter
 */
public SlackAuthFilter(final String defaultFilterProcessesUrl) {
	super(defaultFilterProcessesUrl);
	this.jreader = ParaObjectUtils.getJsonReader(Map.class);
	int timeout = 30 * 1000;
	this.httpclient = HttpClientBuilder.create().
			setConnectionReuseStrategy(new NoConnectionReuseStrategy()).
			setDefaultRequestConfig(RequestConfig.custom().
					setConnectTimeout(timeout).
					setConnectionRequestTimeout(timeout).
					setCookieSpec(CookieSpecs.STANDARD).
					setSocketTimeout(timeout).
					build()).
			build();
}
 

/**
 * Default constructor.
 * @param defaultFilterProcessesUrl the url of the filter
 */
public TwitterAuthFilter(final String defaultFilterProcessesUrl) {
	super(defaultFilterProcessesUrl);
	this.jreader = ParaObjectUtils.getJsonReader(Map.class);
	int timeout = 30 * 1000;
	this.httpclient = HttpClientBuilder.create().
			setConnectionReuseStrategy(new NoConnectionReuseStrategy()).
			setDefaultRequestConfig(RequestConfig.custom().
					setConnectTimeout(timeout).
					setConnectionRequestTimeout(timeout).
					setCookieSpec(CookieSpecs.STANDARD).
					setSocketTimeout(timeout).
					build()).
			build();
}
 

/**
 * Default constructor.
 * @param defaultFilterProcessesUrl the url of the filter
 */
public GoogleAuthFilter(final String defaultFilterProcessesUrl) {
	super(defaultFilterProcessesUrl);
	this.jreader = ParaObjectUtils.getJsonReader(Map.class);
	int timeout = 30 * 1000;
	this.httpclient = HttpClientBuilder.create().
			setConnectionReuseStrategy(new NoConnectionReuseStrategy()).
			setDefaultRequestConfig(RequestConfig.custom().
					setConnectTimeout(timeout).
					setConnectionRequestTimeout(timeout).
					setCookieSpec(CookieSpecs.STANDARD).
					setSocketTimeout(timeout).
					build()).
			build();
}
 

/**
 * Default constructor.
 * @param defaultFilterProcessesUrl the url of the filter
 */
public AmazonAuthFilter(final String defaultFilterProcessesUrl) {
	super(defaultFilterProcessesUrl);
	this.jreader = ParaObjectUtils.getJsonReader(Map.class);
	int timeout = 30 * 1000;
	this.httpclient = HttpClientBuilder.create().
			setConnectionReuseStrategy(new NoConnectionReuseStrategy()).
			setDefaultRequestConfig(RequestConfig.custom().
					setConnectTimeout(timeout).
					setConnectionRequestTimeout(timeout).
					setCookieSpec(CookieSpecs.STANDARD).
					setSocketTimeout(timeout).
					build()).
			build();
}
 

/**
 * Default constructor.
 * @param defaultFilterProcessesUrl the url of the filter
 */
public LinkedInAuthFilter(final String defaultFilterProcessesUrl) {
	super(defaultFilterProcessesUrl);
	this.jreader = ParaObjectUtils.getJsonReader(Map.class);
	int timeout = 30 * 1000;
	this.httpclient = HttpClientBuilder.create().
			setConnectionReuseStrategy(new NoConnectionReuseStrategy()).
			setDefaultRequestConfig(RequestConfig.custom().
					setConnectTimeout(timeout).
					setConnectionRequestTimeout(timeout).
					setCookieSpec(CookieSpecs.STANDARD).
					setSocketTimeout(timeout).
					build()).
			build();
}
 

/**
 * Get a default HttpClient based on the HttpConfiguration object. If required the defaults can 
 * be altered to meet the requirements of the SDK user. The default client does not use connection 
 * pooling and does not reuse connections. Timeouts for connection and socket are taken from the 
 * {@link HttpConfiguration} object.
 * 
 * @param httpConfiguration
 * @return CloseableHttpClient
 */
public static CloseableHttpClient getDefaultClient(HttpConfiguration httpConfiguration) {

    RequestConfig requestConfig = RequestConfig.custom().setConnectTimeout(httpConfiguration.getTimeout())
            .setSocketTimeout(httpConfiguration.getTimeout()).build();

    HttpClientConnectionManager connectionManager = new BasicHttpClientConnectionManager();
    ConnectionReuseStrategy connectionResuseStrategy = new NoConnectionReuseStrategy();

    logger.debug("Creating HttpClient with simple no pooling/no connection reuse default settings.");
    CloseableHttpClient httpClient = HttpClients.custom().setDefaultRequestConfig(requestConfig).setConnectionManager(connectionManager)
            .setConnectionReuseStrategy(connectionResuseStrategy).build();
    return httpClient;
}
 

public ListenerThread(final ApiServer requestHandler, final int port) {
    try {
        _serverSocket = new ServerSocket(port);
    } catch (final IOException ioex) {
        s_logger.error("error initializing api server", ioex);
        return;
    }

    _params = new BasicHttpParams();
    _params.setIntParameter(CoreConnectionPNames.SO_TIMEOUT, 30000)
           .setIntParameter(CoreConnectionPNames.SOCKET_BUFFER_SIZE, 8 * 1024)
           .setBooleanParameter(CoreConnectionPNames.STALE_CONNECTION_CHECK, false)
           .setBooleanParameter(CoreConnectionPNames.TCP_NODELAY, true)
           .setParameter(CoreProtocolPNames.ORIGIN_SERVER, "HttpComponents/1.1");

    // Set up the HTTP protocol processor
    final BasicHttpProcessor httpproc = new BasicHttpProcessor();
    httpproc.addInterceptor(new ResponseDate());
    httpproc.addInterceptor(new ResponseServer());
    httpproc.addInterceptor(new ResponseContent());
    httpproc.addInterceptor(new ResponseConnControl());

    // Set up request handlers
    final HttpRequestHandlerRegistry reqistry = new HttpRequestHandlerRegistry();
    reqistry.register("*", requestHandler);

    // Set up the HTTP service
    _httpService = new HttpService(httpproc, new NoConnectionReuseStrategy(), new DefaultHttpResponseFactory());
    _httpService.setParams(_params);
    _httpService.setHandlerResolver(reqistry);
}
 

public static CloseableHttpClient getHttpClient(final HttpHost proxy, final boolean redirects, final CookieStore cookieStore) {
	final Builder builder = RequestConfig.custom()
			.setRedirectsEnabled(redirects)
			.setMaxRedirects(5);
	if (proxy != null) builder.setProxy(proxy);
	final RequestConfig requestConfig = builder.build();
	return HttpClients.custom()
			.setDefaultRequestConfig(requestConfig)
			.setDefaultCookieStore(cookieStore)
			.setConnectionReuseStrategy(NoConnectionReuseStrategy.INSTANCE)
			.build();
}
 

/** Creates a new fetching thread.
 *
 * @param frontier a reference to the {@link Frontier}.
 * @param index  the index of this thread (only for logging purposes).
 */
public FetchingThread(final Frontier frontier, final int index) throws NoSuchAlgorithmException, IllegalArgumentException, IOException {
	setName(this.getClass().getSimpleName() + '-' + index);
	setPriority(Thread.MIN_PRIORITY); // Low priority; there will be thousands of this guys around.
	this.frontier = frontier;

	final BasicHttpClientConnectionManager connManager = new BasicHttpClientConnectionManagerWithAlternateDNS(frontier.rc.dnsResolver);
	connManager.closeIdleConnections(0, TimeUnit.MILLISECONDS);
	connManager.setConnectionConfig(ConnectionConfig.custom().setBufferSize(8 * 1024).build()); // TODO: make this configurable

	cookieStore = new BasicCookieStore();

	final BasicHeader[] headers = {
		new BasicHeader("From", frontier.rc.userAgentFrom),
		new BasicHeader("Accept","text/html,application/xhtml+xml,application/xml;q=0.95,text/*;q=0.9,*/*;q=0.8")
	};

	httpClient = HttpClients.custom()
			.setSSLContext(frontier.rc.acceptAllCertificates ? TRUST_ALL_CERTIFICATES_SSL_CONTEXT : TRUST_SELF_SIGNED_SSL_CONTEXT)
			.setConnectionManager(connManager)
			.setConnectionReuseStrategy(frontier.rc.keepAliveTime == 0 ? NoConnectionReuseStrategy.INSTANCE : DefaultConnectionReuseStrategy.INSTANCE)
			.setUserAgent(frontier.rc.userAgent)
			.setDefaultCookieStore(cookieStore)
			.setDefaultHeaders(ObjectArrayList.wrap(headers))
			.build();
  		fetchData = new FetchData(frontier.rc);
}
 

static CloseableHttpClient getHttpClient() {
	if (httpclient == null) {
		int timeout = 5 * 1000;
		httpclient = HttpClientBuilder.create().
				setConnectionReuseStrategy(new NoConnectionReuseStrategy()).
				setDefaultRequestConfig(RequestConfig.custom().
						setConnectTimeout(timeout).
						setConnectionRequestTimeout(timeout).
						setCookieSpec(CookieSpecs.STANDARD).
						setSocketTimeout(timeout).
						build()).
				build();
	}
	return httpclient;
}
 

@Bean
@ConditionalOnMissingBean(HttpClient.class)
public HttpClient httpClientOut() {
	RequestConfig requestConfig = RequestConfig.custom().setConnectTimeout(httpClientPropertiesOut.getConnectTimeout())
			.setSocketTimeout(httpClientPropertiesOut.getSocketTimeout()).build();
	
	HttpClient httpClient = HttpClientBuilder.create().setDefaultRequestConfig(requestConfig)
			.setUserAgent(httpClientPropertiesOut.getAgent()).setMaxConnPerRoute(httpClientPropertiesOut.getMaxPerRoute())
			.setConnectionReuseStrategy(new NoConnectionReuseStrategy()).build();
	return httpClient;
}
 

@VisibleForTesting
HTTPConnectionFactory(final Configurator config) {
    Registry<ConnectionSocketFactory> registry = null;
    try {
        final Configurator cfg = config == null ? ConfigUtil.getConfigInfo(HTTPConnectionFactory.class) : config;
        // if someone doesn't want keep alives...
        if (!cfg.findBooleanEntry(CFG_HTTP_KEEPALIVE, DFLT_KEEPALIVE)) {
            this.connReuseStrategy = NoConnectionReuseStrategy.INSTANCE;
        }
        this.maxConns = cfg.findIntEntry(CFG_HTTP_MAXCONNS, DFLT_MAXCONNS);
        this.userAgent = cfg.findStringEntry(CFG_HTTP_AGENT, DEFAULT_HTTP_AGENT);
        final SSLContext sslContext = build(cfg);
        // mainly for using in test environments where cert name may not match host name
        final HostnameVerifier v = cfg.findBooleanEntry(CFG_NOOP_VERIFIER, false) ? new NoopHostnameVerifier() : new DefaultHostnameVerifier();
        registry =
                RegistryBuilder.<ConnectionSocketFactory>create().register(HTTP, PlainConnectionSocketFactory.getSocketFactory())
                        .register(HTTPS, new SSLConnectionSocketFactory(sslContext, v)).build();
    } catch (IOException | GeneralSecurityException ex) {
        log.error("Error configuring HTTPConnectionFactory. The connection factory will use HTTP Client default settings", ex);
    }
    if (registry == null) {
        this.connMan = new PoolingHttpClientConnectionManager();
    } else {
        this.connMan = new PoolingHttpClientConnectionManager(registry);
    }

    this.connMan.setMaxTotal(this.maxConns);
}
 

protected ConnectionReuseStrategy createConnectionReuseStrategy() {
    return NoConnectionReuseStrategy.INSTANCE;
}
 

/**
 * @param proxy    Proxy configuration
 * @param listener Log listener
 * @param prompt   Prompt for proxy credentials
 * @return Builder for HTTP client
 */
public HttpClientBuilder build(final Proxy proxy, final TranscriptListener listener, final LoginCallback prompt) {
    final HttpClientBuilder configuration = HttpClients.custom();
    // Use HTTP Connect proxy implementation provided here instead of
    // relying on internal proxy support in socket factory
    switch(proxy.getType()) {
        case HTTP:
        case HTTPS:
            final HttpHost h = new HttpHost(proxy.getHostname(), proxy.getPort(), Scheme.http.name());
            if(log.isInfoEnabled()) {
                log.info(String.format("Setup proxy %s", h));
            }
            configuration.setProxy(h);
            configuration.setProxyAuthenticationStrategy(new CallbackProxyAuthenticationStrategy(ProxyCredentialsStoreFactory.get(), host, prompt));
            break;
    }
    configuration.setUserAgent(new PreferencesUseragentProvider().get());
    final int timeout = preferences.getInteger("connection.timeout.seconds") * 1000;
    configuration.setDefaultSocketConfig(SocketConfig.custom()
        .setTcpNoDelay(true)
        .setSoTimeout(timeout)
        .build());
    configuration.setDefaultRequestConfig(this.createRequestConfig(timeout));
    configuration.setDefaultConnectionConfig(ConnectionConfig.custom()
        .setBufferSize(preferences.getInteger("http.socket.buffer"))
        .setCharset(Charset.forName(host.getEncoding()))
        .build());
    if(preferences.getBoolean("http.connections.reuse")) {
        configuration.setConnectionReuseStrategy(new DefaultClientConnectionReuseStrategy());
    }
    else {
        configuration.setConnectionReuseStrategy(new NoConnectionReuseStrategy());
    }
    configuration.setRetryHandler(new ExtendedHttpRequestRetryHandler(preferences.getInteger("http.connections.retry")));
    configuration.setServiceUnavailableRetryStrategy(new DisabledServiceUnavailableRetryStrategy());
    if(!preferences.getBoolean("http.compression.enable")) {
        configuration.disableContentCompression();
    }
    configuration.setRequestExecutor(new LoggingHttpRequestExecutor(listener));
    // Always register HTTP for possible use with proxy. Contains a number of protocol properties such as the
    // default port and the socket factory to be used to create the java.net.Socket instances for the given protocol
    configuration.setConnectionManager(this.createConnectionManager(this.createRegistry()));
    configuration.setDefaultAuthSchemeRegistry(RegistryBuilder.<AuthSchemeProvider>create()
        .register(AuthSchemes.BASIC, new BasicSchemeFactory(
            Charset.forName(preferences.getProperty("http.credentials.charset"))))
        .register(AuthSchemes.DIGEST, new DigestSchemeFactory(
            Charset.forName(preferences.getProperty("http.credentials.charset"))))
        .register(AuthSchemes.NTLM, preferences.getBoolean("webdav.ntlm.windows.authentication.enable") && WinHttpClients.isWinAuthAvailable() ?
            new BackportWindowsNTLMSchemeFactory(null) :
            new NTLMSchemeFactory())
        .register(AuthSchemes.SPNEGO, preferences.getBoolean("webdav.ntlm.windows.authentication.enable") && WinHttpClients.isWinAuthAvailable() ?
            new BackportWindowsNegotiateSchemeFactory(null) :
            new SPNegoSchemeFactory())
        .register(AuthSchemes.KERBEROS, new KerberosSchemeFactory()).build());
    return configuration;
}