Java Code Examples for org.apache.http.impl.conn.PoolingHttpClientConnectionManager#setDefaultMaxPerRoute()

@Bean
public HttpClient httpClient() {
    Registry<ConnectionSocketFactory> registry = RegistryBuilder.<ConnectionSocketFactory>create()
            .register("http", PlainConnectionSocketFactory.getSocketFactory())
            .register("https", SSLConnectionSocketFactory.getSocketFactory())
            .build();
    PoolingHttpClientConnectionManager connectionManager = new PoolingHttpClientConnectionManager(registry);
    connectionManager.setMaxTotal(maxTotal);
    connectionManager.setDefaultMaxPerRoute(defaultMaxPerRoute);

    RequestConfig requestConfig = RequestConfig.custom()
            .setSocketTimeout(socketTimeout)
            .setConnectTimeout(connectTimeout)
            .setConnectionRequestTimeout(connectionRequestTimeout)
            .build();
    return HttpClientBuilder.create()
            .setDefaultRequestConfig(requestConfig)
            .setConnectionManager(connectionManager)
            .build();
}
 

private HttpUtilManager() {
	cm = new PoolingHttpClientConnectionManager();
	cm.setMaxTotal(500);
	cm.setDefaultMaxPerRoute(100);//例如默认每路由最高50并发，具体依据业务来定
	client = Init.httpClientBuilder
			.setConnectionManager(cm)
			.setKeepAliveStrategy(keepAliveStrat)
			.setMaxConnPerRoute(100)
			.setRetryHandler(new HttpRequestRetryHandler() {
				@Override
				public boolean retryRequest(IOException e, int i, HttpContext httpContext) {
					return false;  //不需要retry
				}
			})
			.setMaxConnTotal(100)
			.build();
}
 

@Inject
public HomeGraphAPI(GoogleConfig config,
      GoogleRpcContext rpcContext,
      ProductCatalogManager prodCat,
      GoogleWhitelist whitelist,
      @Named(EXECUTOR_NAME) HashedWheelTimer executor
) {
   this.config = config;
   requestConfig = RequestConfig.custom()
      .setConnectionRequestTimeout(config.getConnectionRequestTimeoutMs())
      .setConnectTimeout(config.getConnectionTimeoutMs())
      .setSocketTimeout(config.getSocketTimeoutMs())
      .build();

   pool = new PoolingHttpClientConnectionManager(config.getTimeToLiveMs(), TimeUnit.MILLISECONDS);
   pool.setDefaultMaxPerRoute(config.getRouteMaxConnections());
   pool.setMaxTotal(config.getMaxConnections());
   pool.setValidateAfterInactivity(config.getValidateAfterInactivityMs());
   this.gRpcContext = rpcContext;
   this.prodCat = prodCat;
   this.whitelist = whitelist;
   this.executor = executor;
}
 

public BmsHttpTransport(HttpRequestInterceptor requestInterceptor) {
    PoolingHttpClientConnectionManager connectionManager = new PoolingHttpClientConnectionManager();
    connectionManager.setMaxTotal(DEFAULT_MAX_CONNECTIONS);
    connectionManager.setDefaultMaxPerRoute(DEFAULT_MAX_PER_ROUTE_CONNECTIONS);

    RequestConfig requestConfig = RequestConfig.custom().
            setConnectTimeout(DEFAULT_CONNECTION_TIMEOUT).
            setConnectionRequestTimeout(DEFAULT_CONNECTION_TIMEOUT).
            setSocketTimeout(DEFAULT_READ_TIMEOUT).
            build();

    HttpClientBuilder httpClientBuilder = HttpClientBuilder.create().
            setConnectionManager(connectionManager).
            setDefaultRequestConfig(requestConfig).
            useSystemProperties();

    if (requestInterceptor != null) {
        httpClientBuilder.addInterceptorFirst(requestInterceptor);
    }

    this.httpClient = httpClientBuilder.build();
}
 

private CloseableHttpClient createHttpClient(String hostname, int port) {
    ConnectionSocketFactory plainsf = PlainConnectionSocketFactory.getSocketFactory();
    LayeredConnectionSocketFactory sslsf = SSLConnectionSocketFactory.getSocketFactory();
    Registry<ConnectionSocketFactory> registry = RegistryBuilder.<ConnectionSocketFactory> create()
            .register("http", plainsf).register("https", sslsf).build();
    PoolingHttpClientConnectionManager cm = new PoolingHttpClientConnectionManager(registry);
    // 将最大连接数增加
    cm.setMaxTotal(maxTotal);
    // 将每个路由基础的连接增加
    cm.setDefaultMaxPerRoute(maxPerRoute);
    HttpHost httpHost = new HttpHost(hostname, port);
    // 将目标主机的最大连接数增加
    cm.setMaxPerRoute(new HttpRoute(httpHost), maxRoute);
    // 请求重试处理
    return HttpClients.custom().setConnectionManager(cm).setRetryHandler(httpRequestRetryHandler).build();
}
 

public static CloseableHttpClient createHttpClient(final int maxRedirects) throws KeyManagementException, NoSuchAlgorithmException, KeyStoreException {
    s_logger.info("Creating new HTTP connection pool and client");
    final Registry<ConnectionSocketFactory> socketFactoryRegistry = createSocketFactoryConfigration();
    final BasicCookieStore cookieStore = new BasicCookieStore();
    final PoolingHttpClientConnectionManager connManager = new PoolingHttpClientConnectionManager(socketFactoryRegistry);
    connManager.setDefaultMaxPerRoute(MAX_ALLOCATED_CONNECTIONS_PER_ROUTE);
    connManager.setMaxTotal(MAX_ALLOCATED_CONNECTIONS);
    final RequestConfig requestConfig = RequestConfig.custom()
                                                     .setCookieSpec(CookieSpecs.DEFAULT)
                                                     .setMaxRedirects(maxRedirects)
                                                     .setSocketTimeout(DEFAULT_SOCKET_TIMEOUT)
                                                     .setConnectionRequestTimeout(DEFAULT_CONNECTION_REQUEST_TIMEOUT)
                                                     .setConnectTimeout(DEFAULT_CONNECT_TIMEOUT)
                                                     .build();
    return HttpClientBuilder.create()
                            .setConnectionManager(connManager)
                            .setRedirectStrategy(new LaxRedirectStrategy())
                            .setDefaultRequestConfig(requestConfig)
                            .setDefaultCookieStore(cookieStore)
                            .setRetryHandler(new StandardHttpRequestRetryHandler())
                            .build();
}
 

public TomHttpClientGenerator() {
    Registry<ConnectionSocketFactory> reg = RegistryBuilder.<ConnectionSocketFactory>create()
            .register("http", PlainConnectionSocketFactory.INSTANCE)
            .register("https", buildSSLConnectionSocketFactory())
            .build();
    connectionManager = new PoolingHttpClientConnectionManager(reg);
    connectionManager.setDefaultMaxPerRoute(100);
}
 

/**
 * Returns a {@link HttpClientConnectionManager}.
 * <p>
 * <b>NOTE:</b> do not {@link HttpClientConnectionManager#shutdown()} this
 * connection manager, it will be close automatically after all tests have
 * finished.
 */
public static synchronized HttpClientConnectionManager getClientConnectionManager() {
  if (clientConnectionManager == null) {
    PoolingHttpClientConnectionManager ccm = new PoolingHttpClientConnectionManager();
    ccm.setDefaultMaxPerRoute(128);
    ccm.setMaxTotal(128);
    clientConnectionManager = ccm;
  }
  
  return clientConnectionManager;
}
 

/**
 * 创建 HttpClient 连接池.
 */
private PoolingHttpClientConnectionManager createHttpClientConnPool(ConnectionConfig connectionConfig) {
    PoolingHttpClientConnectionManager httpClientConnectionManager = new PoolingHttpClientConnectionManager(
            RegistryBuilder.<ConnectionSocketFactory>create().
                    register("http", PlainConnectionSocketFactory.getSocketFactory()).
                    register("https", buildSSLConn()).build());
    httpClientConnectionManager.setMaxTotal(MAX_TOTAL); // 设置连接池线程最大数量
    httpClientConnectionManager.setDefaultMaxPerRoute(MAX_ROUTE_TOTAL); // 设置单个路由最大的连接线程数量
    httpClientConnectionManager.setDefaultConnectionConfig(connectionConfig);
    return httpClientConnectionManager;
}
 

private static CloseableHttpClient client() {
    PoolingHttpClientConnectionManager cm = new PoolingHttpClientConnectionManager();
    cm.setDefaultMaxPerRoute(100);
    cm.setMaxTotal(400);

    MessageConstraints messageConstraints = MessageConstraints.custom()
            .setMaxHeaderCount(200)
            .setMaxLineLength(2000)
            .build();

    ConnectionConfig connectionConfig = ConnectionConfig.custom()
            .setMalformedInputAction(CodingErrorAction.IGNORE)
            .setUnmappableInputAction(CodingErrorAction.IGNORE)
            .setCharset(Consts.UTF_8)
            .setMessageConstraints(messageConstraints)
            .build();

    RequestConfig defaultRequestConfig = RequestConfig.custom()
            .setSocketTimeout(5000)
            .setConnectTimeout(5000)
            .setConnectionRequestTimeout(5000)
            .build();

    cm.setDefaultConnectionConfig(connectionConfig);

    return HttpClients.custom()
            .setConnectionManager(cm)
            .setDefaultRequestConfig(defaultRequestConfig)
            .build();
}
 

@Override
public CloseableHttpClient buildPooledClient() {
    final HttpClientBuilder httpBuilder = HttpClients.custom();
    final PoolingHttpClientConnectionManager manager = new PoolingHttpClientConnectionManager(registry, dnsResolver);
    manager.setDefaultMaxPerRoute(maxConnections);
    httpBuilder.setConnectionManager(manager);
    return httpBuilder.build();
}
 

public  HttpClientConnectionManager createHttpClientConnectionManager() {
	SSLContext sslContext = null;
	try {
		sslContext = SSLContexts.custom().loadTrustMaterial(null, new TrustStrategy() {
			@Override
			public boolean isTrusted(X509Certificate[] chain, String authType) throws CertificateException {
				return false;
			}
		}).build();
	} catch (Exception e) {
		throw new RuntimeException(e);
	}
	SSLConnectionSocketFactory sslSocketFactory = new SSLConnectionSocketFactory(sslContext,
			NoopHostnameVerifier.INSTANCE);
	Registry<ConnectionSocketFactory> socketFactoryRegistry = RegistryBuilder.<ConnectionSocketFactory>create()
			.register("http", PlainConnectionSocketFactory.getSocketFactory()).register("https", sslSocketFactory)
			.build();
	PoolingHttpClientConnectionManager poolingHttpClientConnectionManager = new PoolingHttpClientConnectionManager(
			socketFactoryRegistry);
	// 最大连接数
	poolingHttpClientConnectionManager.setMaxTotal(httpClientConfig.getMaxTotal());
	// 单个站点最大连接数
	poolingHttpClientConnectionManager.setDefaultMaxPerRoute(httpClientConfig.getMaxPerRoute());
	// 长连接
	poolingHttpClientConnectionManager.setDefaultSocketConfig(
			SocketConfig.custom().setSoTimeout(httpClientConfig.getSocketTimeout()).setSoKeepAlive(true).build());
	// 连接不活跃多久检查毫秒 并不是100 % 可信
	poolingHttpClientConnectionManager.setValidateAfterInactivity(httpClientConfig.getValidateAfterInactivity());
	// 空闲扫描线程
	HttpClientIdleConnectionMonitor.registerConnectionManager(poolingHttpClientConnectionManager, httpClientConfig);
	return poolingHttpClientConnectionManager;
}
 

public ApacheHttpClient4Engine createEngine(int maxTotal, int defaultMaxPerRoute, String cookieSpec, boolean followRedirects) {
    PoolingHttpClientConnectionManager cm = new PoolingHttpClientConnectionManager();
    CloseableHttpClient httpClient = HttpClients.custom()
			.setDefaultRequestConfig(RequestConfig.custom().setCookieSpec(cookieSpec).build())
    		.setConnectionManager(cm).build();
    cm.setMaxTotal(maxTotal);
    cm.setDefaultMaxPerRoute(defaultMaxPerRoute);
       final ApacheHttpClient4Engine engine = new ApacheHttpClient4Engine(httpClient);
       engine.setFollowRedirects(followRedirects);
       return engine;
}
 

public PoolingHttpClientConnectionManager createConnectionManager(final Registry<ConnectionSocketFactory> registry) {
    if(log.isDebugEnabled()) {
        log.debug(String.format("Setup connection pool with registry %s", registry));
    }
    final PoolingHttpClientConnectionManager manager = new PoolingHttpClientConnectionManager(registry);
    manager.setMaxTotal(preferences.getInteger("http.connections.total"));
    manager.setDefaultMaxPerRoute(preferences.getInteger("http.connections.route"));
    // Detect connections that have become stale (half-closed) while kept inactive in the pool
    manager.setValidateAfterInactivity(preferences.getInteger("http.connections.stale.check.ms"));
    return manager;
}
 

/**
 * 当使用了请求连接池管理器（比如PoolingClientConnectionManager）后，HttpClient就可以同时执行多个线程的请求了。
 * @return
 */
public CloseableHttpClient getPoolHttpClient() {
	// 单线程跑是基本看不出配置了连接池的好处的，只有使用多线程爬取数据的时候，并且数据量越大效果越明显
	PoolingHttpClientConnectionManager conMgr = new PoolingHttpClientConnectionManager();
	conMgr.setMaxTotal(threadNumber); // 设置整个连接池最大连接数 根据自己的场景决定
	// 是路由的默认最大连接（该值默认为2），限制数量实际使用DefaultMaxPerRoute并非MaxTotal。
	// 设置过小无法支持大并发(ConnectionPoolTimeoutException: Timeout waiting for connection from pool)，路由是对maxTotal的细分。

	conMgr.setDefaultMaxPerRoute(threadNumber);// （目前只有一个路由，因此让他等于最大值）
	CloseableHttpClient httpClient = HttpClients.custom().setConnectionManager(conMgr).build();
	return httpClient;
}
 

/**
 * 描述：创建httpClient连接池，并初始化httpclient
 */
private void initHttpClient() throws ConfigurationException {
    Configuration configuration = new PropertiesConfiguration(CONFIG_FILE);
    //创建httpclient连接池
    PoolingHttpClientConnectionManager httpClientConnectionManager = new PoolingHttpClientConnectionManager();
    httpClientConnectionManager.setMaxTotal(configuration.getInt("http.max.total"));    //设置连接池线程最大数量
    httpClientConnectionManager.setDefaultMaxPerRoute(configuration.getInt("http.max.route"));    //设置单个路由最大的连接线程数量
    //创建http request的配置信息
    RequestConfig requestConfig = RequestConfig.custom()
            .setConnectionRequestTimeout(configuration.getInt("http.request.timeout"))
            .setSocketTimeout(configuration.getInt("http.socket.timeout"))
            .setCookieSpec(CookieSpecs.DEFAULT).build();
    //设置重定向策略
    LaxRedirectStrategy redirectStrategy = new LaxRedirectStrategy() {
        /**
         * false 禁止重定向  true 允许
         */
        @Override
        public boolean isRedirected(HttpRequest request,
                                    HttpResponse response, HttpContext context)
                throws ProtocolException {
            // TODO Auto-generated method stub
            return isRediect ? super.isRedirected(request, response, context) : isRediect;
        }
    };
    //初始化httpclient客户端
    httpClient = HttpClients.custom().setConnectionManager(httpClientConnectionManager)
            .setDefaultRequestConfig(requestConfig)
            //.setUserAgent(NewsConstant.USER_AGENT)
            .setRedirectStrategy(redirectStrategy)
            .build();
}
 

public static CloseableHttpClient getHttpClient() {
	PoolingHttpClientConnectionManager cm = new PoolingHttpClientConnectionManager();
	cm.setMaxTotal(MAX_TOTAL);
	cm.setDefaultMaxPerRoute(MAX_PERROUTE);
	CloseableHttpClient httpClient = HttpClients
			.custom()
			.setConnectionManager(cm)
			.build();
	return httpClient;
}
 

private static HttpClientConnectionManager getPooledConnectionFactory(String certPath, int maxConnections) throws IOException {
    PoolingHttpClientConnectionManager ret =  certPath != null ?
            new PoolingHttpClientConnectionManager(getSslFactoryRegistry(certPath)) :
            new PoolingHttpClientConnectionManager();
    ret.setDefaultMaxPerRoute(maxConnections);
    ret.setMaxTotal(maxConnections);
    return ret;
}
 

public TestCaseAServlet() {
    String backendMockHost = System.getProperty("perf.test.backend.hostname", "127.0.0.1");
    String backendMockPort = System.getProperty("perf.test.backend.port", "8989");
    backendMockUriPrefix = "http://" + backendMockHost + ':' + backendMockPort + "/ws-backend-mock";

    final RequestConfig reqConfig = RequestConfig.custom()
        .setConnectTimeout(PropertyNames.ClientConnectTimeout.getValueAsInt())
        .setSocketTimeout(PropertyNames.ClientSocketTimeout.getValueAsInt())
        .setConnectionRequestTimeout(PropertyNames.ClientConnectionRequestTimeout.getValueAsInt())
        .build();

    // don't care about total vs. per-route right now, will set them to the same
    final PoolingHttpClientConnectionManager connMgr = new PoolingHttpClientConnectionManager();
    connMgr.setMaxTotal(PropertyNames.ClientMaxConnectionsTotal.getValueAsInt());
    connMgr.setDefaultMaxPerRoute(PropertyNames.ClientMaxConnectionsTotal.getValueAsInt());

    client = HttpClients.custom()
        .setDefaultRequestConfig(reqConfig)
        .setConnectionManager(connMgr)
        .build();

    // used for parallel execution
    final int backendRequestThreadPoolSize = PropertyNames.BackendRequestThreadPoolSize.getValueAsInt();

    // setting core and max pool sizes the same since I do not want any queueing in here
    executor = new ThreadPoolExecutor(backendRequestThreadPoolSize,
        backendRequestThreadPoolSize,
        5,
        TimeUnit.MINUTES,
        new LinkedBlockingQueue<Runnable>());
}
 

public static CloseableHttpClient createHttpClient(int concurrency) {
	HttpClientBuilder builder = HttpClientBuilder.create();

	PoolingHttpClientConnectionManager connManager = new PoolingHttpClientConnectionManager();
	connManager.setDefaultMaxPerRoute(concurrency);
	connManager.setMaxTotal(concurrency);

	RequestConfig requestConfig = RequestConfig.custom()//
			.setAuthenticationEnabled(true)//
			.setSocketTimeout(SOCKET_TIMEOUT)//
			.setConnectionRequestTimeout(CONNECTION_REQUEST_TIMEOUT)//
			.setConnectTimeout(CONNECT_TIMEOUT)//
			.setRedirectsEnabled(true)//
			.setRelativeRedirectsAllowed(true)//
			.setMaxRedirects(15)//
			.build();

	SocketConfig socketConfig = SocketConfig.custom()//
			.setSoKeepAlive(true)//
			.setSoReuseAddress(true)//
			.build();

	CookieSpecProvider cookieSpecProvider = new IgnoreSpecProvider();
	Lookup<CookieSpecProvider> cookieSpecRegistry = RegistryBuilder.<CookieSpecProvider>create()//
			.register(CookieSpecs.DEFAULT, cookieSpecProvider)//
			.register(CookieSpecs.STANDARD, cookieSpecProvider)//
			.register(CookieSpecs.STANDARD_STRICT, cookieSpecProvider)//
			.build();

	builder.setConnectionManager(connManager);
	builder.setDefaultSocketConfig(socketConfig);
	builder.setDefaultRequestConfig(requestConfig);
	builder.setDefaultCookieSpecRegistry(cookieSpecRegistry);

	return builder.addInterceptorLast((HttpRequest request, HttpContext context) -> {
		request.removeHeaders("Host");
		request.removeHeaders("Accept-Encoding");
		//request.removeHeaders("Connection");
		request.removeHeaders("User-Agent");
	}).build();
}