Java Code Examples for org.apache.http.impl.conn.PoolingHttpClientConnectionManager#setMaxTotal()

public HttpTransportClient(int retryAttemptsNetworkErrorCount, int retryAttemptsInvalidStatusCount) {
    this.retryAttemptsNetworkErrorCount = retryAttemptsNetworkErrorCount;
    this.retryAttemptsInvalidStatusCount = retryAttemptsInvalidStatusCount;

    CookieStore cookieStore = new BasicCookieStore();
    RequestConfig requestConfig = RequestConfig.custom()
            .setSocketTimeout(SOCKET_TIMEOUT_MS)
            .setConnectTimeout(CONNECTION_TIMEOUT_MS)
            .setConnectionRequestTimeout(CONNECTION_TIMEOUT_MS)
            .setCookieSpec(CookieSpecs.STANDARD)
            .build();

    PoolingHttpClientConnectionManager connectionManager = new PoolingHttpClientConnectionManager();

    connectionManager.setMaxTotal(MAX_SIMULTANEOUS_CONNECTIONS);
    connectionManager.setDefaultMaxPerRoute(MAX_SIMULTANEOUS_CONNECTIONS);

    httpClient = HttpClients.custom()
            .setConnectionManager(connectionManager)
            .setDefaultRequestConfig(requestConfig)
            .setDefaultCookieStore(cookieStore)
            .setUserAgent(USER_AGENT)
            .build();
}
 

private HttpUtilManager() {
	cm = new PoolingHttpClientConnectionManager();
	cm.setMaxTotal(500);
	cm.setDefaultMaxPerRoute(100);//例如默认每路由最高50并发，具体依据业务来定
	client = Init.httpClientBuilder
			.setConnectionManager(cm)
			.setKeepAliveStrategy(keepAliveStrat)
			.setMaxConnPerRoute(100)
			.setRetryHandler(new HttpRequestRetryHandler() {
				@Override
				public boolean retryRequest(IOException e, int i, HttpContext httpContext) {
					return false;  //不需要retry
				}
			})
			.setMaxConnTotal(100)
			.build();
}
 

/**
 * Constructor
 */
public KafkaAuditCountHttpClient (State state) {
  int maxTotal = state.getPropAsInt(CONNECTION_MAX_TOTAL, DEFAULT_CONNECTION_MAX_TOTAL);
  int maxPerRoute = state.getPropAsInt(MAX_PER_ROUTE, DEFAULT_MAX_PER_ROUTE);

  cm = new PoolingHttpClientConnectionManager();
  cm.setMaxTotal(maxTotal);
  cm.setDefaultMaxPerRoute(maxPerRoute);
  httpClient = HttpClients.custom()
          .setConnectionManager(cm)
          .build();

  this.baseUrl = state.getProp(KAFKA_AUDIT_REST_BASE_URL);
  this.maxNumTries = state.getPropAsInt(KAFKA_AUDIT_REST_MAX_TRIES, 5);
  this.startQueryString = state.getProp(KAFKA_AUDIT_REST_START_QUERYSTRING_KEY, KAFKA_AUDIT_REST_START_QUERYSTRING_DEFAULT);
  this.endQueryString = state.getProp(KAFKA_AUDIT_REST_END_QUERYSTRING_KEY, KAFKA_AUDIT_REST_END_QUERYSTRING_DEFAULT);
}
 

private static CloseableHttpClient createClient(SSLConnectionSocketFactory socketFactory) {
    Registry<ConnectionSocketFactory> socketFactoryRegistry = RegistryBuilder.<ConnectionSocketFactory>create()
            .register("http", PlainConnectionSocketFactory.getSocketFactory())
            .register("https", socketFactory)
            .build();

    PoolingHttpClientConnectionManager cm = new PoolingHttpClientConnectionManager(socketFactoryRegistry);
    cm.setMaxTotal(200); // Increase max total connections to 200, which should be enough

    // Have experienced hang in socket read, which may have been because of
    // system defaults, therefore set explicit timeouts.
    return HttpClientBuilder.create()
            .setDefaultRequestConfig(DEFAULT_REQUEST_CONFIG)
            .disableAutomaticRetries()
            .setUserAgent("node-admin")
            .setConnectionManager(cm)
            .build();
}
 

@Override
public void configUpdated(String configKey, String configValue) {
	if (configValue != null && configValue.length() > 0)
	{
		ObjectMapper mapper = new ObjectMapper();
           try {
           	PredictionServerConfig config = mapper.readValue(configValue, PredictionServerConfig.class);
           	cm = new PoolingHttpClientConnectionManager();
               cm.setMaxTotal(config.maxConnections);
               cm.setDefaultMaxPerRoute(config.maxConnections);
               httpClient = HttpClients.custom()
                       .setConnectionManager(cm)
                       .build();
               logger.info("Updated httpclient to use "+config.maxConnections+" max connections");
           } catch (Exception e) {
               throw new RuntimeException(String.format("* Error * parsing statsd configValue[%s]", configValue),e);
           }
	}
	
}
 

public BmsHttpTransport(HttpRequestInterceptor requestInterceptor) {
    PoolingHttpClientConnectionManager connectionManager = new PoolingHttpClientConnectionManager();
    connectionManager.setMaxTotal(DEFAULT_MAX_CONNECTIONS);
    connectionManager.setDefaultMaxPerRoute(DEFAULT_MAX_PER_ROUTE_CONNECTIONS);

    RequestConfig requestConfig = RequestConfig.custom().
            setConnectTimeout(DEFAULT_CONNECTION_TIMEOUT).
            setConnectionRequestTimeout(DEFAULT_CONNECTION_TIMEOUT).
            setSocketTimeout(DEFAULT_READ_TIMEOUT).
            build();

    HttpClientBuilder httpClientBuilder = HttpClientBuilder.create().
            setConnectionManager(connectionManager).
            setDefaultRequestConfig(requestConfig).
            useSystemProperties();

    if (requestInterceptor != null) {
        httpClientBuilder.addInterceptorFirst(requestInterceptor);
    }

    this.httpClient = httpClientBuilder.build();
}
 

/**
 * Ctor.
 * @param socketFile Unix socket on disk.
 */
UnixHttpClient(final File socketFile) {
    this(() -> {
        final PoolingHttpClientConnectionManager pool =
            new PoolingHttpClientConnectionManager(
                RegistryBuilder
                    .<ConnectionSocketFactory>create()
                    .register("unix", new UnixSocketFactory(socketFile))
                    .build()
            );
        pool.setDefaultMaxPerRoute(10);
        pool.setMaxTotal(10);
        return HttpClientBuilder.create()
            .setConnectionManager(pool)
            .addInterceptorFirst(new UserAgentRequestHeader())
            .build();
    });
}
 

public HttpClientConnectionManager getHttpConnectionManager() {
    PoolingHttpClientConnectionManager cm = new PoolingHttpClientConnectionManager();
    // TODO: Introduce configurable variable for Max total and max per router variables.
    cm.setMaxTotal(MAX_TOTAL_CONNECTIONS);
    cm.setDefaultMaxPerRoute(DEFAULT_MAX_PER_ROUTE);
    //HttpHost localhost = new HttpHost("localhost", 80);
    //cm.setMaxPerRoute(new HttpRoute(localhost), 50);
    return cm;
}
 

public B fromConfig(Config config) {
  config = config.withFallback(FALLBACK);
  RequestConfig requestConfig = RequestConfig.copy(RequestConfig.DEFAULT)
                                             .setSocketTimeout(config.getInt(REQUEST_TIME_OUT_MS_KEY))
                                             .setConnectTimeout(config.getInt(CONNECTION_TIME_OUT_MS_KEY))
                                             .setConnectionRequestTimeout(config.getInt(CONNECTION_TIME_OUT_MS_KEY))
                                             .build();

  getHttpClientBuilder().setDefaultRequestConfig(requestConfig);

  if (config.hasPath(STATIC_SVC_ENDPOINT)) {
    try {
      svcEndpoint = Optional.of(new URI(config.getString(AbstractHttpWriterBuilder.STATIC_SVC_ENDPOINT)));
    } catch (URISyntaxException e) {
      throw new RuntimeException(e);
    }
  }

  String connMgrStr = config.getString(HTTP_CONN_MANAGER);
  switch (ConnManager.valueOf(connMgrStr.toUpperCase())) {
    case BASIC:
      httpConnManager = new BasicHttpClientConnectionManager();
      break;
    case POOLING:
      PoolingHttpClientConnectionManager poolingConnMgr = new PoolingHttpClientConnectionManager();
      poolingConnMgr.setMaxTotal(config.getInt(POOLING_CONN_MANAGER_MAX_TOTAL_CONN));
      poolingConnMgr.setDefaultMaxPerRoute(config.getInt(POOLING_CONN_MANAGER_MAX_PER_CONN));
      httpConnManager = poolingConnMgr;
      break;
    default:
      throw new IllegalArgumentException(connMgrStr + " is not supported");
  }
  LOG.info("Using " + httpConnManager.getClass().getSimpleName());
  return typedSelf();
}
 

protected PoolingHttpClientConnectionManager createConnectionPooling(SSLContext sslContext) {
    if (sslContext == null) {
        return null;
    }
    SSLConnectionSocketFactory sslsf = new SSLConnectionSocketFactory(sslContext);
    Registry<ConnectionSocketFactory> registry = RegistryBuilder.<ConnectionSocketFactory>create().register("https", sslsf).build();
    PoolingHttpClientConnectionManager poolingHttpClientConnectionManager = new PoolingHttpClientConnectionManager(registry);

    //route is host + port.  Since we have only one, set the max and the route the same
    poolingHttpClientConnectionManager.setDefaultMaxPerRoute(maxPoolPerRoute);
    poolingHttpClientConnectionManager.setMaxTotal(maxPoolTotal);
    return poolingHttpClientConnectionManager;
}
 

private HttpClientConnectionManager newConnectionManager() {
  Registry<ConnectionSocketFactory> r = RegistryBuilder.<ConnectionSocketFactory>create()
    .register("http", PlainConnectionSocketFactory.getSocketFactory())
    .register("https", sslsf)
    .build();
  PoolingHttpClientConnectionManager connManager = new PoolingHttpClientConnectionManager(r);
  connManager.setMaxTotal(100);
  connManager.setDefaultMaxPerRoute(100);
  return connManager;
}
 

@Test
// Example 4.1
public final void whenIncreasingConnectionPool_thenNoEceptions() {
    poolingConnManager = new PoolingHttpClientConnectionManager();
    poolingConnManager.setMaxTotal(5);
    poolingConnManager.setDefaultMaxPerRoute(4);

    final HttpHost localhost = new HttpHost("locahost", 80);
    poolingConnManager.setMaxPerRoute(new HttpRoute(localhost), 5);
}
 

private Singleton() {
  PoolingHttpClientConnectionManager cm = new PoolingHttpClientConnectionManager();
  // Increase max total connection to 200
  cm.setMaxTotal(200);
  // Increase default max connection per route to 20
  cm.setDefaultMaxPerRoute(20);
  // Build the client.
  threadSafeClient = HttpClients.custom().setConnectionManager(cm).build();
  // Start up an eviction thread.
  monitor = new IdleConnectionMonitorThread(cm);
  // Don't stop quitting.
  monitor.setDaemon(true);
  monitor.start();
}
 

@Override
public void configUpdated(String configKey, String configValue) {
	if (configValue != null && configValue.length() > 0)
	{
		ObjectMapper mapper = new ObjectMapper();
           try {
           	PredictionServerConfig config = mapper.readValue(configValue, PredictionServerConfig.class);
           	cm = new PoolingHttpClientConnectionManager();
               cm.setMaxTotal(config.maxConnections);
               cm.setDefaultMaxPerRoute(config.maxConnections);
               
               RequestConfig requestConfig = RequestConfig.custom()
                       .setConnectionRequestTimeout(DEFAULT_REQ_TIMEOUT)
                       .setConnectTimeout(DEFAULT_CON_TIMEOUT)
                       .setSocketTimeout(DEFAULT_SOCKET_TIMEOUT).build();
               

               httpClient = HttpClients.custom()
                       .setConnectionManager(cm)
                       .setDefaultRequestConfig(requestConfig)
                       .build();
               logger.info("Updated httpclient to use "+config.maxConnections+" max connections");
           } catch (Exception e) {
               throw new RuntimeException(String.format("* Error * parsing statsd configValue[%s]", configValue),e);
           }
	}
	
}
 

/**
 * Create a http client connection manager based on given ssl context
 * @param sslContext ssl context containing keystore with client key/cert
 * @return connection manager object
 */
PoolingHttpClientConnectionManager createConnectionPooling(SSLContext sslContext) {

    SSLConnectionSocketFactory sslsf = new SSLConnectionSocketFactory(sslContext);
    Registry<ConnectionSocketFactory> registry = RegistryBuilder.<ConnectionSocketFactory>create().register("https", sslsf).build();
    PoolingHttpClientConnectionManager poolingHttpClientConnectionManager = new PoolingHttpClientConnectionManager(registry);
    
    //route is host + port.  Since we have only one, set the max and the route the same

    poolingHttpClientConnectionManager.setDefaultMaxPerRoute(DEFAULT_MAX_POOL_PER_ROUTE);
    poolingHttpClientConnectionManager.setMaxTotal(DEFAULT_MAX_POOL_TOTAL);
    return poolingHttpClientConnectionManager;
}
 

private CloseableHttpClient getHttpClient() {
    if (this.httpClient != null) {
        return this.httpClient;
    }
    PoolingHttpClientConnectionManager connManager = new PoolingHttpClientConnectionManager();
    connManager.setDefaultMaxPerRoute(6);
    connManager.setMaxTotal(20);
    connManager.closeIdleConnections(120, TimeUnit.SECONDS);
    HttpClientBuilder httpClientBuilder = HttpClientBuilder.create();
    //SOFATracer
    SofaTracerHttpClientBuilder.clientBuilder(httpClientBuilder);
    httpClient = httpClientBuilder.setConnectionManager(connManager).disableAutomaticRetries()
        .setUserAgent("CLIENT_VERSION").build();
    return httpClient;
}
 

protected HttpClient() {
    proxies = new ArrayList<HttpHost>();

    HttpClientBuilder builder = HttpClientBuilder.create();

    SSLContext sslContext;
    try {
        sslContext = new SSLContextBuilder().loadTrustMaterial(null, new TrustStrategy() {
            public boolean isTrusted(X509Certificate[] chain, String authType) throws CertificateException {
                return true;
            }
        }).build();
    } catch (Exception e) {
        e.printStackTrace(System.err);
        System.exit(0);
        return;
    }

    builder.setSSLContext(sslContext);
    HostnameVerifier hostnameVerifier = NoopHostnameVerifier.INSTANCE;
    SSLConnectionSocketFactory sslSocketFactory = new SSLConnectionSocketFactory(sslContext, hostnameVerifier);
    Registry<ConnectionSocketFactory> socketFactoryRegistry = RegistryBuilder.<ConnectionSocketFactory>create()
            .register("http", PlainConnectionSocketFactory.getSocketFactory())
            .register("https", sslSocketFactory)
            .build();

    PoolingHttpClientConnectionManager connMgr = new PoolingHttpClientConnectionManager(socketFactoryRegistry);
    connMgr.setMaxTotal(500);
    connMgr.setDefaultMaxPerRoute(connMgr.getMaxTotal());
    builder.setConnectionManager(connMgr);

    config = RequestConfig.custom()
            .setCookieSpec(CookieSpecs.DEFAULT)
            .build();
    builder.setDefaultRequestConfig(config);

    cookieStore = new BasicCookieStore();
    builder.setDefaultCookieStore(cookieStore);
    client = builder.build();

    connectTimeout = DEFAULT_CONNECTION_TIMEOUT;
    soTimeout = DEFAULT_SO_TIMEOUT;
}
 

/**
 * ClientHttpRequestFactory接口的另一种实现方式（推荐使用），即：
 * HttpComponentsClientHttpRequestFactory：底层使用Httpclient连接池的方式创建Http连接请求
 * @return HttpComponentsClientHttpRequestFactory
 */
@Bean
public HttpComponentsClientHttpRequestFactory httpComponentsClientHttpRequestFactory(){
	//Httpclient连接池，长连接保持30秒
	PoolingHttpClientConnectionManager connectionManager = new PoolingHttpClientConnectionManager(30, TimeUnit.SECONDS);
	
	//设置总连接数
	connectionManager.setMaxTotal(1000);
	//设置同路由的并发数
	connectionManager.setDefaultMaxPerRoute(1000);
	
	//设置header
	List<Header> headers = new ArrayList<Header>();
	headers.add(new BasicHeader("User-Agent", "Mozilla/5.0 (Windows NT 6.3; rv:36.0) Gecko/20100101 Firefox/36.04"));
	headers.add(new BasicHeader("Accept-Encoding", "gzip, deflate"));
	headers.add(new BasicHeader("Accept-Language", "zh-CN,zh;q=0.8,en-US;q=0.5,en;q=0.3"));
	headers.add(new BasicHeader("Connection", "keep-alive"));
	
	//创建HttpClient
       HttpClient httpClient = HttpClientBuilder.create()
               .setConnectionManager(connectionManager)
               .setDefaultHeaders(headers)
               .setRetryHandler(new DefaultHttpRequestRetryHandler(3, true)) //设置重试次数
               .setKeepAliveStrategy(new DefaultConnectionKeepAliveStrategy()) //设置保持长连接
               .build();
	
       //创建HttpComponentsClientHttpRequestFactory实例
       HttpComponentsClientHttpRequestFactory requestFactory = 
			new HttpComponentsClientHttpRequestFactory(httpClient);
	
       //设置客户端和服务端建立连接的超时时间
       requestFactory.setConnectTimeout(5000);
	//设置客户端从服务端读取数据的超时时间
       requestFactory.setReadTimeout(5000);
       //设置从连接池获取连接的超时时间，不宜过长
       requestFactory.setConnectionRequestTimeout(200);
       //缓冲请求数据，默认为true。通过POST或者PUT大量发送数据时，建议将此更改为false，以免耗尽内存
       requestFactory.setBufferRequestBody(false);
       
       return requestFactory;
}
 

@PostConstruct
public void init() {

    poolingHttpClientConnectionManager = new PoolingHttpClientConnectionManager();
    poolingHttpClientConnectionManager.setMaxTotal(maxTotalConnections);
    poolingHttpClientConnectionManager
            .setDefaultMaxPerRoute(this.maxConnectionsPerRoute);

    ConnectionKeepAliveStrategy myStrategy = new ConnectionKeepAliveStrategy() {
        public long getKeepAliveDuration(HttpResponse response,
                                         HttpContext context) {
            HeaderElementIterator it = new BasicHeaderElementIterator(
                    response.headerIterator(HTTP.CONN_KEEP_ALIVE));
            while (it.hasNext()) {
                HeaderElement he = it.nextElement();
                String param = he.getName();
                String value = he.getValue();
                if (value != null && param.equalsIgnoreCase("timeout")) {
                    try {
                        return Long.parseLong(value) * 1000;
                    } catch (NumberFormatException ignore) {
                    }
                }
            }
            return keepAliveDurationMilliseconds;
        }

    };
    RequestConfig.Builder requestConfigBuilder = RequestConfig.custom();
    if (isProxyNeeded) {
        requestConfigBuilder.setProxy(new HttpHost(proxyHost, proxyPort));
    }
    RequestConfig requestConfig = requestConfigBuilder
            .setConnectionRequestTimeout(this.connReqTimeoutMilliseconds)
            .setConnectTimeout(connTimeoutMilliseconds)
            .setSocketTimeout(this.socketTimeoutMilliseconds)
            .setStaleConnectionCheckEnabled(false)
            .setRedirectsEnabled(true).setMaxRedirects(maxRedirects)
            .build();
    HttpClientBuilder builder = HttpClientBuilder.create()
            .setDefaultRequestConfig(requestConfig);

    httpClient = builder
            .setConnectionManager(poolingHttpClientConnectionManager)
            .setKeepAliveStrategy(myStrategy).build();
    if (reapInterval > 0 && idleConnectionsTimeout > 0) {
        LOGGER.debug(String
                .format("Initializing idle connection monitor thread with reap interval %s ms and idle connection time out %s ms",
                        reapInterval, idleConnectionsTimeout));
        idcm = new IdleConnectionMonitorThread(
                poolingHttpClientConnectionManager, reapInterval,
                idleConnectionsTimeout);
        idcm.start();
    }
}
 

public static void init() throws RuntimeException {
        try {
            logger.warn(NOTICELINE + " httpUtil init begin " + NOTICELINE);
            SSLContextBuilder sslContextBuilder = new SSLContextBuilder();
//            sslContextBuilder.loadTrustMaterial(null, new TrustSelfSignedStrategy());
            sslContextBuilder.loadTrustMaterial(null,new TrustAnyTrustManager());
            SSLConnectionSocketFactory sslConnectionSocketFactory =
                    new SSLConnectionSocketFactory(
                            sslContextBuilder.build(), SSLConnectionSocketFactory.ALLOW_ALL_HOSTNAME_VERIFIER);

            Registry<ConnectionSocketFactory> registry = RegistryBuilder.<ConnectionSocketFactory>create().
                    register("http", new PlainConnectionSocketFactory()).
                    register("https", sslConnectionSocketFactory).
                    build();


            logger.warn(NOTICELINE + " SSL context init done " + NOTICELINE);

            //init connectionManager , ThreadSafe pooled conMgr
            PoolingHttpClientConnectionManager poolingHttpClientConnectionManager = new PoolingHttpClientConnectionManager(registry);
            poolingHttpClientConnectionManager.setMaxTotal(30);
            poolingHttpClientConnectionManager.setDefaultMaxPerRoute(3);
            //init request config. pooltimeout,sotime,contimeout
            RequestConfig requestConfig = RequestConfig.custom().setConnectionRequestTimeout(POOL_TIMECOUT).setConnectTimeout(CON_TIMEOUT).setSocketTimeout(SO_TIMEOUT).build();
            // begin construct httpclient
            HttpClientBuilder httpClientBuilder = HttpClients.custom();
            httpClientBuilder.setConnectionManager(poolingHttpClientConnectionManager);
            httpClientBuilder.setDefaultRequestConfig(requestConfig);
            httpClientBuilder.setRetryHandler(new HttpRequestRetryHandler() {
                @Override
                public boolean retryRequest(IOException exception, int executionCount, HttpContext context) {
                    if (executionCount >= HTTP_RETRY_COUNT) {
                        return false;
                    }
                    if (exception instanceof InterruptedIOException) {
                        // Timeout
                        logger.warn("httpUtil retry for InterruptIOException");
                        return true;
                    }
                    if (exception instanceof UnknownHostException) {
                        // Unknown host
                        return false;
                    }
                    if (exception instanceof SSLException) {
                        // SSL handshake exception
                        return false;
                    }
                    HttpClientContext clientContext = HttpClientContext.adapt(context);
                    HttpRequest request = clientContext.getRequest();
                    boolean idempotent = !(request instanceof HttpEntityEnclosingRequest);
                    if (idempotent) {
                        // Retry if the request is considered idempotent
                        logger.warn("httpUtil retry for idempotent");
                        return true;
                    }
                    return false;
                }
            });
            logger.warn(NOTICELINE + " poolManager , requestconfig init done " + NOTICELINE);

            httpclient = httpClientBuilder.build();
            logger.warn(NOTICELINE + " httpUtil init done " + NOTICELINE);
        } catch (Exception e) {
            logger.error(NOTICELINE + "httpclient init fail" + NOTICELINE, e);
            throw new RuntimeException(e);
        }
    }