org.apache.http.impl.conn.BasicHttpClientConnectionManager Java Examples

/**
 * This code sets up the httpclient to accept any SSL certificate. The 
 * SSL certificate generated by the instructions above is not correctly
 * signed, so we need ignore the problem. 
 * This code should not, under any circumstances, be allowed anywhere 
 * the production code. 
 * @return
 */
private CloseableHttpClient createClient () {
    try {
        HttpClientBuilder builder = HttpClientBuilder.create();
        SSLContext ctx = SSLContext.getInstance("TLS");
        ctx.init(null, new TrustManager[]{getTrustManager()}, null);
        SSLConnectionSocketFactory scsf = new SSLConnectionSocketFactory(ctx, SSLConnectionSocketFactory.ALLOW_ALL_HOSTNAME_VERIFIER);
        builder.setSSLSocketFactory(scsf);
        Registry<ConnectionSocketFactory> registry = RegistryBuilder.<ConnectionSocketFactory>create()
                .register("https", scsf)
                .build();

        HttpClientConnectionManager ccm = new BasicHttpClientConnectionManager(registry);

        builder.setConnectionManager(ccm);
        return builder.build();
    } catch (Exception ex) {
        ex.printStackTrace();
        return null;
    }
}
 

@Test
public final void givenAcceptingAllCertificates_whenHttpsUrlIsConsumed_thenOk_2() throws GeneralSecurityException {

    final TrustStrategy acceptingTrustStrategy = (cert, authType) -> true;
    final SSLContext sslContext = SSLContexts.custom().loadTrustMaterial(null, acceptingTrustStrategy).build();
    final SSLConnectionSocketFactory sslsf = new SSLConnectionSocketFactory(sslContext, NoopHostnameVerifier.INSTANCE);
    final Registry<ConnectionSocketFactory> socketFactoryRegistry = RegistryBuilder.<ConnectionSocketFactory> create()
            .register("https", sslsf)
            .register("http", new PlainConnectionSocketFactory())
            .build();

    final BasicHttpClientConnectionManager connectionManager = new BasicHttpClientConnectionManager(socketFactoryRegistry);
    final CloseableHttpClient httpClient = HttpClients.custom()
            .setSSLSocketFactory(sslsf)
            .setConnectionManager(connectionManager)
            .build();

    final HttpComponentsClientHttpRequestFactory requestFactory = new HttpComponentsClientHttpRequestFactory(httpClient);
    final ResponseEntity<String> response = new RestTemplate(requestFactory).exchange(urlOverHttps, HttpMethod.GET, null, String.class);
    assertThat(response.getStatusCode().value(), equalTo(200));
}
 

@Test
// @Ignore
// 2.2 IN ARTICLE
public final void whenOpeningLowLevelConnectionWithSocketTimeout_thenNoExceptions() throws InterruptedException, ExecutionException, IOException, HttpException {
    basicConnManager = new BasicHttpClientConnectionManager();
    context = HttpClientContext.create();
    final ConnectionRequest connRequest = basicConnManager.requestConnection(route, null);
    conn = connRequest.get(1000, TimeUnit.SECONDS);
    if (!conn.isOpen()) {
        basicConnManager.connect(conn, route, 1000, context);
    }
    conn.setSocketTimeout(30000);

    assertTrue(conn.getSocketTimeout() == 30000);
    assertTrue(conn.isOpen());
}
 

protected CloseableHttpClient constructHttpClient() throws IOException {
  RequestConfig config = RequestConfig.custom()
                                      .setConnectTimeout(20 * 1000)
                                      .setConnectionRequestTimeout(20 * 1000)
                                      .setSocketTimeout(20 * 1000)
                                      .setMaxRedirects(20)
                                      .build();

  URL                 mmsc          = new URL(apn.getMmsc());
  CredentialsProvider credsProvider = new BasicCredentialsProvider();

  if (apn.hasAuthentication()) {
    credsProvider.setCredentials(new AuthScope(mmsc.getHost(), mmsc.getPort() > -1 ? mmsc.getPort() : mmsc.getDefaultPort()),
                                 new UsernamePasswordCredentials(apn.getUsername(), apn.getPassword()));
  }

  return HttpClients.custom()
                    .setConnectionReuseStrategy(new NoConnectionReuseStrategyHC4())
                    .setRedirectStrategy(new LaxRedirectStrategy())
                    .setUserAgent(SilencePreferences.getMmsUserAgent(context, USER_AGENT))
                    .setConnectionManager(new BasicHttpClientConnectionManager())
                    .setDefaultRequestConfig(config)
                    .setDefaultCredentialsProvider(credsProvider)
                    .build();
}
 

@Override
public void freeManager(HTTPMethod method) {
  synchronized (this) {
    HttpClientConnectionManager mgr = methodmap.get(method);
    if (mgr == null)
      throw new IllegalStateException();
    mgrmap.remove(mgr, method);
    methodmap.remove(method);
    ((BasicHttpClientConnectionManager) mgr).close();
    if (method.executed) {
      // only decrement if method was executed
      this.actualconnections--;
    }
    if (TRACE)
      System.err.println("HTTPConnections: close connection: " + method.hashCode());
  }
}
 

public ProcessConnection(ProcessDirector director, PhantomJSProcess process)
{
	this.process = process;
	
	HttpClientBuilder clientBuilder = HttpClients.custom();
	
	// single connection
	BasicHttpClientConnectionManager connManager = new BasicHttpClientConnectionManager();
	clientBuilder.setConnectionManager(connManager);
	
	RequestConfig requestConfig = RequestConfig.custom()
			// ignore cookies for now
			.setCookieSpec(CookieSpecs.IGNORE_COOKIES)
			.setSocketTimeout(director.getRequestTimeout()).build();
	clientBuilder.setDefaultRequestConfig(requestConfig);
	
	this.httpClient = clientBuilder.build();
}
 

@Test  
public void testServerErrors() throws Exception {
  // tests the behaviour of the client when the server sends an error
  // must use BasicClientConnectionManager to test whether the client is closed correctly
  BasicHttpClientConnectionManager conMgr = new BasicHttpClientConnectionManager();
  Replicator replicator = new HttpReplicator(host, port, ReplicationService.REPLICATION_CONTEXT + "/s1", conMgr);
  ReplicationClient client = new ReplicationClient(replicator, new IndexReplicationHandler(handlerIndexDir, null), 
      new PerSessionDirectoryFactory(clientWorkDir));
  
  try {
    publishRevision(5);

    replicationServlet.setRespondWithError(true);
    expectThrows(Exception.class, client::updateNow);
    
    replicationServlet.setRespondWithError(false);
    client.updateNow(); // now it should work
    reopenReader();
    assertEquals(5, Integer.parseInt(reader.getIndexCommit().getUserData().get("ID"), 16));
    
    client.close();
  } finally {
    replicationServlet.setRespondWithError(false);
  }
}
 

protected CloseableHttpClient createHttpClient(Map<String, Object> parameters)
{
	HttpClientBuilder clientBuilder = HttpClients.custom();
	
	// single connection
	BasicHttpClientConnectionManager connManager = new BasicHttpClientConnectionManager();
	clientBuilder.setConnectionManager(connManager);
	
	// ignore cookies for now
	RequestConfig requestConfig = RequestConfig.custom().setCookieSpec(CookieSpecs.IGNORE_COOKIES).build();
	clientBuilder.setDefaultRequestConfig(requestConfig);
	
	setAuthentication(parameters, clientBuilder);
	
	CloseableHttpClient client = clientBuilder.build();
	return client;
}
 

/**
 * Get a default HttpClient based on the HttpConfiguration object. If required the defaults can 
 * be altered to meet the requirements of the SDK user. The default client does not use connection 
 * pooling and does not reuse connections. Timeouts for connection and socket are taken from the 
 * {@link HttpConfiguration} object.
 * 
 * @param httpConfiguration
 * @return CloseableHttpClient
 */
public static CloseableHttpClient getDefaultClient(HttpConfiguration httpConfiguration) {

    RequestConfig requestConfig = RequestConfig.custom().setConnectTimeout(httpConfiguration.getTimeout())
            .setSocketTimeout(httpConfiguration.getTimeout()).build();

    HttpClientConnectionManager connectionManager = new BasicHttpClientConnectionManager();
    ConnectionReuseStrategy connectionResuseStrategy = new NoConnectionReuseStrategy();

    logger.debug("Creating HttpClient with simple no pooling/no connection reuse default settings.");
    CloseableHttpClient httpClient = HttpClients.custom().setDefaultRequestConfig(requestConfig).setConnectionManager(connectionManager)
            .setConnectionReuseStrategy(connectionResuseStrategy).build();
    return httpClient;
}
 

@Test
// 2.1 IN ARTCLE
public final void whenLowLevelConnectionIsEstablished_thenNoExceptions() throws IOException, HttpException, InterruptedException, ExecutionException {
    basicConnManager = new BasicHttpClientConnectionManager();
    final ConnectionRequest connRequest = basicConnManager.requestConnection(route, null);
    assertTrue(connRequest.get(1000, TimeUnit.SECONDS) != null);
}
 

public TransportConnection(SignalFxEndpoint endpoint, int timeoutMs) {
    super(endpoint, timeoutMs, new BasicHttpClientConnectionManager());

    this.transportRequestConfig = RequestConfig.custom().setSocketTimeout(0)
            .setConnectionRequestTimeout(this.requestConfig.getConnectionRequestTimeout())
            .setConnectTimeout(this.requestConfig.getConnectTimeout())
            .setProxy(this.requestConfig.getProxy()).build();

    log.debug("constructed request config: {}", this.transportRequestConfig.toString());
}
 

public static HttpClientConnectionManager withTimeoutMs(int timeoutMs) {
  BasicHttpClientConnectionManager httpClientConnectionManager = new BasicHttpClientConnectionManager(
      RegistryBuilder.<ConnectionSocketFactory>create()
          .register("http", PlainConnectionSocketFactory.getSocketFactory())
          .register("https", new SSLConnectionSocketFactoryWithTimeout(timeoutMs))
          .build());

  httpClientConnectionManager.setSocketConfig(
      SocketConfig.custom().setSoTimeout(timeoutMs).build());

  return httpClientConnectionManager;
}
 

private void initializeHttpClient() {
  Registry<ConnectionSocketFactory> registry = createConnectionSocketFactoryRegistry();
  HttpClientConnectionManager httpConnectionManager = new BasicHttpClientConnectionManager(registry);
  AuthenticationStrategy authStrategy = new CookieProcessingTargetAuthenticationStrategy();

  httpClient = HttpClients.custom()
    .setConnectionManager(httpConnectionManager)
    .setTargetAuthenticationStrategy(authStrategy)
    .build();
}
 

public B fromConfig(Config config) {
  config = config.withFallback(FALLBACK);
  RequestConfig requestConfig = RequestConfig.copy(RequestConfig.DEFAULT)
                                             .setSocketTimeout(config.getInt(REQUEST_TIME_OUT_MS_KEY))
                                             .setConnectTimeout(config.getInt(CONNECTION_TIME_OUT_MS_KEY))
                                             .setConnectionRequestTimeout(config.getInt(CONNECTION_TIME_OUT_MS_KEY))
                                             .build();

  getHttpClientBuilder().setDefaultRequestConfig(requestConfig);

  if (config.hasPath(STATIC_SVC_ENDPOINT)) {
    try {
      svcEndpoint = Optional.of(new URI(config.getString(AbstractHttpWriterBuilder.STATIC_SVC_ENDPOINT)));
    } catch (URISyntaxException e) {
      throw new RuntimeException(e);
    }
  }

  String connMgrStr = config.getString(HTTP_CONN_MANAGER);
  switch (ConnManager.valueOf(connMgrStr.toUpperCase())) {
    case BASIC:
      httpConnManager = new BasicHttpClientConnectionManager();
      break;
    case POOLING:
      PoolingHttpClientConnectionManager poolingConnMgr = new PoolingHttpClientConnectionManager();
      poolingConnMgr.setMaxTotal(config.getInt(POOLING_CONN_MANAGER_MAX_TOTAL_CONN));
      poolingConnMgr.setDefaultMaxPerRoute(config.getInt(POOLING_CONN_MANAGER_MAX_PER_CONN));
      httpConnManager = poolingConnMgr;
      break;
    default:
      throw new IllegalArgumentException(connMgrStr + " is not supported");
  }
  LOG.info("Using " + httpConnManager.getClass().getSimpleName());
  return typedSelf();
}
 

/**
 * Create a {@link CloseableHttpClient} used to communicate with Azkaban server.
 * Derived class can configure different http client by overriding this method.
 *
 * @return A closeable http client.
 */
private CloseableHttpClient createHttpClient() throws AzkabanClientException {
  try {
  // SSLSocketFactory using custom TrustStrategy that ignores warnings about untrusted certificates
  // Self sign SSL
  SSLContextBuilder sslcb = new SSLContextBuilder();
  sslcb.loadTrustMaterial(null, (TrustStrategy) new TrustSelfSignedStrategy());
  SSLConnectionSocketFactory sslsf = new SSLConnectionSocketFactory(sslcb.build());

  HttpClientBuilder builder = HttpClientBuilder.create();
  RequestConfig requestConfig = RequestConfig.copy(RequestConfig.DEFAULT)
        .setSocketTimeout(10000)
        .setConnectTimeout(10000)
        .setConnectionRequestTimeout(10000)
        .build();

    builder.disableCookieManagement()
        .useSystemProperties()
        .setDefaultRequestConfig(requestConfig)
        .setConnectionManager(new BasicHttpClientConnectionManager())
        .setSSLSocketFactory(sslsf);

    return builder.build();
  } catch (Exception e) {
    throw new AzkabanClientException("HttpClient cannot be created", e);
  }
}
 

private HttpClientConnectionManager getConnectionManager(Builder builder) {
  if (builder.uri.getScheme().equals(NPIPE_SCHEME)) {
    final BasicHttpClientConnectionManager bm = 
        new BasicHttpClientConnectionManager(getSchemeRegistry(builder));
    return bm;
  } else {
    final PoolingHttpClientConnectionManager cm =
        new PoolingHttpClientConnectionManager(getSchemeRegistry(builder));
    // Use all available connections instead of artificially limiting ourselves to 2 per server.
    cm.setMaxTotal(builder.connectionPoolSize);
    cm.setDefaultMaxPerRoute(cm.getMaxTotal());
    return cm;
  }
}
 

public BingSearchResults(String accountKeyFilepath) {
  this.cacheOnly = false;
  this.bingCacheMongoDB = new BingCacheMongoDB(BING_CACHE_HOST, BING_CACHE_MONGO_PORT, BING_CACHE_MONGO_DATABASE);
  this.basicConnManager = new BasicHttpClientConnectionManager();
  try {
    this.accountKey = getAccountKey(accountKeyFilepath);
  } catch (IOException e) {
    String msg = String.format("Bing Searcher could not find account key at %s", accountKeyFilepath);
    LOGGER.error(msg);
    throw new RuntimeException(msg);
  }
}
 

@BeforeClass
public static void setupProxying() {
    DnsResolver dnsResolver = prepareProxiedDnsResolver();
    DefaultSchemePortResolver schemePortResolver = prepareSchemePortResolver();
    BasicHttpClientConnectionManager connManager = prepareConnectionManager(dnsResolver, schemePortResolver);
    HttpClient httpClient = prepareHttpClient(connManager);
    originalHttpClient = (HttpClient) Options.getOption(Option.HTTPCLIENT);
    Unirest.setHttpClient(httpClient);
}
 

/**
* You should call {@link LogDataProvider#obtainUrl()} before
* */
@Override
public InputStream provide() {
    HttpClientConnectionManager connectionManager = new BasicHttpClientConnectionManager();
    HttpClient httpClient = HttpClientBuilder.create()
            .setConnectionManager(connectionManager)
            .build();

    String uri = url + "?s=" + userSessionSource.getUserSession().getId();
    if (downloadFullLog) {
        uri += "&full=true";
    }

    HttpGet httpGet = new HttpGet(uri);

    try {
        HttpResponse httpResponse = httpClient.execute(httpGet);
        int httpStatus = httpResponse.getStatusLine().getStatusCode();
        if (httpStatus == HttpStatus.SC_OK) {
            HttpEntity httpEntity = httpResponse.getEntity();
            if (httpEntity != null) {
                inputStream = httpEntity.getContent();
            } else {
                log.debug("Unable to download log from " + url + "\nHttpEntity is null");

                throw new RuntimeException("Unable to download log from " + url + "\nHttpEntity is null");
            }
        } else {
            log.debug("Unable to download log from " + url + "\n" + httpResponse.getStatusLine());
            throw new RuntimeException("Unable to download log from " + url + "\n" + httpResponse.getStatusLine());
        }
    } catch (IOException e) {
        log.debug("Unable to download log from " + url + "\n" + e);
        throw new RuntimeException(e);
    }

    return inputStream;
}
 

public SimpleHttpClient(SSLContext sslContext, List<String> enabledProtocols, List<String> enabledCiphers,
                        int listenPort, boolean useCompression) {
    HttpClientBuilder builder = HttpClientBuilder.create();
    if (!useCompression) {
        builder.disableContentCompression();
    }
    if (sslContext != null) {
        SSLConnectionSocketFactory sslConnectionFactory = new SSLConnectionSocketFactory(
                sslContext,
                toArray(enabledProtocols),
                toArray(enabledCiphers),
                new DefaultHostnameVerifier());
        builder.setSSLSocketFactory(sslConnectionFactory);

        Registry<ConnectionSocketFactory> registry = RegistryBuilder.<ConnectionSocketFactory>create()
                .register("https", sslConnectionFactory)
                .build();
        builder.setConnectionManager(new BasicHttpClientConnectionManager(registry));
        scheme = "https";
    } else {
        scheme = "http";
    }
    this.delegate = builder.build();
    this.listenPort = listenPort;
}
 

/** Creates a new fetching thread.
 *
 * @param frontier a reference to the {@link Frontier}.
 * @param index  the index of this thread (only for logging purposes).
 */
public FetchingThread(final Frontier frontier, final int index) throws NoSuchAlgorithmException, IllegalArgumentException, IOException {
	setName(this.getClass().getSimpleName() + '-' + index);
	setPriority(Thread.MIN_PRIORITY); // Low priority; there will be thousands of this guys around.
	this.frontier = frontier;

	final BasicHttpClientConnectionManager connManager = new BasicHttpClientConnectionManagerWithAlternateDNS(frontier.rc.dnsResolver);
	connManager.closeIdleConnections(0, TimeUnit.MILLISECONDS);
	connManager.setConnectionConfig(ConnectionConfig.custom().setBufferSize(8 * 1024).build()); // TODO: make this configurable

	cookieStore = new BasicCookieStore();

	final BasicHeader[] headers = {
		new BasicHeader("From", frontier.rc.userAgentFrom),
		new BasicHeader("Accept","text/html,application/xhtml+xml,application/xml;q=0.95,text/*;q=0.9,*/*;q=0.8")
	};

	httpClient = HttpClients.custom()
			.setSSLContext(frontier.rc.acceptAllCertificates ? TRUST_ALL_CERTIFICATES_SSL_CONTEXT : TRUST_SELF_SIGNED_SSL_CONTEXT)
			.setConnectionManager(connManager)
			.setConnectionReuseStrategy(frontier.rc.keepAliveTime == 0 ? NoConnectionReuseStrategy.INSTANCE : DefaultConnectionReuseStrategy.INSTANCE)
			.setUserAgent(frontier.rc.userAgent)
			.setDefaultCookieStore(cookieStore)
			.setDefaultHeaders(ObjectArrayList.wrap(headers))
			.build();
  		fetchData = new FetchData(frontier.rc);
}
 

private static BasicHttpClientConnectionManager prepareConnectionManager(DnsResolver dnsResolver, DefaultSchemePortResolver schemePortResolver) {
    return new BasicHttpClientConnectionManager(
            RegistryBuilder.<ConnectionSocketFactory>create()
                    .register("http", PlainConnectionSocketFactory.getSocketFactory())
                    .register("https", SSLConnectionSocketFactory.getSocketFactory())
                    .build(),
            null,
            schemePortResolver,
            dnsResolver
    );
}
 

@Override
public CloseableHttpClient buildBasicClient() throws IOException {
    BasicHttpClientConnectionManager manager = new BasicHttpClientConnectionManager(registry, null, null, dnsResolver);
    return HttpClients.custom().setConnectionManager(manager).build();
}
 

private static HttpClientConnectionManager getBasicConnectionFactory(String certPath) throws IOException {
    return certPath != null ?
        new BasicHttpClientConnectionManager(getSslFactoryRegistry(certPath)) :
        new BasicHttpClientConnectionManager();
}
 

@Override
public CloseableHttpClient buildBasicClient() throws IOException {
    BasicHttpClientConnectionManager manager = new BasicHttpClientConnectionManager(registry, null, null, dnsResolver);
    return HttpClients.custom().setConnectionManager(manager).build();
}
 

@Test
public final void givenHttpClientIsConfiguredWithCustomConnectionManager_whenExecutingRequest_thenNoExceptions() throws IOException {
    instance = HttpClientBuilder.create().setConnectionManager(new BasicHttpClientConnectionManager()).build();
    response = instance.execute(new HttpGet(SAMPLE_URL));
}
 

public PageManagerImpl() {
    httpClient = HttpClientBuilder
            .create()
            .setConnectionManager(new BasicHttpClientConnectionManager())
            .build();
}
 

private static CloseableHttpClient prepareHttpClient(BasicHttpClientConnectionManager connManager) {
    return HttpClientBuilder.create()
            .setConnectionManager(connManager)
            .build();
}
 

private static HttpClientConnectionManager getBasicConnectionFactory(String certPath) throws IOException {
    return certPath != null ?
        new BasicHttpClientConnectionManager(getSslFactoryRegistry(certPath)) :
        new BasicHttpClientConnectionManager();
}
 

protected InputStream openStreamWithServlet(FileDescriptor fd) throws FileStorageException {
    ClientConfig clientConfig = configuration.getConfig(ClientConfig.class);
    String fileDownloadContext = clientConfig.getFileDownloadContext();

    Object context = serverSelector.initContext();
    String selectedUrl = serverSelector.getUrl(context);
    if (selectedUrl == null) {
        throw new FileStorageException(FileStorageException.Type.IO_EXCEPTION, fd.getName());
    }
    while (true) {
        String url = selectedUrl + fileDownloadContext +
                "?s=" + userSessionSource.getUserSession().getId() +
                "&f=" + fd.getId().toString();

        HttpClientConnectionManager connectionManager = new BasicHttpClientConnectionManager();
        HttpClient httpClient = HttpClientBuilder.create()
                .setConnectionManager(connectionManager)
                .build();

        HttpGet httpGet = new HttpGet(url);

        try {
            HttpResponse httpResponse = httpClient.execute(httpGet);
            int httpStatus = httpResponse.getStatusLine().getStatusCode();
            if (httpStatus == HttpStatus.SC_OK) {
                HttpEntity httpEntity = httpResponse.getEntity();
                if (httpEntity != null) {
                    return httpEntity.getContent();
                } else {
                    log.debug("Unable to download file from {}\nHttpEntity is null", url);
                    selectedUrl = failAndGetNextUrl(context);
                    if (selectedUrl == null) {
                        throw new FileStorageException(FileStorageException.Type.IO_EXCEPTION, fd.getName());
                    }
                }
            } else {
                log.debug("Unable to download file from {}\n{}", url, httpResponse.getStatusLine());
                selectedUrl = failAndGetNextUrl(context);
                if (selectedUrl == null) {
                    throw new FileStorageException(FileStorageException.Type.fromHttpStatus(httpStatus), fd.getName());
                }
            }
        } catch (InterruptedIOException e) {
            log.trace("Downloading has been interrupted");
            throw new FileStorageException(FileStorageException.Type.IO_EXCEPTION, fd.getName(), e);
        } catch (IOException ex) {
            log.debug("Unable to download file from {}\n{}", url, ex);
            selectedUrl = failAndGetNextUrl(context);
            if (selectedUrl == null) {
                throw new FileStorageException(FileStorageException.Type.IO_EXCEPTION, fd.getName(), ex);
            }
        }
    }
}