Java Code Examples for org.apache.http.impl.client.HttpClientBuilder#disableContentCompression()

public RequestEntityRestStorageService(final S3Session session, final HttpClientBuilder configuration) {
    super(null, new PreferencesUseragentProvider().get(), null, toProperties(session.getHost(), session.getSignatureVersion()));
    this.session = session;
    this.properties = this.getJetS3tProperties();
    // Client configuration
    configuration.disableContentCompression();
    configuration.setRetryHandler(new S3HttpRequestRetryHandler(this, preferences.getInteger("http.connections.retry")));
    configuration.setRedirectStrategy(new DefaultRedirectStrategy() {
        @Override
        public HttpUriRequest getRedirect(final HttpRequest request, final HttpResponse response, final HttpContext context) throws ProtocolException {
            if(response.containsHeader("x-amz-bucket-region")) {
                final String host = ((HttpUriRequest) request).getURI().getHost();
                if(!StringUtils.equals(session.getHost().getHostname(), host)) {
                    regionEndpointCache.putRegionForBucketName(
                        StringUtils.split(StringUtils.removeEnd(((HttpUriRequest) request).getURI().getHost(), session.getHost().getHostname()), ".")[0],
                        response.getFirstHeader("x-amz-bucket-region").getValue());
                }
            }
            return super.getRedirect(request, response, context);
        }
    });
    this.setHttpClient(configuration.build());
}
 

@Override
public Client connect(final Proxy proxy, final HostKeyCallback key, final LoginCallback prompt) {
    // Always inject new pool to builder on connect because the pool is shutdown on disconnect
    final HttpClientBuilder pool = builder.build(proxy, this, prompt);
    pool.disableContentCompression();
    return new Client(pool.build());
}
 

public SimpleHttpClient(SSLContext sslContext, List<String> enabledProtocols, List<String> enabledCiphers,
                        int listenPort, boolean useCompression) {
    HttpClientBuilder builder = HttpClientBuilder.create();
    if (!useCompression) {
        builder.disableContentCompression();
    }
    if (sslContext != null) {
        SSLConnectionSocketFactory sslConnectionFactory = new SSLConnectionSocketFactory(
                sslContext,
                toArray(enabledProtocols),
                toArray(enabledCiphers),
                new DefaultHostnameVerifier());
        builder.setSSLSocketFactory(sslConnectionFactory);

        Registry<ConnectionSocketFactory> registry = RegistryBuilder.<ConnectionSocketFactory>create()
                .register("https", sslConnectionFactory)
                .build();
        builder.setConnectionManager(new BasicHttpClientConnectionManager(registry));
        scheme = "https";
    } else {
        scheme = "http";
    }
    this.delegate = builder.build();
    this.listenPort = listenPort;
}
 

@Override
protected HttpClient createDefaultHttpClient() {
    logger.info("Bootstrapping http engine with request retry handler...");
    final HttpClientBuilder builder = HttpClientBuilder.create();
    RequestConfig.Builder requestBuilder = RequestConfig.custom();
    if (defaultProxy != null) {
        requestBuilder.setProxy(defaultProxy);
    }
    builder.disableContentCompression();
    builder.setDefaultRequestConfig(requestBuilder.build());

    HttpRequestRetryHandler retryHandler = new StandardHttpRequestRetryHandler();
    builder.setRetryHandler(retryHandler);
    return builder.build();
}
 

@SuppressWarnings("unchecked")
public List<T> load() throws IOException {
  InputStream input = null;

  RequestConfig requestConfig =
      RequestConfig.custom()
          .setConnectTimeout(connectTimeout)
          .setConnectionRequestTimeout(connectTimeout)
          .setSocketTimeout(readTimeout).build();
  HttpClientBuilder httpClientBuilder =
      HttpClientBuilder.create().setDefaultRequestConfig(requestConfig);

  if (! useCompression) {
    LOG.info("Not using compression!");
    httpClientBuilder.disableContentCompression();
  } else {
    LOG.debug("Using compression by default! Trying gzip, deflate");
  }

  CloseableHttpClient httpClient = httpClientBuilder.build();
  HttpGet httpGet = new HttpGet(endpointURL);
  HttpResponse response = httpClient.execute(httpGet);

  try {
    input = response.getEntity().getContent();
    return (List<T>) JSONUtil.readJson(input, typeRef);
  } finally {
    IOUtils.closeQuietly(input);
    IOUtils.closeQuietly(httpClient);
  }
}
 

/**
 * @param proxy    Proxy configuration
 * @param listener Log listener
 * @param prompt   Prompt for proxy credentials
 * @return Builder for HTTP client
 */
public HttpClientBuilder build(final Proxy proxy, final TranscriptListener listener, final LoginCallback prompt) {
    final HttpClientBuilder configuration = HttpClients.custom();
    // Use HTTP Connect proxy implementation provided here instead of
    // relying on internal proxy support in socket factory
    switch(proxy.getType()) {
        case HTTP:
        case HTTPS:
            final HttpHost h = new HttpHost(proxy.getHostname(), proxy.getPort(), Scheme.http.name());
            if(log.isInfoEnabled()) {
                log.info(String.format("Setup proxy %s", h));
            }
            configuration.setProxy(h);
            configuration.setProxyAuthenticationStrategy(new CallbackProxyAuthenticationStrategy(ProxyCredentialsStoreFactory.get(), host, prompt));
            break;
    }
    configuration.setUserAgent(new PreferencesUseragentProvider().get());
    final int timeout = preferences.getInteger("connection.timeout.seconds") * 1000;
    configuration.setDefaultSocketConfig(SocketConfig.custom()
        .setTcpNoDelay(true)
        .setSoTimeout(timeout)
        .build());
    configuration.setDefaultRequestConfig(this.createRequestConfig(timeout));
    configuration.setDefaultConnectionConfig(ConnectionConfig.custom()
        .setBufferSize(preferences.getInteger("http.socket.buffer"))
        .setCharset(Charset.forName(host.getEncoding()))
        .build());
    if(preferences.getBoolean("http.connections.reuse")) {
        configuration.setConnectionReuseStrategy(new DefaultClientConnectionReuseStrategy());
    }
    else {
        configuration.setConnectionReuseStrategy(new NoConnectionReuseStrategy());
    }
    configuration.setRetryHandler(new ExtendedHttpRequestRetryHandler(preferences.getInteger("http.connections.retry")));
    configuration.setServiceUnavailableRetryStrategy(new DisabledServiceUnavailableRetryStrategy());
    if(!preferences.getBoolean("http.compression.enable")) {
        configuration.disableContentCompression();
    }
    configuration.setRequestExecutor(new LoggingHttpRequestExecutor(listener));
    // Always register HTTP for possible use with proxy. Contains a number of protocol properties such as the
    // default port and the socket factory to be used to create the java.net.Socket instances for the given protocol
    configuration.setConnectionManager(this.createConnectionManager(this.createRegistry()));
    configuration.setDefaultAuthSchemeRegistry(RegistryBuilder.<AuthSchemeProvider>create()
        .register(AuthSchemes.BASIC, new BasicSchemeFactory(
            Charset.forName(preferences.getProperty("http.credentials.charset"))))
        .register(AuthSchemes.DIGEST, new DigestSchemeFactory(
            Charset.forName(preferences.getProperty("http.credentials.charset"))))
        .register(AuthSchemes.NTLM, preferences.getBoolean("webdav.ntlm.windows.authentication.enable") && WinHttpClients.isWinAuthAvailable() ?
            new BackportWindowsNTLMSchemeFactory(null) :
            new NTLMSchemeFactory())
        .register(AuthSchemes.SPNEGO, preferences.getBoolean("webdav.ntlm.windows.authentication.enable") && WinHttpClients.isWinAuthAvailable() ?
            new BackportWindowsNegotiateSchemeFactory(null) :
            new SPNegoSchemeFactory())
        .register(AuthSchemes.KERBEROS, new KerberosSchemeFactory()).build());
    return configuration;
}
 

@Override
public ConnectionManagerAwareHttpClient create(HttpClientSettings settings) {
    final HttpClientBuilder builder = HttpClients.custom();
    // Note that it is important we register the original connection manager with the
    // IdleConnectionReaper as it's required for the successful deregistration of managers
    // from the reaper. See https://github.com/aws/aws-sdk-java/issues/722.
    final HttpClientConnectionManager cm = cmFactory.create(settings);

    builder.setRequestExecutor(new SdkHttpRequestExecutor())
            .setKeepAliveStrategy(buildKeepAliveStrategy(settings))
            .disableRedirectHandling()
            .disableAutomaticRetries()
            .setConnectionManager(ClientConnectionManagerFactory.wrap(cm));

    // By default http client enables Gzip compression. So we disable it
    // here.
    // Apache HTTP client removes Content-Length, Content-Encoding and
    // Content-MD5 headers when Gzip compression is enabled. Currently
    // this doesn't affect S3 or Glacier which exposes these headers.
    //
    if (!(settings.useGzip())) {
        builder.disableContentCompression();
    }

    HttpResponseInterceptor itcp = new CRC32ChecksumResponseInterceptor();
    if (settings.calculateCRC32FromCompressedData()) {
        builder.addInterceptorFirst(itcp);
    } else {
        builder.addInterceptorLast(itcp);
    }

    addProxyConfig(builder, settings);

    final ConnectionManagerAwareHttpClient httpClient = new SdkHttpClient(builder.build(), cm);

    if (settings.useReaper()) {
        IdleConnectionReaper.registerConnectionManager(cm, settings.getMaxIdleConnectionTime());
    }

    return httpClient;
}
 

private static HttpClientBuilder setupBuilder(HttpClientBuilder builder, SolrParams config) {
 
  Builder requestConfigBuilder = RequestConfig.custom()
      .setRedirectsEnabled(config.getBool(HttpClientUtil.PROP_FOLLOW_REDIRECTS, false)).setDecompressionEnabled(false)
      .setConnectTimeout(config.getInt(HttpClientUtil.PROP_CONNECTION_TIMEOUT, DEFAULT_CONNECT_TIMEOUT))
      .setSocketTimeout(config.getInt(HttpClientUtil.PROP_SO_TIMEOUT, DEFAULT_SO_TIMEOUT));

  String cpolicy = cookiePolicy;
  if (cpolicy != null) {
    requestConfigBuilder.setCookieSpec(cpolicy);
  }
  
  RequestConfig requestConfig = requestConfigBuilder.build();
  
  HttpClientBuilder retBuilder = builder.setDefaultRequestConfig(requestConfig);

  if (config.getBool(HttpClientUtil.PROP_USE_RETRY, true)) {
    retBuilder = retBuilder.setRetryHandler(new SolrHttpRequestRetryHandler(Integer.getInteger("solr.httpclient.retries", 3)));

  } else {
    retBuilder = retBuilder.setRetryHandler(NO_RETRY);
  }

  final String basicAuthUser = config.get(HttpClientUtil.PROP_BASIC_AUTH_USER);
  final String basicAuthPass = config.get(HttpClientUtil.PROP_BASIC_AUTH_PASS);
  
  if (basicAuthUser != null && basicAuthPass != null) {
    CredentialsProvider credsProvider = new BasicCredentialsProvider();
    credsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials(basicAuthUser, basicAuthPass));
    retBuilder.setDefaultCredentialsProvider(credsProvider);
  }
  
  if (config.getBool(HttpClientUtil.PROP_ALLOW_COMPRESSION, false)) {
    retBuilder.addInterceptorFirst(new UseCompressionRequestInterceptor());
    retBuilder.addInterceptorFirst(new UseCompressionResponseInterceptor());
  } else {
    retBuilder.disableContentCompression();
  }

  return retBuilder;
}
 

public HttpClient createClient() {
    HttpClientBuilder clientBuilder;
    if (connectionParams.useTlsConfigFromEnvironment()) {
        clientBuilder = VespaHttpClientBuilder.create();
    } else {
        clientBuilder = HttpClientBuilder.create();
        if (connectionParams.getSslContext() != null) {
            setSslContext(clientBuilder, connectionParams.getSslContext());
        } else {
            SslContextBuilder builder = new SslContextBuilder();
            if (connectionParams.getPrivateKey() != null && connectionParams.getCertificate() != null) {
                builder.withKeyStore(connectionParams.getPrivateKey(), connectionParams.getCertificate());
            }
            if (connectionParams.getCaCertificates() != null) {
                builder.withTrustStore(connectionParams.getCaCertificates());
            }
            setSslContext(clientBuilder, builder.build());
        }
        if (connectionParams.getHostnameVerifier() != null) {
            clientBuilder.setSSLHostnameVerifier(connectionParams.getHostnameVerifier());
        }
        clientBuilder.setUserTokenHandler(context -> null); // https://stackoverflow.com/a/42112034/1615280
    }
    clientBuilder.setMaxConnPerRoute(1);
    clientBuilder.setMaxConnTotal(1);
    clientBuilder.setConnectionTimeToLive(connectionParams.getConnectionTimeToLive().getSeconds(), TimeUnit.SECONDS);
    clientBuilder.setUserAgent(String.format("vespa-http-client (%s)", Vtag.currentVersion));
    clientBuilder.setDefaultHeaders(Collections.singletonList(new BasicHeader(Headers.CLIENT_VERSION, Vtag.currentVersion)));
    clientBuilder.disableContentCompression();
    // Try to disable the disabling to see if system tests become stable again.
    // clientBuilder.disableAutomaticRetries();
    RequestConfig.Builder requestConfigBuilder = RequestConfig.custom();
    requestConfigBuilder.setSocketTimeout(0);
    if (connectionParams.getProxyHost() != null) {
        requestConfigBuilder.setProxy(new HttpHost(connectionParams.getProxyHost(), connectionParams.getProxyPort()));
    }
    clientBuilder.setDefaultRequestConfig(requestConfigBuilder.build());

    log.fine("Creating HttpClient: " + " ConnectionTimeout "
                    + " SocketTimeout 0 secs "
                    + " proxyhost (can be null) " + connectionParams.getProxyHost()
                    + ":" + connectionParams.getProxyPort()
                    + (useSsl ? " using ssl " : " not using ssl")
    );
    return clientBuilder.build();
}
 

@Override
public HttpClient createHttpClient(FilterConfig filterConfig) {
  final String serviceRole = filterConfig.getInitParameter(PARAMETER_SERVICE_ROLE);
  HttpClientBuilder builder;
  GatewayConfig gatewayConfig = (GatewayConfig) filterConfig.getServletContext().getAttribute(GatewayConfig.GATEWAY_CONFIG_ATTRIBUTE);
  GatewayServices services = (GatewayServices) filterConfig.getServletContext()
      .getAttribute(GatewayServices.GATEWAY_SERVICES_ATTRIBUTE);
  if (gatewayConfig != null && gatewayConfig.isMetricsEnabled()) {
    MetricsService metricsService = services.getService(ServiceType.METRICS_SERVICE);
    builder = metricsService.getInstrumented(HttpClientBuilder.class);
  } else {
    builder = HttpClients.custom();
  }

  // Conditionally set a custom SSLContext
  SSLContext sslContext = createSSLContext(services, filterConfig, serviceRole);
  if(sslContext != null) {
    builder.setSSLSocketFactory(new SSLConnectionSocketFactory(sslContext));
  }

  if (Boolean.parseBoolean(System.getProperty(GatewayConfig.HADOOP_KERBEROS_SECURED))) {
    CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
    credentialsProvider.setCredentials(AuthScope.ANY, new UseJaasCredentials());

    Registry<AuthSchemeProvider> authSchemeRegistry = RegistryBuilder.<AuthSchemeProvider>create()
        .register(AuthSchemes.SPNEGO, new KnoxSpnegoAuthSchemeFactory(true))
        .build();

    builder.setDefaultAuthSchemeRegistry(authSchemeRegistry)
        .setDefaultCookieStore(new HadoopAuthCookieStore(gatewayConfig))
        .setDefaultCredentialsProvider(credentialsProvider);
  } else {
    builder.setDefaultCookieStore(new NoCookieStore());
  }

  builder.setKeepAliveStrategy( DefaultConnectionKeepAliveStrategy.INSTANCE );
  builder.setConnectionReuseStrategy( DefaultConnectionReuseStrategy.INSTANCE );
  builder.setRedirectStrategy( new NeverRedirectStrategy() );
  builder.setRetryHandler( new NeverRetryHandler() );

  int maxConnections = getMaxConnections( filterConfig );
  builder.setMaxConnTotal( maxConnections );
  builder.setMaxConnPerRoute( maxConnections );

  builder.setDefaultRequestConfig(getRequestConfig(filterConfig, serviceRole));

  // See KNOX-1530 for details
  builder.disableContentCompression();

  return builder.build();
}