Java Code Examples for org.apache.solr.client.solrj.impl.HttpClientUtil#createClient()

@SuppressWarnings("unchecked")
private List<LukeResponse> getLukeResponsesForCores(CloudSolrClient solrClient) {
  ZkStateReader zkStateReader = solrClient.getZkStateReader();
  Collection<Slice> activeSlices = zkStateReader.getClusterState().getCollection(solrClient.getDefaultCollection()).getActiveSlices();
  
  List<LukeResponse> lukeResponses = new ArrayList<>();
  for (Slice slice : activeSlices) {
    for (Replica replica : slice.getReplicas()) {
      try (CloseableHttpClient httpClient = HttpClientUtil.createClient(null)) {
        HttpGet request = new HttpGet(replica.getCoreUrl() + LUKE_REQUEST_URL_SUFFIX);
        HttpResponse response = httpClient.execute(request);
        @SuppressWarnings("resource") // JavaBinCodec implements Closeable, yet it can't be closed if it is used for unmarshalling only
        NamedList<Object> lukeData = (NamedList<Object>) new JavaBinCodec().unmarshal(response.getEntity().getContent());
        LukeResponse lukeResponse = new LukeResponse();
        lukeResponse.setResponse(lukeData);
        lukeResponses.add(lukeResponse);
      } catch (IOException e) {
        logger.error("Exception during getting luke responses", e);
      }
    }
  }
  return lukeResponses;
}
 

private static HttpClient createHttpClient(SolrCore core, String httpBasicAuthUser, String httpBasicAuthPassword, boolean useCompression) {
  final ModifiableSolrParams httpClientParams = new ModifiableSolrParams();
  httpClientParams.set(HttpClientUtil.PROP_BASIC_AUTH_USER, httpBasicAuthUser);
  httpClientParams.set(HttpClientUtil.PROP_BASIC_AUTH_PASS, httpBasicAuthPassword);
  httpClientParams.set(HttpClientUtil.PROP_ALLOW_COMPRESSION, useCompression);

  return HttpClientUtil.createClient(httpClientParams, core.getCoreContainer().getUpdateShardHandler().getRecoveryOnlyConnectionManager(), true);
}
 

public static synchronized CloseableHttpClient createClient(String fusionPrincipal) {
  if (logger.isDebugEnabled()) {
    System.setProperty("sun.security.krb5.debug", "true");
  }
  if (fusionPrincipal == null) {
    logger.error("fusion.user (principal) must be set in order to use kerberos!");
  }
  HttpClientUtil.setConfigurer(new FusionKrb5HttpClientConfigurer(fusionPrincipal));
  CloseableHttpClient httpClient = HttpClientUtil.createClient(null);
  HttpClientUtil.setMaxConnections(httpClient, 500);
  HttpClientUtil.setMaxConnectionsPerHost(httpClient, 100);
  return httpClient;
}
 

private HttpClient createHttpClient() {
  // This is bug in Solr, if we don't create a customize HttpClient,
  // UpdateRequest with commit flag will throw an authentication error.
  ModifiableSolrParams params = new ModifiableSolrParams();
  params.set(HttpClientUtil.PROP_BASIC_AUTH_USER, getUsername());
  params.set(HttpClientUtil.PROP_BASIC_AUTH_PASS, getPassword());
  return HttpClientUtil.createClient(params);
}
 

private void protectConfigsHandler() throws Exception {
  String authcPrefix = "/admin/authentication";
  String authzPrefix = "/admin/authorization";

  String securityJson = "{\n" +
      "  'authentication':{\n" +
      "    'class':'solr.BasicAuthPlugin',\n" +
      "    'blockUnknown': false,\n" +
      "    'credentials':{'solr':'orwp2Ghgj39lmnrZOTm7Qtre1VqHFDfwAEzr0ApbN3Y= Ju5osoAqOX8iafhWpPP01E5P+sg8tK8tHON7rCYZRRw='}},\n" +
      "  'authorization':{\n" +
      "    'class':'solr.RuleBasedAuthorizationPlugin',\n" +
      "    'user-role':{'solr':'admin'},\n" +
      "    'permissions':[{'name':'security-edit','role':'admin'}, {'name':'config-edit','role':'admin'}]}}";

  HttpClient cl = null;
  try {
    cl = HttpClientUtil.createClient(null);
    JettySolrRunner randomJetty = solrCluster.getRandomJetty(random());
    String baseUrl = randomJetty.getBaseUrl().toString();

    zkClient().setData("/security.json", securityJson.replaceAll("'", "\"").getBytes(UTF_8), true);
    BasicAuthIntegrationTest.verifySecurityStatus(cl, baseUrl + authcPrefix, "authentication/class", "solr.BasicAuthPlugin", 50);
    BasicAuthIntegrationTest.verifySecurityStatus(cl, baseUrl + authzPrefix, "authorization/class", "solr.RuleBasedAuthorizationPlugin", 50);
  } finally {
    if (cl != null) {
      HttpClientUtil.close(cl);
    }
  }
  Thread.sleep(1000); // TODO: Without a delay, the test fails. Some problem with Authc/Authz framework?
}
 

private void unprotectConfigsHandler() throws Exception {
  HttpClient cl = null;
  try {
    cl = HttpClientUtil.createClient(null);
    zkClient().setData("/security.json", "{}".getBytes(UTF_8), true);
  } finally {
    if (cl != null) {
      HttpClientUtil.close(cl);
    }
  }
  Thread.sleep(1000); // TODO: Without a delay, the test fails. Some problem with Authc/Authz framework?
}
 

@Test
public void testMetrics() throws Exception {
  boolean isUseV2Api = random().nextBoolean();
  String authcPrefix = "/admin/authentication";
  if(isUseV2Api){
    authcPrefix = "/____v2/cluster/security/authentication";
  }
  String baseUrl = cluster.getRandomJetty(random()).getBaseUrl().toString();
  CloseableHttpClient cl = HttpClientUtil.createClient(null);
  
  createCollection(COLLECTION);
  
  // Missing token
  getAndFail(baseUrl + "/" + COLLECTION + "/query?q=*:*", null);
  assertAuthMetricsMinimums(2, 1, 0, 0, 1, 0);
  executeCommand(baseUrl + authcPrefix, cl, "{set-property : { blockUnknown: false}}", jws);
  verifySecurityStatus(cl, baseUrl + authcPrefix, "authentication/blockUnknown", "false", 20, jws);
  // Pass through
  verifySecurityStatus(cl, baseUrl + "/admin/info/key", "key", NOT_NULL_PREDICATE, 20);
  // Now succeeds since blockUnknown=false 
  get(baseUrl + "/" + COLLECTION + "/query?q=*:*", null);
  executeCommand(baseUrl + authcPrefix, cl, "{set-property : { blockUnknown: true}}", null);
  verifySecurityStatus(cl, baseUrl + authcPrefix, "authentication/blockUnknown", "true", 20, jws);

  assertAuthMetricsMinimums(9, 4, 4, 0, 1, 0);
  
  // Wrong Credentials
  getAndFail(baseUrl + "/" + COLLECTION + "/query?q=*:*", jwtTokenWrongSignature);
  assertAuthMetricsMinimums(10, 4, 4, 1, 1, 0);

  // JWT parse error
  getAndFail(baseUrl + "/" + COLLECTION + "/query?q=*:*", "foozzz");
  assertAuthMetricsMinimums(11, 4, 4, 1, 1, 1);
  
  HttpClientUtil.close(cl);
}
 

public static HttpClient getHttpClient() {
  ModifiableSolrParams params = new ModifiableSolrParams();
  params.set(HttpClientUtil.PROP_MAX_CONNECTIONS, 128);
  params.set(HttpClientUtil.PROP_MAX_CONNECTIONS_PER_HOST, 32);
  params.set(HttpClientUtil.PROP_FOLLOW_REDIRECTS, false);
  return HttpClientUtil.createClient(params);
}
 

public static CloseableHttpClient getHttpClient() {
  ModifiableSolrParams params = new ModifiableSolrParams();
  params.set(HttpClientUtil.PROP_MAX_CONNECTIONS, 128);
  params.set(HttpClientUtil.PROP_MAX_CONNECTIONS_PER_HOST, 32);
  params.set(HttpClientUtil.PROP_FOLLOW_REDIRECTS, false);
  return HttpClientUtil.createClient(params);
}
 

private CloseableHttpClient getHttpClient() {
  ModifiableSolrParams params = new ModifiableSolrParams();
  params.set(HttpClientUtil.PROP_MAX_CONNECTIONS, 128);
  params.set(HttpClientUtil.PROP_MAX_CONNECTIONS_PER_HOST, 32);
  CloseableHttpClient httpClient = HttpClientUtil.createClient(params);

  return httpClient;
}
 

@Test
// commented out on: 24-Dec-2018   @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // added 20-Sep-2018
public void testSolrException() throws Throwable {
  // test a connection to a solr server that probably doesn't exist
  // this is a very simple test and most of the test should be considered verified 
  // if the compiler won't let you by without the try/catch
  boolean gotExpectedError = false;
  CloseableHttpClient httpClient = null;
  try {
    // switched to a local address to avoid going out on the net, ns lookup issues, etc.
    // set a 1ms timeout to let the connection fail faster.
    httpClient = HttpClientUtil.createClient(null);
    try (HttpSolrClient client = getHttpSolrClient("http://" + SolrTestCaseJ4.DEAD_HOST_1 + "/solr/", httpClient, 1)) {
      SolrQuery query = new SolrQuery("test123");
      client.query(query);
    }
    httpClient.close();
  } catch (SolrServerException sse) {
    gotExpectedError = true;
    /***
    assertTrue(UnknownHostException.class == sse.getRootCause().getClass()
            //If one is using OpenDNS, then you don't get UnknownHostException, instead you get back that the query couldn't execute
            || (sse.getRootCause().getClass() == SolrException.class && ((SolrException) sse.getRootCause()).code() == 302 && sse.getMessage().equals("Error executing query")));
    ***/
  } finally {
    if (httpClient != null) HttpClientUtil.close(httpClient);
  }
  assertTrue(gotExpectedError);
}
 

public void testReliability() throws Exception {
  String[] s = new String[solr.length];
  for (int i = 0; i < solr.length; i++) {
    s[i] = solr[i].getUrl();
  }

  CloseableHttpClient myHttpClient = HttpClientUtil.createClient(null);
  try {
    try (LBHttpSolrClient client = getLBHttpSolrClient(myHttpClient, 500, 500, s)) {
      client.setAliveCheckInterval(500);

      // Kill a server and test again
      solr[1].jetty.stop();
      solr[1].jetty = null;

      // query the servers
      for (String value : s)
        client.query(new SolrQuery("*:*"));

      // Start the killed server once again
      solr[1].startJetty();
      // Wait for the alive check to complete
      waitForServer(30, client, 3, solr[1].name);
    }
  } finally {
    HttpClientUtil.close(myHttpClient);
  }
}
 

@Override
public void setUp() throws Exception {
  super.setUp();
  httpClient = HttpClientUtil.createClient(null);

  for (int i = 0; i < solr.length; i++) {
    solr[i] = new SolrInstance("solr/collection1" + i, createTempDir("instance-" + i).toFile(), 0);
    solr[i].setUp();
    solr[i].startJetty();
    addDocs(solr[i]);
  }
}
 

/**
 * Creates a new client from URL and username/password, where all operations will use
 * the given auth.
 * <p>
 * DEV NOTE: Implementation must be maintained with the superclass; the default values
 * are taken from {@link HttpSolrClient.Builder} and are subject to change at solrj updates.
 */
@SuppressWarnings("deprecation")
public static HttpSolrClient create(String baseURL, HttpClient httpClient, String solrUsername, String solrPassword,
        Integer maxConnections, Integer maxConnectionsPerHost, Integer connectTimeout, Integer socketTimeout) {

    if (httpClient == null) {
        ModifiableSolrParams params = new ModifiableSolrParams();
        if (maxConnections != null) {
            params.set(HttpClientUtil.PROP_MAX_CONNECTIONS, maxConnections);
        }
        if (maxConnectionsPerHost != null) {
            params.set(HttpClientUtil.PROP_MAX_CONNECTIONS_PER_HOST, maxConnectionsPerHost);
        }
        params.set(HttpClientUtil.PROP_FOLLOW_REDIRECTS, true);
        httpClient = HttpClientUtil.createClient(params);
    }

    // DEV NOTE: the defaults must match what HttpSolrClient.Builder does! Must keep up to date!
    HttpSolrClient client = new ScipioHttpSolrClient(baseURL, httpClient, new BinaryResponseParser(),
            false, new ModifiableSolrParams(), solrUsername, solrPassword);

    // TODO: In Solr 7, these are deprecated and moved to Builder/constructor
    if (connectTimeout != null) {
        client.setConnectionTimeout(connectTimeout);
    }
    if (socketTimeout != null) {
        client.setSoTimeout(socketTimeout);
    }

    return client;
}
 

private SolrClient createSolrClient() {
    if(logger.isDebugEnabled()) {
        logger.debug("HttpClientBuilder = {}", HttpClientUtil.getHttpClientBuilder(), new Exception());
    }
    final ModifiableSolrParams clientParams = new ModifiableSolrParams();
    SolrClient solrClient = null;

    Mode mode = Mode.parse(configuration.get(SOLR_MODE));
    switch (mode) {
        case CLOUD:
            final CloudSolrClient cloudServer = new CloudSolrClient.Builder()
                    .withLBHttpSolrClientBuilder(
                            new LBHttpSolrClient.Builder()
                                    .withHttpSolrClientBuilder(new HttpSolrClient.Builder().withInvariantParams(clientParams))
                                    .withBaseSolrUrls(configuration.get(HTTP_URLS))
                    )
                    .withZkHost(getZookeeperURLs(configuration))
                    .sendUpdatesOnlyToShardLeaders()
                    .build();
            cloudServer.connect();
            solrClient = cloudServer;
            logger.info("Created solr client using Cloud based configuration.");
            break;
        case HTTP:
            clientParams.add(HttpClientUtil.PROP_ALLOW_COMPRESSION, configuration.get(HTTP_ALLOW_COMPRESSION).toString());
            clientParams.add(HttpClientUtil.PROP_CONNECTION_TIMEOUT, configuration.get(HTTP_CONNECTION_TIMEOUT).toString());
            clientParams.add(HttpClientUtil.PROP_MAX_CONNECTIONS_PER_HOST, configuration.get(HTTP_MAX_CONNECTIONS_PER_HOST).toString());
            clientParams.add(HttpClientUtil.PROP_MAX_CONNECTIONS, configuration.get(HTTP_GLOBAL_MAX_CONNECTIONS).toString());
            final HttpClient client = HttpClientUtil.createClient(clientParams);
            solrClient = new LBHttpSolrClient.Builder()
                    .withHttpClient(client)
                    .withBaseSolrUrls(configuration.get(HTTP_URLS))
                    .build();
            logger.info("Created solr client using HTTP based configuration.");
            break;
        default:
            throw new IllegalArgumentException("Unsupported Solr operation mode: " + mode);
    }
    return solrClient;
}
 

private void updateMaximumNumberOfShardsPerCore(Collection<Slice> slices, SolrPropsConfig solrPropsConfig) throws IOException {
  String baseUrl = getRandomBaseUrl(slices);
  if (baseUrl != null) {
    CloseableHttpClient httpClient = HttpClientUtil.createClient(null);
    HttpGet request = new HttpGet(baseUrl + String.format(MODIFY_COLLECTION_QUERY,
      solrPropsConfig.getCollection(), MAX_SHARDS_PER_NODE, calculateMaxShardsPerNode(solrPropsConfig)));
    HttpResponse response = httpClient.execute(request);
    if (response.getStatusLine().getStatusCode() != Response.Status.OK.getStatusCode()) {
      throw new IllegalStateException(String.format("Cannot update collection (%s) - increase max number of nodes per core", solrPropsConfig.getCollection()));
    }
  } else {
    throw new IllegalStateException(String.format("Cannot get any core url for updating collection (%s)", solrPropsConfig.getCollection()));
  }
}
 

/**
 * Create a SolrClient based on the type of Solr specified.
 *
 * @param context
 *          The context
 * @return an HttpSolrClient or CloudSolrClient
 */
protected SolrClient createSolrClient(final ProcessContext context, final String solrLocation) {
    final Integer socketTimeout = context.getProperty(SOLR_SOCKET_TIMEOUT).asTimePeriod(TimeUnit.MILLISECONDS).intValue();
    final Integer connectionTimeout = context.getProperty(SOLR_CONNECTION_TIMEOUT).asTimePeriod(TimeUnit.MILLISECONDS).intValue();
    final Integer maxConnections = context.getProperty(SOLR_MAX_CONNECTIONS).asInteger();
    final Integer maxConnectionsPerHost = context.getProperty(SOLR_MAX_CONNECTIONS_PER_HOST).asInteger();
    final SSLContextService sslContextService = context.getProperty(SSL_CONTEXT_SERVICE).asControllerService(SSLContextService.class);
    final String jaasClientAppName = context.getProperty(JAAS_CLIENT_APP_NAME).getValue();

    final ModifiableSolrParams params = new ModifiableSolrParams();
    params.set(HttpClientUtil.PROP_SO_TIMEOUT, socketTimeout);
    params.set(HttpClientUtil.PROP_CONNECTION_TIMEOUT, connectionTimeout);
    params.set(HttpClientUtil.PROP_MAX_CONNECTIONS, maxConnections);
    params.set(HttpClientUtil.PROP_MAX_CONNECTIONS_PER_HOST, maxConnectionsPerHost);

    // has to happen before the client is created below so that correct configurer would be set if neeeded
    if (!StringUtils.isEmpty(jaasClientAppName)) {
        System.setProperty("solr.kerberos.jaas.appname", jaasClientAppName);
        HttpClientUtil.setConfigurer(new Krb5HttpClientConfigurer());
    }

    final HttpClient httpClient = HttpClientUtil.createClient(params);

    if (sslContextService != null) {
        final SSLContext sslContext = sslContextService.createSSLContext(SSLContextService.ClientAuth.REQUIRED);
        final SSLSocketFactory sslSocketFactory = new SSLSocketFactory(sslContext);
        final Scheme httpsScheme = new Scheme("https", 443, sslSocketFactory);
        httpClient.getConnectionManager().getSchemeRegistry().register(httpsScheme);
    }

    if (SOLR_TYPE_STANDARD.equals(context.getProperty(SOLR_TYPE).getValue())) {
        return new HttpSolrClient(solrLocation, httpClient);
    } else {
        final String collection = context.getProperty(COLLECTION).evaluateAttributeExpressions().getValue();
        final Integer zkClientTimeout = context.getProperty(ZK_CLIENT_TIMEOUT).asTimePeriod(TimeUnit.MILLISECONDS).intValue();
        final Integer zkConnectionTimeout = context.getProperty(ZK_CONNECTION_TIMEOUT).asTimePeriod(TimeUnit.MILLISECONDS).intValue();

        CloudSolrClient cloudSolrClient = new CloudSolrClient(solrLocation, httpClient);
        cloudSolrClient.setDefaultCollection(collection);
        cloudSolrClient.setZkClientTimeout(zkClientTimeout);
        cloudSolrClient.setZkConnectTimeout(zkConnectionTimeout);
        return cloudSolrClient;
    }
}
 

public UpdateShardHandler(UpdateShardHandlerConfig cfg) {
  updateOnlyConnectionManager = new InstrumentedPoolingHttpClientConnectionManager(HttpClientUtil.getSocketFactoryRegistryProvider().getSocketFactoryRegistry());
  recoveryOnlyConnectionManager = new InstrumentedPoolingHttpClientConnectionManager(HttpClientUtil.getSocketFactoryRegistryProvider().getSocketFactoryRegistry());
  defaultConnectionManager = new InstrumentedPoolingHttpClientConnectionManager(HttpClientUtil.getSocketFactoryRegistryProvider().getSocketFactoryRegistry());
  ModifiableSolrParams clientParams = new ModifiableSolrParams();
  if (cfg != null ) {
    updateOnlyConnectionManager.setMaxTotal(cfg.getMaxUpdateConnections());
    updateOnlyConnectionManager.setDefaultMaxPerRoute(cfg.getMaxUpdateConnectionsPerHost());
    recoveryOnlyConnectionManager.setMaxTotal(cfg.getMaxUpdateConnections());
    recoveryOnlyConnectionManager.setDefaultMaxPerRoute(cfg.getMaxUpdateConnectionsPerHost());
    defaultConnectionManager.setMaxTotal(cfg.getMaxUpdateConnections());
    defaultConnectionManager.setDefaultMaxPerRoute(cfg.getMaxUpdateConnectionsPerHost());
    clientParams.set(HttpClientUtil.PROP_SO_TIMEOUT, cfg.getDistributedSocketTimeout());
    clientParams.set(HttpClientUtil.PROP_CONNECTION_TIMEOUT, cfg.getDistributedConnectionTimeout());
    // following is done only for logging complete configuration.
    // The maxConnections and maxConnectionsPerHost have already been specified on the connection manager
    clientParams.set(HttpClientUtil.PROP_MAX_CONNECTIONS, cfg.getMaxUpdateConnections());
    clientParams.set(HttpClientUtil.PROP_MAX_CONNECTIONS_PER_HOST, cfg.getMaxUpdateConnectionsPerHost());
    socketTimeout = cfg.getDistributedSocketTimeout();
    connectionTimeout = cfg.getDistributedConnectionTimeout();
  }
  log.debug("Created default UpdateShardHandler HTTP client with params: {}", clientParams);

  httpRequestExecutor = new InstrumentedHttpRequestExecutor(getMetricNameStrategy(cfg));
  updateHttpListenerFactory = new InstrumentedHttpListenerFactory(getNameStrategy(cfg));
  recoveryOnlyClient = HttpClientUtil.createClient(clientParams, recoveryOnlyConnectionManager, false, httpRequestExecutor);
  defaultClient = HttpClientUtil.createClient(clientParams, defaultConnectionManager, false, httpRequestExecutor);

  Http2SolrClient.Builder updateOnlyClientBuilder = new Http2SolrClient.Builder();
  if (cfg != null) {
    updateOnlyClientBuilder
        .connectionTimeout(cfg.getDistributedConnectionTimeout())
        .idleTimeout(cfg.getDistributedSocketTimeout())
        .maxConnectionsPerHost(cfg.getMaxUpdateConnectionsPerHost());
  }
  updateOnlyClient = updateOnlyClientBuilder.build();
  updateOnlyClient.addListenerFactory(updateHttpListenerFactory);
  Set<String> queryParams = new HashSet<>(2);
  queryParams.add(DistributedUpdateProcessor.DISTRIB_FROM);
  queryParams.add(DistributingUpdateProcessorFactory.DISTRIB_UPDATE_PARAM);
  updateOnlyClient.setQueryParams(queryParams);

  ThreadFactory recoveryThreadFactory = new SolrNamedThreadFactory("recoveryExecutor");
  if (cfg != null && cfg.getMaxRecoveryThreads() > 0) {
    if (log.isDebugEnabled()) {
      log.debug("Creating recoveryExecutor with pool size {}", cfg.getMaxRecoveryThreads());
    }
    recoveryExecutor = ExecutorUtil.newMDCAwareFixedThreadPool(cfg.getMaxRecoveryThreads(), recoveryThreadFactory);
  } else {
    log.debug("Creating recoveryExecutor with unbounded pool");
    recoveryExecutor = ExecutorUtil.newMDCAwareCachedThreadPool(recoveryThreadFactory);
  }
}
 

public MetronSolrClient(String zkHost, Map<String, Object> solrHttpConfig) {
  super(zkHost, HttpClientUtil.createClient(toSolrProps(solrHttpConfig)));
}
 

/**
 * Factory method that returns a {@link HttpClient} instance used for interfacing with a source Solr service.
 * One can override this method to return a differently configured {@link HttpClient} instance.
 * For example configure https and http authentication.
 *
 * @return a {@link HttpClient} instance used for interfacing with a source Solr service
 */
protected HttpClient getHttpClient() {
  return HttpClientUtil.createClient(null);
}