org.apache.solr.client.solrj.impl.LBHttpSolrClient Java Examples

/**
 * Creates a new Solr client. If solr urls are provided create a LB client (Use simple Http client if only 1 provided),
 * otherwise create a cloud client. That means at least providing zookeeper connection string or Solr urls are required.
 * @param zkConnectionString zookeeper connection string, e.g.: localhost1:2181,localhost2:2181/solr
 * @param solrUrls list of solr urls
 * @param collection name of the Solr collection
 * @param discover use cloud solr client to discover solr nodes, then uses LB client
 * @return created client
 */
public SolrClient createSolrClient(String zkConnectionString, String[] solrUrls, String collection, boolean discover) {
  logger.info("Creating solr client ...");
  logger.info("Using collection=" + collection);
  if (discover && zkConnectionString.length() > 0) {
    final CloudSolrClient discoverNodesClient = createSolrCloudClient(zkConnectionString, collection);
    return createLBClientsWithDiscoverNodes(discoverNodesClient, collection);
  }
  else if (solrUrls != null && solrUrls.length > 0) {
    logger.info("Using lbHttpSolrClient with urls: {}",
      StringUtils.join(appendTo("/" + collection, solrUrls), ","));
    LBHttpSolrClient.Builder builder = new LBHttpSolrClient.Builder();
    builder.withBaseSolrUrls(solrUrls);
    return builder.build();
  } else {
    return createSolrCloudClient(zkConnectionString, collection);
  }
}
 

/**
 * Builds a new SolRDF proxy instance.
 * 
 * @return a new SolRDF proxy instance.
 * @throws UnableToBuildSolRDFClientException in case of build failure.
 */
public SolRDF build() throws UnableToBuildSolRDFClientException {
	if (endpoints.isEmpty()) {
		endpoints.add(DEFAULT_ENDPOINT);
	}

	// FIXME: for DatasetAccessor and (HTTP) query execution service we also need something like LBHttpSolrServer
	final String firstEndpointAddress = endpoints.iterator().next();
	try {
		return new SolRDF(
				DatasetAccessorFactory.createHTTP(
						firstEndpointAddress +
						graphStoreProtocolEndpointPath),
				firstEndpointAddress + sparqlEndpointPath,		
				zkHost != null
					? new CloudSolrClient(zkHost)
					: (endpoints.size() == 1)
						? new HttpSolrClient(endpoints.iterator().next(), httpClient)
						: new LBHttpSolrClient(httpClient, endpoints.toArray(new String[endpoints.size()])));
	} catch (final Exception exception) {
		throw new UnableToBuildSolRDFClientException(exception);
	}	
}
 

private void waitForServer(int maxSeconds, LBHttpSolrClient client, int nServers, String serverName) throws Exception {
  final TimeOut timeout = new TimeOut(maxSeconds, TimeUnit.SECONDS, TimeSource.NANO_TIME);
  while (! timeout.hasTimedOut()) {
    QueryResponse resp;
    try {
      resp = client.query(new SolrQuery("*:*"));
    } catch (Exception e) {
      log.warn("", e);
      continue;
    }
    String name = resp.getResults().get(0).getFieldValue("name").toString();
    if (name.equals(serverName))
      return;
    
    Thread.sleep(500);
  }
}
 

private static LBHttpSolrClient cloneSolr4LBHttpServer(SolrClient solrClient, String core)
		throws MalformedURLException, InstantiationException, IllegalAccessException, IllegalArgumentException,
		InvocationTargetException {
	Map<String, ?> map = readField(solrClient, "aliveServers");

	String[] servers = new String[map.size()];
	int i = 0;
	for (String key : map.keySet()) {
		servers[i] = appendCoreToBaseUrl(key, core);
		i++;
	}

	Boolean isInternalCient = readField(solrClient, "clientIsInternal");

	if (isInternalCient != null && !isInternalCient) {
		HttpClient clientToUse = readAndCloneHttpClient(solrClient);
		return new LBHttpSolrClient(clientToUse, servers);
	}
	return new LBHttpSolrClient(servers);
}
 

private static SolrClient cloneCloudSolrClient(SolrClient solrClient, String core) {
	if (VersionUtil.isSolr3XAvailable() || solrClient == null) {
		return null;
	}

	CloudSolrClient cloudServer = (CloudSolrClient) solrClient;
	String zkHost = readField(solrClient, "zkHost");

	Constructor<? extends SolrClient> constructor = (Constructor<? extends SolrClient>) ClassUtils
			.getConstructorIfAvailable(solrClient.getClass(), String.class, LBHttpSolrClient.class);

	CloudSolrClient clone = (CloudSolrClient) BeanUtils.instantiateClass(constructor, zkHost,
			cloneLBHttpSolrClient(cloudServer.getLbClient(), core));

	if (org.springframework.util.StringUtils.hasText(core)) {
		clone.setDefaultCollection(core);
	}
	return clone;
}
 

private static LBHttpSolrClient cloneLBHttpSolrClient(SolrClient solrClient, String core) {
	if (solrClient == null) {
		return null;
	}

	LBHttpSolrClient clone = null;
	try {
		if (VersionUtil.isSolr3XAvailable()) {
			clone = cloneSolr3LBHttpServer(solrClient, core);
		} else if (VersionUtil.isSolr4XAvailable()) {
			clone = cloneSolr4LBHttpServer(solrClient, core);
		}
	} catch (Exception e) {
		throw new BeanInstantiationException(solrClient.getClass(),
				"Cannot create instace of " + solrClient.getClass() + ". ", e);
	}
	Object o = readField(solrClient, "interval");
	if (o != null) {
		clone.setAliveCheckInterval(Integer.valueOf(o.toString()).intValue());
	}
	return clone;
}
 

/**
 * @param client
 */
protected void destroy(SolrClient client) {
	if (client instanceof HttpSolrClient) {
		((HttpSolrClient) client).shutdown();
	} else if (client instanceof LBHttpSolrClient) {
		((LBHttpSolrClient) client).shutdown();
	} else {
		if (VersionUtil.isSolr4XAvailable()) {
			if (client instanceof CloudSolrClient) {
				((CloudSolrClient) client).shutdown();
			}
		}
	}
}
 

private static LBHttpSolrClient cloneSolr3LBHttpServer(SolrClient solrClient, String core)
		throws MalformedURLException {
	CopyOnWriteArrayList<?> list = readField(solrClient, "aliveServers");

	String[] servers = new String[list.size()];
	for (int i = 0; i < list.size(); i++) {
		servers[i] = appendCoreToBaseUrl(list.get(i).toString(), core);
	}
	return new LBHttpSolrClient(servers);
}
 

private SolrClient createSolrClient() {
    if(logger.isDebugEnabled()) {
        logger.debug("HttpClientBuilder = {}", HttpClientUtil.getHttpClientBuilder(), new Exception());
    }
    final ModifiableSolrParams clientParams = new ModifiableSolrParams();
    SolrClient solrClient = null;

    Mode mode = Mode.parse(configuration.get(SOLR_MODE));
    switch (mode) {
        case CLOUD:
            final CloudSolrClient cloudServer = new CloudSolrClient.Builder()
                    .withLBHttpSolrClientBuilder(
                            new LBHttpSolrClient.Builder()
                                    .withHttpSolrClientBuilder(new HttpSolrClient.Builder().withInvariantParams(clientParams))
                                    .withBaseSolrUrls(configuration.get(HTTP_URLS))
                    )
                    .withZkHost(getZookeeperURLs(configuration))
                    .sendUpdatesOnlyToShardLeaders()
                    .build();
            cloudServer.connect();
            solrClient = cloudServer;
            logger.info("Created solr client using Cloud based configuration.");
            break;
        case HTTP:
            clientParams.add(HttpClientUtil.PROP_ALLOW_COMPRESSION, configuration.get(HTTP_ALLOW_COMPRESSION).toString());
            clientParams.add(HttpClientUtil.PROP_CONNECTION_TIMEOUT, configuration.get(HTTP_CONNECTION_TIMEOUT).toString());
            clientParams.add(HttpClientUtil.PROP_MAX_CONNECTIONS_PER_HOST, configuration.get(HTTP_MAX_CONNECTIONS_PER_HOST).toString());
            clientParams.add(HttpClientUtil.PROP_MAX_CONNECTIONS, configuration.get(HTTP_GLOBAL_MAX_CONNECTIONS).toString());
            final HttpClient client = HttpClientUtil.createClient(clientParams);
            solrClient = new LBHttpSolrClient.Builder()
                    .withHttpClient(client)
                    .withBaseSolrUrls(configuration.get(HTTP_URLS))
                    .build();
            logger.info("Created solr client using HTTP based configuration.");
            break;
        default:
            throw new IllegalArgumentException("Unsupported Solr operation mode: " + mode);
    }
    return solrClient;
}
 

public void testTwoServers() throws Exception {
  try (LBHttpSolrClient client = getLBHttpSolrClient(httpClient, solr[0].getUrl(), solr[1].getUrl())) {
    client.setAliveCheckInterval(500);
    SolrQuery solrQuery = new SolrQuery("*:*");
    QueryResponse resp = null;
    solr[0].jetty.stop();
    solr[0].jetty = null;
    resp = client.query(solrQuery);
    String name = resp.getResults().get(0).getFieldValue("name").toString();
    Assert.assertEquals("solr/collection11", name);
    resp = client.query(solrQuery);
    name = resp.getResults().get(0).getFieldValue("name").toString();
    Assert.assertEquals("solr/collection11", name);
    solr[1].jetty.stop();
    solr[1].jetty = null;
    solr[0].startJetty();
    Thread.sleep(1200);
    try {
      resp = client.query(solrQuery);
    } catch (SolrServerException e) {
      // try again after a pause in case the error is lack of time to start server
      Thread.sleep(3000);
      resp = client.query(solrQuery);
    }
    name = resp.getResults().get(0).getFieldValue("name").toString();
    Assert.assertEquals("solr/collection10", name);
  }
}
 

public void testReliability() throws Exception {
  String[] s = new String[solr.length];
  for (int i = 0; i < solr.length; i++) {
    s[i] = solr[i].getUrl();
  }

  CloseableHttpClient myHttpClient = HttpClientUtil.createClient(null);
  try {
    try (LBHttpSolrClient client = getLBHttpSolrClient(myHttpClient, 500, 500, s)) {
      client.setAliveCheckInterval(500);

      // Kill a server and test again
      solr[1].jetty.stop();
      solr[1].jetty = null;

      // query the servers
      for (String value : s)
        client.query(new SolrQuery("*:*"));

      // Start the killed server once again
      solr[1].startJetty();
      // Wait for the alive check to complete
      waitForServer(30, client, 3, solr[1].name);
    }
  } finally {
    HttpClientUtil.close(myHttpClient);
  }
}
 

private LBHttpSolrClient createLBClientsWithDiscoverNodes(CloudSolrClient discoverClient, String collection) {
  final List<String> baseUrls = waitUntilAvailableBaseUrls(discoverClient, collection);
  final String[] finalBaseUrls = appendTo("/" + collection, baseUrls.toArray(new String[0]));
  logger.info("Following URLs will be used for LB Solr client (collection: '{}'): {}", collection, StringUtils.join(finalBaseUrls));
  return new LBHttpSolrClient.Builder()
    .withBaseSolrUrls(finalBaseUrls)
    .build();
}
 

/**
 * This method <i>may</i> randomize unspecified aspects of the resulting SolrClient.
 * Tests that do not wish to have any randomized behavior should use the 
 * {@link org.apache.solr.client.solrj.impl.LBHttpSolrClient.Builder} class directly
 */ 
public static LBHttpSolrClient getLBHttpSolrClient(HttpClient client, String... solrUrls) {
  return new LBHttpSolrClient.Builder()
      .withHttpClient(client)
      .withBaseSolrUrls(solrUrls)
      .build();
}
 

/**
 * This method <i>may</i> randomize unspecified aspects of the resulting SolrClient.
 * Tests that do not wish to have any randomized behavior should use the 
 * {@link org.apache.solr.client.solrj.impl.LBHttpSolrClient.Builder} class directly
 */ 
public static LBHttpSolrClient getLBHttpSolrClient(HttpClient client, int connectionTimeoutMillis,
    int socketTimeoutMillis, String... solrUrls) {
  return new LBHttpSolrClient.Builder()
      .withHttpClient(client)
      .withBaseSolrUrls(solrUrls)
      .withConnectionTimeout(connectionTimeoutMillis)
      .withSocketTimeout(socketTimeoutMillis)
      .build();
}
 

@SuppressWarnings({"unchecked"})
private int getStatusCode(String token, final String user, final String op, HttpSolrClient client)
throws Exception {
  SolrClient delegationTokenClient;
  if (random().nextBoolean()) delegationTokenClient = new HttpSolrClient.Builder(client.getBaseURL().toString())
      .withKerberosDelegationToken(token)
      .withResponseParser(client.getParser())
      .build();
  else delegationTokenClient = new CloudSolrClient.Builder(Collections.singletonList(cluster.getZkServer().getZkAddress()), Optional.empty())
      .withLBHttpSolrClientBuilder(new LBHttpSolrClient.Builder()
          .withResponseParser(client.getParser())
          .withSocketTimeout(30000).withConnectionTimeout(15000)
          .withHttpSolrClientBuilder(
              new HttpSolrClient.Builder()
                  .withKerberosDelegationToken(token)
          ))
      .build();
  try {
    ModifiableSolrParams p = new ModifiableSolrParams();
    if (user != null) p.set(PseudoAuthenticator.USER_NAME, user);
    if (op != null) p.set("op", op);
    @SuppressWarnings({"rawtypes"})
    SolrRequest req = getAdminRequest(p);
    if (user != null || op != null) {
      Set<String> queryParams = new HashSet<>();
      if (user != null) queryParams.add(PseudoAuthenticator.USER_NAME);
      if (op != null) queryParams.add("op");
      req.setQueryParams(queryParams);
    }
    try {
      delegationTokenClient.request(req, null);
      return HttpStatus.SC_OK;
    } catch (BaseHttpSolrClient.RemoteSolrException re) {
      return re.code();
    }
  } finally {
    delegationTokenClient.close();
  }
}
 

@SuppressWarnings({"unchecked"})
private int getStatusCode(String token, final String user, final String op, HttpSolrClient client)
throws Exception {
  SolrClient delegationTokenClient;
  if (random().nextBoolean()) delegationTokenClient = new HttpSolrClient.Builder(client.getBaseURL().toString())
      .withKerberosDelegationToken(token)
      .withResponseParser(client.getParser())
      .build();
  else delegationTokenClient = new CloudSolrClient.Builder(Collections.singletonList(miniCluster.getZkServer().getZkAddress()), Optional.empty())
      .withLBHttpSolrClientBuilder(new LBHttpSolrClient.Builder()
          .withSocketTimeout(30000).withConnectionTimeout(15000)
          .withResponseParser(client.getParser())
          .withHttpSolrClientBuilder(
              new HttpSolrClient.Builder()
                  .withKerberosDelegationToken(token)
          ))
      .build();
  try {
    ModifiableSolrParams p = new ModifiableSolrParams();
    if (user != null) p.set(USER_PARAM, user);
    if (op != null) p.set("op", op);
    @SuppressWarnings({"rawtypes"})
    SolrRequest req = getAdminRequest(p);
    if (user != null || op != null) {
      Set<String> queryParams = new HashSet<>();
      if (user != null) queryParams.add(USER_PARAM);
      if (op != null) queryParams.add("op");
      req.setQueryParams(queryParams);
    }
    try {
      delegationTokenClient.request(req, null);
      return HttpStatus.SC_OK;
    } catch (BaseHttpSolrClient.RemoteSolrException re) {
      return re.code();
    }
  } finally {
    delegationTokenClient.close();
  }
}
 

private void waitUntilSolrIsUp() throws Exception {
  int maxTries = 30;
  boolean solrIsUp = false;
  String lastExceptionMessage = null;
  for (int tries = 1; tries < maxTries; tries++) {
    try {
      SolrClient solrClient = new LBHttpSolrClient.Builder()
        .withBaseSolrUrl(String.format("http://%s:%d/solr/%s_shard1_replica_n1",
          StoryDataRegistry.INSTANCE.getDockerHost(),
          StoryDataRegistry.INSTANCE.getSolrPort(),
          StoryDataRegistry.INSTANCE.getServiceLogsCollection()))
        .build();
      StoryDataRegistry.INSTANCE.setSolrClient(solrClient);
      SolrPingResponse pingResponse = solrClient.ping();
      if (pingResponse.getStatus() != 0) {
        LOG.info("Solr is not up yet, Retrying... ({} tries)", tries);
        Thread.sleep(2000);
      } else {
        solrIsUp = true;
        LOG.info("Solr is up and running");
        break;
      }
    } catch (Exception e) {
      LOG.info("Error occurred during pinging solr. Retrying... ({} tries)", tries);
      lastExceptionMessage = e.getMessage();
      Thread.sleep(2000);
    }
  }

  if (!solrIsUp) {
    throw new IllegalStateException(String.format("Solr is not up after %d tries. Exception: %s", maxTries, lastExceptionMessage));
  }
}
 

/**
 * This method <i>may</i> randomize unspecified aspects of the resulting SolrClient.
 * Tests that do not wish to have any randomized behavior should use the 
 * {@link org.apache.solr.client.solrj.impl.LBHttpSolrClient.Builder} class directly
 */ 
public static LBHttpSolrClient getLBHttpSolrClient(String... solrUrls) throws MalformedURLException {
  return new LBHttpSolrClient.Builder()
      .withBaseSolrUrls(solrUrls)
      .build();
}
 

/**
 * @param router to route updates with
 * @param col DocCollection for the updates
 * @param urlMap of the cluster
 * @param params params to use
 * @param idField the id field
 * @return a Map of urls to requests
 * @deprecated since 8.0, uses {@link #getRoutesToCollection(DocRouter, DocCollection, Map, ModifiableSolrParams, String)} instead
 */
@Deprecated
public Map<String,LBHttpSolrClient.Req> getRoutes(DocRouter router,
    DocCollection col, Map<String,List<String>> urlMap,
    ModifiableSolrParams params, String idField) {
  return getRoutes(router, col, urlMap, params, idField, LBHttpSolrClient.Req::new);
}